From 4b755e514aca679b7bd76f78522e40e8d3112a69 Mon Sep 17 00:00:00 2001
From: Jeremy Ruppel <jeremyruppel@users.noreply.github.com>
Date: Mon, 20 Apr 2026 10:21:36 -0400
Subject: [PATCH] fix(site): stabilize app.spec.ts e2e test (#24400)

The `app` e2e test flakes in CI with `waitForEvent("page")` or
`isVisible()` timeouts when opening a workspace app in a new tab.

The test had several timing hazards: the HTTP server address was read
before the socket finished binding, the popup event listener was
registered after the click, and content visibility used a point-in-time
check with the default 5s timeout.

Rewritten to wait for the server to listen, register the popup promise
before clicking, use `getByRole("link")` for a precise locator, and
replace `isVisible()` with a retrying `toBeVisible({ timeout: 30_000 })`
assertion that tolerates the intermediate `about:blank` page and
app-proxy startup delay. The test body is wrapped in `try/finally` to
ensure the HTTP server is always closed.

Fixes coder/internal#577

> Generated by Coder Agents

---

someone wise once said

> and I really don't know what we could do to make it more reliable
other than maybe trying to just rewrite it from scratch.
---
 site/e2e/tests/app.spec.ts | 116 ++++++++++++++++++++++---------------
 1 file changed, 69 insertions(+), 47 deletions(-)

diff --git a/site/e2e/tests/app.spec.ts b/site/e2e/tests/app.spec.ts
index 0d933c833f..b2600f1b30 100644
--- a/site/e2e/tests/app.spec.ts
+++ b/site/e2e/tests/app.spec.ts
@@ -1,6 +1,6 @@
 import { randomUUID } from "node:crypto";
 import * as http from "node:http";
-import { test } from "@playwright/test";
+import { expect, test } from "@playwright/test";
 import {
 	createTemplate,
 	createWorkspace,
@@ -20,54 +20,76 @@ test.beforeEach(async ({ page }) => {
 test("app", async ({ context, page }) => {
 	const appContent = "Hello World";
 	const token = randomUUID();
-	const srv = http
-		.createServer((_req, res) => {
-			res.writeHead(200, { "Content-Type": "text/plain" });
-			res.end(appContent);
-		})
-		.listen(0);
-	const addr = srv.address();
-	if (typeof addr !== "object" || !addr) {
-		throw new Error("Expected addr to be an object");
-	}
 	const appName = "test-app";
-	const template = await createTemplate(page, {
-		graph: [
-			{
-				graph: {
-					resources: [
-						{
-							agents: [
-								{
-									token,
-									apps: [
-										{
-											id: randomUUID(),
-											url: `http://localhost:${addr.port}`,
-											displayName: appName,
-											order: 0,
-											openIn: AppOpenIn.SLIM_WINDOW,
-										},
-									],
-									order: 0,
-								},
-							],
-						},
-					],
-				},
-			},
-		],
+
+	// Start an HTTP server to act as the workspace app backend.
+	const server = http.createServer((_req, res) => {
+		res.writeHead(200, { "Content-Type": "text/plain" });
+		res.end(appContent);
 	});
-	const workspaceName = await createWorkspace(page, template);
-	const agent = await startAgent(page, token);
 
-	// Wait for the web terminal to open in a new tab
-	const pagePromise = context.waitForEvent("page", { timeout: 10_000 });
-	await page.getByText(appName).click({ timeout: 10_000 });
-	const app = await pagePromise;
-	await app.waitForLoadState("domcontentloaded");
-	await app.getByText(appContent).isVisible();
+	// Wait for the server to be fully listening before proceeding.
+	// Using a callback avoids the race where address() is called
+	// before the socket is bound.
+	const port = await new Promise<number>((resolve, reject) => {
+		server.on("error", reject);
+		server.listen(0, () => {
+			const addr = server.address();
+			if (typeof addr !== "object" || !addr) {
+				reject(new Error("Expected address to be an AddressInfo"));
+				return;
+			}
+			resolve(addr.port);
+		});
+	});
 
-	await stopWorkspace(page, workspaceName);
-	await stopAgent(agent);
+	try {
+		const template = await createTemplate(page, {
+			graph: [
+				{
+					graph: {
+						resources: [
+							{
+								agents: [
+									{
+										token,
+										apps: [
+											{
+												id: randomUUID(),
+												url: `http://localhost:${port}`,
+												displayName: appName,
+												order: 0,
+												openIn: AppOpenIn.SLIM_WINDOW,
+											},
+										],
+										order: 0,
+									},
+								],
+							},
+						],
+					},
+				},
+			],
+		});
+		const workspaceName = await createWorkspace(page, template);
+		const agent = await startAgent(page, token);
+
+		// Register the popup listener before clicking so we never miss
+		// the event.
+		const appPagePromise = context.waitForEvent("page");
+		await page.getByRole("link", { name: appName }).click();
+		const appPage = await appPagePromise;
+
+		// SLIM_WINDOW opens about:blank first, then sets location.href
+		// to the proxied app URL. A retrying assertion tolerates the
+		// intermediate blank page and any app-proxy startup delay.
+		await expect(appPage.getByText(appContent)).toBeVisible({
+			timeout: 30_000,
+		});
+
+		await stopWorkspace(page, workspaceName);
+		await stopAgent(agent);
+	} finally {
+		server.close();
+	}
 });