test(site): improve E2E framework (#9438)

.github/workflows/ci.yaml

@@ -554,7 +554,7 @@ jobs:
       - run: pnpm playwright:install
         working-directory: site
 
-      - run: pnpm playwright:test
+      - run: pnpm playwright:test --workers 1
         env:
           DEBUG: pw:api
         working-directory: site

site/e2e/hooks.ts

@@ -0,0 +1,53 @@
+import { Page } from "@playwright/test"
+
+export const beforeCoderTest = async (page: Page) => {
+  // eslint-disable-next-line no-console -- Show everything that was printed with console.log()
+  page.on("console", (msg) => console.log("[onConsole] " + msg.text()))
+
+  page.on("request", (request) => {
+    if (!isApiCall(request.url())) {
+      return
+    }
+
+    // eslint-disable-next-line no-console -- Log HTTP requests for debugging purposes
+    console.log(
+      `[onRequest] method=${request.method()} url=${request.url()} postData=${
+        request.postData() ? request.postData() : ""
+      }`,
+    )
+  })
+  page.on("response", async (response) => {
+    if (!isApiCall(response.url())) {
+      return
+    }
+
+    const shouldLogResponse =
+      !response.url().endsWith("/api/v2/deployment/config") &&
+      !response.url().endsWith("/api/v2/debug/health")
+
+    let responseText = ""
+    try {
+      if (shouldLogResponse) {
+        const buffer = await response.body()
+        responseText = buffer.toString("utf-8")
+        responseText = responseText.replace(/\n$/g, "")
+      } else {
+        responseText = "skipped..."
+      }
+    } catch (error) {
+      responseText = "not_available"
+    }
+
+    // eslint-disable-next-line no-console -- Log HTTP requests for debugging purposes
+    console.log(
+      `[onResponse] url=${response.url()} status=${response.status()} body=${responseText}`,
+    )
+  })
+}
+
+const isApiCall = (urlString: string): boolean => {
+  const url = new URL(urlString)
+  const apiPath = "/api/v2"
+
+  return url.pathname.startsWith(apiPath)
+}

site/e2e/playwright.config.ts

@@ -30,6 +30,7 @@ export default defineConfig({
       timeout: 60000,
     },
   ],
+  reporter: [["./reporter.ts"]],
   use: {
     baseURL: `http://localhost:${port}`,
     video: "retain-on-failure",

site/e2e/reporter.ts

@@ -0,0 +1,53 @@
+import type {
+  FullConfig,
+  Suite,
+  TestCase,
+  TestResult,
+  FullResult,
+  Reporter,
+} from "@playwright/test/reporter"
+
+class CoderReporter implements Reporter {
+  onBegin(config: FullConfig, suite: Suite) {
+    // eslint-disable-next-line no-console -- Helpful for debugging
+    console.log(`Starting the run with ${suite.allTests().length} tests`)
+  }
+
+  onTestBegin(test: TestCase) {
+    // eslint-disable-next-line no-console -- Helpful for debugging
+    console.log(`Starting test ${test.title}`)
+  }
+
+  onStdOut(chunk: string, test: TestCase, _: TestResult): void {
+    // eslint-disable-next-line no-console -- Helpful for debugging
+    console.log(
+      `[stdout] [${test ? test.title : "unknown"}]: ${chunk.replace(
+        /\n$/g,
+        "",
+      )}`,
+    )
+  }
+
+  onStdErr(chunk: string, test: TestCase, _: TestResult): void {
+    // eslint-disable-next-line no-console -- Helpful for debugging
+    console.log(
+      `[stderr] [${test ? test.title : "unknown"}]: ${chunk.replace(
+        /\n$/g,
+        "",
+      )}`,
+    )
+  }
+
+  onTestEnd(test: TestCase, result: TestResult) {
+    // eslint-disable-next-line no-console -- Helpful for debugging
+    console.log(`Finished test ${test.title}: ${result.status}`)
+  }
+
+  onEnd(result: FullResult) {
+    // eslint-disable-next-line no-console -- Helpful for debugging
+    console.log(`Finished the run: ${result.status}`)
+  }
+}
+
+// eslint-disable-next-line no-unused-vars -- Playwright config uses it
+export default CoderReporter

site/e2e/tests/app.spec.ts

@@ -2,6 +2,9 @@ import { test } from "@playwright/test"
 import { randomUUID } from "crypto"
 import * as http from "http"
 import { createTemplate, createWorkspace, startAgent } from "../helpers"
+import { beforeCoderTest } from "../hooks"
+
+test.beforeEach(async ({ page }) => await beforeCoderTest(page))
 
 test("app", async ({ context, page }) => {
   const appContent = "Hello World"

site/e2e/tests/createWorkspace.spec.ts

@@ -1,4 +1,4 @@
-import { test, Page } from "@playwright/test"
+import { test } from "@playwright/test"
 import {
   createTemplate,
   createWorkspace,
@@ -16,11 +16,9 @@ import {
   sixthParameter,
 } from "../parameters"
 import { RichParameter } from "../provisionerGenerated"
+import { beforeCoderTest } from "../hooks"
 
-test.beforeEach(async ({ page }: { page: Page }) => {
-  // eslint-disable-next-line no-console -- For debugging purposes
-  page.on("console", (msg) => console.log("Console: " + msg.text()))
-})
+test.beforeEach(async ({ page }) => await beforeCoderTest(page))
 
 test("create workspace", async ({ page }) => {
   const template = await createTemplate(page, {

site/e2e/tests/gitAuth.spec.ts

@@ -3,6 +3,9 @@ import { gitAuth } from "../constants"
 import { Endpoints } from "@octokit/types"
 import { GitAuthDevice } from "api/typesGenerated"
 import { Awaiter, createServer } from "../helpers"
+import { beforeCoderTest } from "../hooks"
+
+test.beforeEach(async ({ page }) => await beforeCoderTest(page))
 
 // Ensures that a Git auth provider with the device flow functions and completes!
 test("git auth device", async ({ page }) => {

site/e2e/tests/outdatedAgent.spec.ts

@@ -7,9 +7,12 @@ import {
   sshIntoWorkspace,
   startAgentWithCommand,
 } from "../helpers"
+import { beforeCoderTest } from "../hooks"
 
 const agentVersion = "v0.14.0"
 
+test.beforeEach(async ({ page }) => await beforeCoderTest(page))
+
 test("ssh with agent " + agentVersion, async ({ page }) => {
   const token = randomUUID()
   const template = await createTemplate(page, {

site/e2e/tests/outdatedCLI.spec.ts

@@ -7,9 +7,12 @@ import {
   sshIntoWorkspace,
   startAgent,
 } from "../helpers"
+import { beforeCoderTest } from "../hooks"
 
 const clientVersion = "v0.14.0"
 
+test.beforeEach(async ({ page }) => await beforeCoderTest(page))
+
 test("ssh with client " + clientVersion, async ({ page }) => {
   const token = randomUUID()
   const template = await createTemplate(page, {

site/e2e/tests/restartWorkspace.spec.ts

@@ -9,6 +9,9 @@ import {
 
 import { firstBuildOption, secondBuildOption } from "../parameters"
 import { RichParameter } from "../provisionerGenerated"
+import { beforeCoderTest } from "../hooks"
+
+test.beforeEach(async ({ page }) => await beforeCoderTest(page))
 
 test("restart workspace with ephemeral parameters", async ({ page }) => {
   const richParameters: RichParameter[] = [firstBuildOption, secondBuildOption]

site/e2e/tests/updateWorkspace.spec.ts

@@ -1,4 +1,4 @@
-import { test, Page } from "@playwright/test"
+import { test } from "@playwright/test"
 
 import {
   createTemplate,
@@ -18,11 +18,9 @@ import {
   secondBuildOption,
 } from "../parameters"
 import { RichParameter } from "../provisionerGenerated"
+import { beforeCoderTest } from "../hooks"
 
-test.beforeEach(async ({ page }: { page: Page }) => {
-  // eslint-disable-next-line no-console -- For debugging purposes
-  page.on("console", (msg) => console.log("Console: " + msg.text()))
-})
+test.beforeEach(async ({ page }) => await beforeCoderTest(page))
 
 test("update workspace, new optional, immutable parameter added", async ({
   page,

site/e2e/tests/webTerminal.spec.ts

@@ -1,6 +1,9 @@
 import { test } from "@playwright/test"
 import { createTemplate, createWorkspace, startAgent } from "../helpers"
 import { randomUUID } from "crypto"
+import { beforeCoderTest } from "../hooks"
+
+test.beforeEach(async ({ page }) => await beforeCoderTest(page))
 
 test("web terminal", async ({ context, page }) => {
   const token = randomUUID()