From 2377ab66248e02f11283eb635ce3c3bacc3014ff Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Wed, 30 Aug 2023 17:34:45 +0200 Subject: [PATCH] test(site): improve E2E framework (#9438) --- .github/workflows/ci.yaml | 2 +- site/e2e/hooks.ts | 53 +++++++++++++++++++++++++ site/e2e/playwright.config.ts | 1 + site/e2e/reporter.ts | 53 +++++++++++++++++++++++++ site/e2e/tests/app.spec.ts | 3 ++ site/e2e/tests/createWorkspace.spec.ts | 8 ++-- site/e2e/tests/gitAuth.spec.ts | 3 ++ site/e2e/tests/outdatedAgent.spec.ts | 3 ++ site/e2e/tests/outdatedCLI.spec.ts | 3 ++ site/e2e/tests/restartWorkspace.spec.ts | 3 ++ site/e2e/tests/updateWorkspace.spec.ts | 8 ++-- site/e2e/tests/webTerminal.spec.ts | 3 ++ 12 files changed, 132 insertions(+), 11 deletions(-) create mode 100644 site/e2e/hooks.ts create mode 100644 site/e2e/reporter.ts diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 17782be963..384073e754 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -554,7 +554,7 @@ jobs: - run: pnpm playwright:install working-directory: site - - run: pnpm playwright:test + - run: pnpm playwright:test --workers 1 env: DEBUG: pw:api working-directory: site diff --git a/site/e2e/hooks.ts b/site/e2e/hooks.ts new file mode 100644 index 0000000000..b4f26e0ac2 --- /dev/null +++ b/site/e2e/hooks.ts @@ -0,0 +1,53 @@ +import { Page } from "@playwright/test" + +export const beforeCoderTest = async (page: Page) => { + // eslint-disable-next-line no-console -- Show everything that was printed with console.log() + page.on("console", (msg) => console.log("[onConsole] " + msg.text())) + + page.on("request", (request) => { + if (!isApiCall(request.url())) { + return + } + + // eslint-disable-next-line no-console -- Log HTTP requests for debugging purposes + console.log( + `[onRequest] method=${request.method()} url=${request.url()} postData=${ + request.postData() ? request.postData() : "" + }`, + ) + }) + page.on("response", async (response) => { + if (!isApiCall(response.url())) { + return + } + + const shouldLogResponse = + !response.url().endsWith("/api/v2/deployment/config") && + !response.url().endsWith("/api/v2/debug/health") + + let responseText = "" + try { + if (shouldLogResponse) { + const buffer = await response.body() + responseText = buffer.toString("utf-8") + responseText = responseText.replace(/\n$/g, "") + } else { + responseText = "skipped..." + } + } catch (error) { + responseText = "not_available" + } + + // eslint-disable-next-line no-console -- Log HTTP requests for debugging purposes + console.log( + `[onResponse] url=${response.url()} status=${response.status()} body=${responseText}`, + ) + }) +} + +const isApiCall = (urlString: string): boolean => { + const url = new URL(urlString) + const apiPath = "/api/v2" + + return url.pathname.startsWith(apiPath) +} diff --git a/site/e2e/playwright.config.ts b/site/e2e/playwright.config.ts index b25393e529..87c8b9facd 100644 --- a/site/e2e/playwright.config.ts +++ b/site/e2e/playwright.config.ts @@ -30,6 +30,7 @@ export default defineConfig({ timeout: 60000, }, ], + reporter: [["./reporter.ts"]], use: { baseURL: `http://localhost:${port}`, video: "retain-on-failure", diff --git a/site/e2e/reporter.ts b/site/e2e/reporter.ts new file mode 100644 index 0000000000..80fb16f88f --- /dev/null +++ b/site/e2e/reporter.ts @@ -0,0 +1,53 @@ +import type { + FullConfig, + Suite, + TestCase, + TestResult, + FullResult, + Reporter, +} from "@playwright/test/reporter" + +class CoderReporter implements Reporter { + onBegin(config: FullConfig, suite: Suite) { + // eslint-disable-next-line no-console -- Helpful for debugging + console.log(`Starting the run with ${suite.allTests().length} tests`) + } + + onTestBegin(test: TestCase) { + // eslint-disable-next-line no-console -- Helpful for debugging + console.log(`Starting test ${test.title}`) + } + + onStdOut(chunk: string, test: TestCase, _: TestResult): void { + // eslint-disable-next-line no-console -- Helpful for debugging + console.log( + `[stdout] [${test ? test.title : "unknown"}]: ${chunk.replace( + /\n$/g, + "", + )}`, + ) + } + + onStdErr(chunk: string, test: TestCase, _: TestResult): void { + // eslint-disable-next-line no-console -- Helpful for debugging + console.log( + `[stderr] [${test ? test.title : "unknown"}]: ${chunk.replace( + /\n$/g, + "", + )}`, + ) + } + + onTestEnd(test: TestCase, result: TestResult) { + // eslint-disable-next-line no-console -- Helpful for debugging + console.log(`Finished test ${test.title}: ${result.status}`) + } + + onEnd(result: FullResult) { + // eslint-disable-next-line no-console -- Helpful for debugging + console.log(`Finished the run: ${result.status}`) + } +} + +// eslint-disable-next-line no-unused-vars -- Playwright config uses it +export default CoderReporter diff --git a/site/e2e/tests/app.spec.ts b/site/e2e/tests/app.spec.ts index aa69475dc8..9e4d676e38 100644 --- a/site/e2e/tests/app.spec.ts +++ b/site/e2e/tests/app.spec.ts @@ -2,6 +2,9 @@ import { test } from "@playwright/test" import { randomUUID } from "crypto" import * as http from "http" import { createTemplate, createWorkspace, startAgent } from "../helpers" +import { beforeCoderTest } from "../hooks" + +test.beforeEach(async ({ page }) => await beforeCoderTest(page)) test("app", async ({ context, page }) => { const appContent = "Hello World" diff --git a/site/e2e/tests/createWorkspace.spec.ts b/site/e2e/tests/createWorkspace.spec.ts index 8a4937ee01..c78a088348 100644 --- a/site/e2e/tests/createWorkspace.spec.ts +++ b/site/e2e/tests/createWorkspace.spec.ts @@ -1,4 +1,4 @@ -import { test, Page } from "@playwright/test" +import { test } from "@playwright/test" import { createTemplate, createWorkspace, @@ -16,11 +16,9 @@ import { sixthParameter, } from "../parameters" import { RichParameter } from "../provisionerGenerated" +import { beforeCoderTest } from "../hooks" -test.beforeEach(async ({ page }: { page: Page }) => { - // eslint-disable-next-line no-console -- For debugging purposes - page.on("console", (msg) => console.log("Console: " + msg.text())) -}) +test.beforeEach(async ({ page }) => await beforeCoderTest(page)) test("create workspace", async ({ page }) => { const template = await createTemplate(page, { diff --git a/site/e2e/tests/gitAuth.spec.ts b/site/e2e/tests/gitAuth.spec.ts index ab5659f921..7fb5a23d28 100644 --- a/site/e2e/tests/gitAuth.spec.ts +++ b/site/e2e/tests/gitAuth.spec.ts @@ -3,6 +3,9 @@ import { gitAuth } from "../constants" import { Endpoints } from "@octokit/types" import { GitAuthDevice } from "api/typesGenerated" import { Awaiter, createServer } from "../helpers" +import { beforeCoderTest } from "../hooks" + +test.beforeEach(async ({ page }) => await beforeCoderTest(page)) // Ensures that a Git auth provider with the device flow functions and completes! test("git auth device", async ({ page }) => { diff --git a/site/e2e/tests/outdatedAgent.spec.ts b/site/e2e/tests/outdatedAgent.spec.ts index e10c3f6edb..06bf6bb2e2 100644 --- a/site/e2e/tests/outdatedAgent.spec.ts +++ b/site/e2e/tests/outdatedAgent.spec.ts @@ -7,9 +7,12 @@ import { sshIntoWorkspace, startAgentWithCommand, } from "../helpers" +import { beforeCoderTest } from "../hooks" const agentVersion = "v0.14.0" +test.beforeEach(async ({ page }) => await beforeCoderTest(page)) + test("ssh with agent " + agentVersion, async ({ page }) => { const token = randomUUID() const template = await createTemplate(page, { diff --git a/site/e2e/tests/outdatedCLI.spec.ts b/site/e2e/tests/outdatedCLI.spec.ts index 1b09fccf5e..e4d398536b 100644 --- a/site/e2e/tests/outdatedCLI.spec.ts +++ b/site/e2e/tests/outdatedCLI.spec.ts @@ -7,9 +7,12 @@ import { sshIntoWorkspace, startAgent, } from "../helpers" +import { beforeCoderTest } from "../hooks" const clientVersion = "v0.14.0" +test.beforeEach(async ({ page }) => await beforeCoderTest(page)) + test("ssh with client " + clientVersion, async ({ page }) => { const token = randomUUID() const template = await createTemplate(page, { diff --git a/site/e2e/tests/restartWorkspace.spec.ts b/site/e2e/tests/restartWorkspace.spec.ts index eb5d0c99c0..f7de9c624c 100644 --- a/site/e2e/tests/restartWorkspace.spec.ts +++ b/site/e2e/tests/restartWorkspace.spec.ts @@ -9,6 +9,9 @@ import { import { firstBuildOption, secondBuildOption } from "../parameters" import { RichParameter } from "../provisionerGenerated" +import { beforeCoderTest } from "../hooks" + +test.beforeEach(async ({ page }) => await beforeCoderTest(page)) test("restart workspace with ephemeral parameters", async ({ page }) => { const richParameters: RichParameter[] = [firstBuildOption, secondBuildOption] diff --git a/site/e2e/tests/updateWorkspace.spec.ts b/site/e2e/tests/updateWorkspace.spec.ts index ef84f19ba4..6f26dd3fe6 100644 --- a/site/e2e/tests/updateWorkspace.spec.ts +++ b/site/e2e/tests/updateWorkspace.spec.ts @@ -1,4 +1,4 @@ -import { test, Page } from "@playwright/test" +import { test } from "@playwright/test" import { createTemplate, @@ -18,11 +18,9 @@ import { secondBuildOption, } from "../parameters" import { RichParameter } from "../provisionerGenerated" +import { beforeCoderTest } from "../hooks" -test.beforeEach(async ({ page }: { page: Page }) => { - // eslint-disable-next-line no-console -- For debugging purposes - page.on("console", (msg) => console.log("Console: " + msg.text())) -}) +test.beforeEach(async ({ page }) => await beforeCoderTest(page)) test("update workspace, new optional, immutable parameter added", async ({ page, diff --git a/site/e2e/tests/webTerminal.spec.ts b/site/e2e/tests/webTerminal.spec.ts index 8869d352d3..86e2b3eda9 100644 --- a/site/e2e/tests/webTerminal.spec.ts +++ b/site/e2e/tests/webTerminal.spec.ts @@ -1,6 +1,9 @@ import { test } from "@playwright/test" import { createTemplate, createWorkspace, startAgent } from "../helpers" import { randomUUID } from "crypto" +import { beforeCoderTest } from "../hooks" + +test.beforeEach(async ({ page }) => await beforeCoderTest(page)) test("web terminal", async ({ context, page }) => { const token = randomUUID()