diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 661564a42a..757f3276ad 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -987,14 +987,14 @@ jobs: # Cache the Coder release binaries downloaded by the outdatedCLI / # outdatedAgent e2e tests so most runs skip the flaky GitHub release - # download entirely. The cache key is keyed off the test files that pin - # the downloaded versions, so it invalidates when those versions change. + # download entirely. The cache key is keyed off the file that pins the + # downloaded versions, so it invalidates when those versions change. - name: Restore e2e Coder release binary cache id: coder-e2e-cache uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: /tmp/coder-e2e-cache - key: coder-e2e-cache-${{ runner.os }}-${{ hashFiles('site/e2e/tests/outdatedCLI.spec.ts', 'site/e2e/tests/outdatedAgent.spec.ts') }} + key: coder-e2e-cache-${{ runner.os }}-${{ hashFiles('site/e2e/constants.ts') }} restore-keys: | coder-e2e-cache-${{ runner.os }}- diff --git a/site/e2e/constants.ts b/site/e2e/constants.ts index 4e95d642ea..946f10c022 100644 --- a/site/e2e/constants.ts +++ b/site/e2e/constants.ts @@ -2,6 +2,16 @@ import * as path from "node:path"; export const coderBinary = path.join(__dirname, "./bin/coder"); +// The oldest client and agent versions that Coder still supports. The +// compatibility tests download these release binaries and run them against the +// current server. Changing either value changes which release asset the e2e +// suite fetches, and invalidates the CI cache that stores them. +// +// we no longer support versions prior to Tailnet v2 API support: https://github.com/coder/coder/commit/059e533544a0268acbc8831006b2858ead2f0d8e +export const oldestSupportedCLIVersion = "v2.8.0"; +// we no longer support versions w/o DRPC +export const oldestSupportedAgentVersion = "v2.12.1"; + // Default port from the server export const coderPort = process.env.CODER_E2E_PORT ? Number(process.env.CODER_E2E_PORT) diff --git a/site/e2e/setup/downloadCoderVersions.spec.ts b/site/e2e/setup/downloadCoderVersions.spec.ts new file mode 100644 index 0000000000..fb81c9bccb --- /dev/null +++ b/site/e2e/setup/downloadCoderVersions.spec.ts @@ -0,0 +1,27 @@ +import { test } from "@playwright/test"; +import { oldestSupportedCLIVersion } from "../constants"; +import { downloadCoderVersion } from "../helpers"; + +// Fetching a release binary is network-bound and effectively unbounded: the +// install script downloads an 84 MiB asset and retries with backoff. Doing it +// here rather than inside the compatibility test keeps that time off the test's +// timeout, so a slow GitHub response can no longer surface as an SSH failure. +test("download outdated Coder CLI", async () => { + // Generous because this covers a cold download plus install.sh's retries. + test.setTimeout(300_000); + + // A failure here is deliberately not fatal. Tests in the `tests` project + // depend on this project, and a failing dependency stops all of them from + // running, so a GitHub outage would block the entire suite rather than the + // one test that needs this binary. outdatedCLI.spec.ts calls + // downloadCoderVersion itself, so it retries inline and fails alone. + try { + await downloadCoderVersion(oldestSupportedCLIVersion); + } catch (error) { + console.error( + `Failed to prefetch the Coder ${oldestSupportedCLIVersion} CLI. ` + + "outdatedCLI.spec.ts will download it inline and may time out.", + error, + ); + } +}); diff --git a/site/e2e/tests/outdatedAgent.spec.ts b/site/e2e/tests/outdatedAgent.spec.ts index 9992a5476e..43f3a4685e 100644 --- a/site/e2e/tests/outdatedAgent.spec.ts +++ b/site/e2e/tests/outdatedAgent.spec.ts @@ -1,5 +1,6 @@ import { randomUUID } from "node:crypto"; import { test } from "@playwright/test"; +import { oldestSupportedAgentVersion } from "../constants"; import { createTemplate, createWorkspace, @@ -12,15 +13,12 @@ import { } from "../helpers"; import { beforeCoderTest } from "../hooks"; -// we no longer support versions w/o DRPC -const agentVersion = "v2.12.1"; - test.beforeEach(async ({ page }) => { beforeCoderTest(page); await login(page); }); -test.skip(`ssh with agent ${agentVersion}`, async ({ page }) => { +test.skip(`ssh with agent ${oldestSupportedAgentVersion}`, async ({ page }) => { test.setTimeout(60_000); const token = randomUUID(); @@ -43,7 +41,7 @@ test.skip(`ssh with agent ${agentVersion}`, async ({ page }) => { ], }); const workspaceName = await createWorkspace(page, template); - const binaryPath = await downloadCoderVersion(agentVersion); + const binaryPath = await downloadCoderVersion(oldestSupportedAgentVersion); const agent = await startAgentWithCommand(page, token, binaryPath); const client = await sshIntoWorkspace(page, workspaceName); diff --git a/site/e2e/tests/outdatedCLI.spec.ts b/site/e2e/tests/outdatedCLI.spec.ts index cad37bb05a..a7d07970e3 100644 --- a/site/e2e/tests/outdatedCLI.spec.ts +++ b/site/e2e/tests/outdatedCLI.spec.ts @@ -1,5 +1,6 @@ import { randomUUID } from "node:crypto"; import { test } from "@playwright/test"; +import { oldestSupportedCLIVersion } from "../constants"; import { createTemplate, createWorkspace, @@ -12,15 +13,17 @@ import { } from "../helpers"; import { beforeCoderTest } from "../hooks"; -// we no longer support versions prior to Tailnet v2 API support: https://github.com/coder/coder/commit/059e533544a0268acbc8831006b2858ead2f0d8e -const clientVersion = "v2.8.0"; - test.beforeEach(async ({ page }) => { beforeCoderTest(page); await login(page); }); -test(`ssh with client ${clientVersion}`, async ({ page }) => { +test(`ssh with client ${oldestSupportedCLIVersion}`, async ({ page }) => { + // setup/downloadCoderVersions.spec.ts normally has the binary cached by now, + // leaving this a local-only test. The extra headroom covers the case where + // that prefetch failed and downloadCoderVersion has to fetch it inline. + test.setTimeout(60_000); + const token = randomUUID(); const template = await createTemplate(page, { graph: [ @@ -42,7 +45,7 @@ test(`ssh with client ${clientVersion}`, async ({ page }) => { }); const workspaceName = await createWorkspace(page, template); const agent = await startAgent(page, token); - const binaryPath = await downloadCoderVersion(clientVersion); + const binaryPath = await downloadCoderVersion(oldestSupportedCLIVersion); const client = await sshIntoWorkspace(page, workspaceName, binaryPath); await new Promise((resolve, reject) => {