ci: overlap core E2E startup with web build preparation (#43116)

This commit is contained in:
Stephen Zhou
2026-09-29 02:54:26 +00:00
committed by GitHub
parent 90f7fbd468
commit f7e5946d81
10 changed files with 492 additions and 39 deletions
@@ -0,0 +1,104 @@
const { execFile } = require('node:child_process')
const { appendFile, mkdtemp, rm } = require('node:fs/promises')
const os = require('node:os')
const path = require('node:path')
const { promisify } = require('node:util')
const waitForBuild = require('./wait.cjs')
const execFileAsync = promisify(execFile)
async function downloadWebBuild({
env = process.env,
execute = execFileAsync,
webDir = path.resolve(__dirname, '../../../web'),
} = {}) {
const repository = env.GITHUB_REPOSITORY
const runId = env.GITHUB_RUN_ID
const runAttempt = Number(env.GITHUB_RUN_ATTEMPT)
if (
!repository ||
!/^\d+$/.test(runId || '') ||
!Number.isInteger(runAttempt) ||
runAttempt < 1 ||
!env.GH_TOKEN
) {
throw new Error(
'Downloading the Web build requires GITHUB_REPOSITORY, GITHUB_RUN_ID, GITHUB_RUN_ATTEMPT, and GH_TOKEN.',
)
}
const run = (command, args, timeout = 60_000) =>
execute(command, args, {
env,
timeout,
maxBuffer: 32 * 1024 * 1024,
})
const api = async (endpoint, paginate = false) => {
const { stdout } = await run('gh', [
'api',
endpoint,
...(paginate ? ['--paginate', '--slurp'] : []),
])
return JSON.parse(stdout)
}
const prefix = `repos/${repository}/actions`
let summary = ''
const artifactId = await waitForBuild({
github: {
rest: {
actions: { listJobsForWorkflowRunAttempt: 'jobs', listWorkflowRunArtifacts: 'artifacts' },
},
paginate: async (kind) => {
const endpoint =
kind === 'jobs'
? `${prefix}/runs/${runId}/attempts/${runAttempt}/jobs?per_page=100`
: `${prefix}/runs/${runId}/artifacts?per_page=100`
const pages = await api(endpoint, true)
return pages.flatMap((page) => page[kind])
},
},
context: { repo: {}, runId },
core: {
info: console.log,
summary: {
addHeading(text) {
summary += `### ${text}\n\n`
return this
},
addRaw(text) {
summary += `${text}\n`
return this
},
async write() {
if (env.GITHUB_STEP_SUMMARY) await appendFile(env.GITHUB_STEP_SUMMARY, summary)
},
},
},
runAttempt,
})
const artifact = await api(`${prefix}/artifacts/${artifactId}`)
const directory = await mkdtemp(path.join(os.tmpdir(), 'dify-e2e-web-build-'))
try {
// Artifact names are immutable within a run. Use the name selected by wait.cjs,
// including its matching upload timestamp when only consumer jobs are rerun.
await run(
'gh',
['run', 'download', runId, '--repo', repository, '--name', artifact.name, '--dir', directory],
300_000,
)
await run('tar', ['-xf', path.join(directory, 'e2e-web-build.tar'), '-C', webDir])
console.log(`Web build artifact ${artifactId} extracted and ready.`)
} finally {
await rm(directory, { force: true, recursive: true })
}
}
module.exports = downloadWebBuild
if (require.main === module) {
downloadWebBuild().catch((error) => {
console.error(error)
process.exitCode = 1
})
}
@@ -0,0 +1,112 @@
const assert = require('node:assert/strict')
const { access } = require('node:fs/promises')
const { test } = require('node:test')
const downloadWebBuild = require('./download.cjs')
function fixture({ downloadFails = false, producerFails = false } = {}) {
const calls = []
const env = {
GITHUB_REPOSITORY: 'owner/repo',
GITHUB_RUN_ID: '123',
GITHUB_RUN_ATTEMPT: '4',
GH_TOKEN: 'test-token',
}
const execute = async (command, args) => {
calls.push({ command, args })
if (command === 'tar') return { stdout: '' }
if (args[0] === 'run') {
if (downloadFails) throw new Error('artifact download failed')
return { stdout: '' }
}
if (args[1].includes('/jobs?')) {
return {
stdout: JSON.stringify([
{
jobs: [
{
name: 'Run Web Full-Stack E2E / Prepare Core E2E Web Build',
status: 'completed',
conclusion: producerFails ? 'failure' : 'success',
steps: [
{
name: 'Upload Web build',
conclusion: 'success',
started_at: '2026-09-22T04:08:08Z',
completed_at: '2026-09-22T04:08:12Z',
},
],
},
],
},
]),
}
}
if (args[1].includes('/artifacts?')) {
return {
stdout: JSON.stringify([
{ artifacts: [] },
{
artifacts: [
{
id: 42,
name: 'core-e2e-web-build-2',
created_at: '2026-09-22T04:08:11Z',
expired: false,
},
],
},
]),
}
}
assert.equal(args[1], 'repos/owner/repo/actions/artifacts/42')
return { stdout: JSON.stringify({ name: 'core-e2e-web-build-2' }) }
}
return { calls, options: { env, execute, webDir: '/test/web' } }
}
test('downloads the producer-selected artifact on consumer-only reruns, including paginated results', async () => {
const { calls, options } = fixture()
await downloadWebBuild(options)
assert.match(calls[0].args[1], /runs\/123\/attempts\/4\/jobs/)
const download = calls.find(({ args }) => args[0] === 'run')
assert.deepEqual(download.args.slice(0, 7), [
'run',
'download',
'123',
'--repo',
'owner/repo',
'--name',
'core-e2e-web-build-2',
])
const extract = calls.at(-1)
assert.equal(extract.command, 'tar')
assert.deepEqual(extract.args.slice(-2), ['-C', '/test/web'])
await assert.rejects(access(download.args.at(-1)), { code: 'ENOENT' })
})
test('does not extract a failed download and removes its temporary directory', async () => {
const { calls, options } = fixture({ downloadFails: true })
await assert.rejects(downloadWebBuild(options), /artifact download failed/)
assert.equal(
calls.some(({ command }) => command === 'tar'),
false,
)
await assert.rejects(access(calls.at(-1).args.at(-1)), { code: 'ENOENT' })
})
test('does not download or extract when the producer fails', async () => {
const { calls, options } = fixture({ producerFails: true })
await assert.rejects(downloadWebBuild(options), /Web build finished with failure/)
assert.equal(calls.length, 1)
})
test('requires CI metadata and credentials before starting preparation', async () => {
const { calls, options } = fixture()
for (const key of Object.keys(options.env)) {
await assert.rejects(
downloadWebBuild({ ...options, env: { ...options.env, [key]: '' } }),
/requires GITHUB/,
)
}
assert.equal(calls.length, 0)
})
+9 -33
View File
@@ -26,7 +26,7 @@ jobs:
persist-credentials: false
- name: Test Web build synchronization
run: node --test .github/actions/wait-e2e-web-build/wait.test.cjs
run: node --test .github/actions/wait-e2e-web-build/*.test.cjs
- name: Compute build source hash
id: build-source
@@ -119,24 +119,12 @@ jobs:
with:
browser: chromium
# Install dependencies while core-build runs, then wait only for its artifact.
- name: Wait for Web build
id: web-build
timeout-minutes: 35
uses: ./.github/actions/wait-e2e-web-build
- name: Download Web build
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
artifact-ids: ${{ steps.web-build.outputs.artifact-id }}
github-token: ${{ github.token }}
- name: Extract Web build
run: tar -xf e2e-web-build.tar -C web
# The runner prepares the backend while waiting for and downloading the build.
- name: Run Chromium full shard
working-directory: ./e2e
run: vp run e2e:full -- --shard ${{ matrix.shard }}/3
env:
GH_TOKEN: ${{ github.token }}
run: vp run e2e:full --download-web-build -- --shard ${{ matrix.shard }}/3
- name: Upload Cucumber report
if: ${{ !cancelled() }}
@@ -183,24 +171,12 @@ jobs:
with:
browser: webkit
# Install dependencies while core-build runs, then wait only for its artifact.
- name: Wait for Web build
id: web-build
timeout-minutes: 35
uses: ./.github/actions/wait-e2e-web-build
- name: Download Web build
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
artifact-ids: ${{ steps.web-build.outputs.artifact-id }}
github-token: ${{ github.token }}
- name: Extract Web build
run: tar -xf e2e-web-build.tar -C web
# Keep the same overlap and teardown ownership for the WebKit smoke lane.
- name: Run WebKit browser smoke
working-directory: ./e2e
run: vp run e2e:full -- --tags '@browser-smoke'
env:
GH_TOKEN: ${{ github.token }}
run: vp run e2e:full --download-web-build -- --tags '@browser-smoke'
- name: Upload Cucumber report
if: ${{ !cancelled() }}
+2
View File
@@ -23,6 +23,8 @@ Run commands from the repository root. Install dependencies and browsers once wi
The runner reuses `web/.next/BUILD_ID` when present. Set `E2E_FORCE_WEB_BUILD=1` to force a frontend rebuild. Use `E2E_BROWSER=webkit` for focused cross-browser runs and `E2E_SLOW_MO=500` with a headed command for local action debugging.
Core CI runs add `--download-web-build` before the Cucumber `--` separator. The runner downloads the current workflow's shared Web artifact while starting middleware and backend services, then joins before starting the frontend. This mode requires GitHub Actions run metadata, `GH_TOKEN` with Actions read access, and the `gh` CLI; ordinary local runs do not use it.
## Runtime Ownership
- `scripts/setup.ts` owns reset, middleware, backend, and frontend startup.
+22 -6
View File
@@ -4,6 +4,7 @@ import path from 'node:path'
import { runCleanupTasks } from '../support/cleanup.ts'
import { assertCucumberScenariosStarted } from '../support/cucumber-messages.ts'
import { startLoggedProcess, stopManagedProcess, waitForUrl } from '../support/process.ts'
import { startWebBuildDownload } from '../support/web-build.ts'
import { startWebServer, stopWebServer } from '../support/web-server.ts'
import { apiURL, baseURL, reuseExistingWebServer } from '../test-env.ts'
import { e2eDir, isMainModule, runCommand } from './common.ts'
@@ -73,8 +74,8 @@ const waitForManagedProcess = async ({
}
}
const main = async () => {
const { forwardArgs, full, headed, seed, seedOnly } = parseRunOptions(process.argv.slice(2))
export const runCucumber = async (argv: string[]) => {
const { downloadWebBuild, forwardArgs, full, headed, seed, seedOnly } = parseRunOptions(argv)
const startAgentBackendForRun = shouldStartManagedAgentBackend()
const cucumberReportDir = path.join(e2eDir, 'cucumber-report')
const logDir = path.join(e2eDir, '.logs')
@@ -83,12 +84,14 @@ const main = async () => {
let difyAgentProcess: ManagedProcess | undefined
let middlewareStarted = false
let shellctlProcess: ManagedProcess | undefined
let webBuild: Awaited<ReturnType<typeof startWebBuildDownload>> | undefined
let cleanupPromise: Promise<void> | undefined
const cleanup = async () => {
if (!cleanupPromise) {
cleanupPromise = (async () => {
const cleanupErrors = await runCleanupTasks([
{ label: 'Stop Web build download', run: () => stopManagedProcess(webBuild?.process) },
{ label: 'Stop web server', run: stopWebServer },
{ label: 'Stop celery worker', run: () => stopManagedProcess(celeryProcess) },
{ label: 'Stop API server', run: () => stopManagedProcess(apiProcess) },
@@ -120,15 +123,18 @@ const main = async () => {
try {
if (full) await resetState()
if (!seedOnly) await rm(cucumberReportDir, { force: true, recursive: true })
await mkdir(logDir, { recursive: true })
// The runner owns both preparation branches and tears them down on failure.
// Download the shared build while middleware and backend services start.
if (downloadWebBuild) webBuild = await startWebBuildDownload(logDir)
if (full) {
middlewareStarted = true
await startMiddleware()
}
if (!seedOnly) await rm(cucumberReportDir, { force: true, recursive: true })
await mkdir(logDir, { recursive: true })
if (startAgentBackendForRun) {
shellctlProcess = await startLoggedProcess({
command: process.execPath,
@@ -182,6 +188,16 @@ const main = async () => {
logFilePath: path.join(logDir, 'cucumber-celery.log'),
})
if (webBuild) {
const waitStarted = Date.now()
console.log('Backend services are ready; waiting for the shared Web build.')
const error = await webBuild.completed
if (error) throw error
console.log(
`Shared Web build ready after ${Math.round((Date.now() - waitStarted) / 1000)}s of additional waiting.`,
)
}
await startWebServer({
baseURL,
command: process.execPath,
@@ -229,7 +245,7 @@ const main = async () => {
}
if (isMainModule(import.meta.url)) {
void main().catch((error) => {
void runCucumber(process.argv.slice(2)).catch((error) => {
console.error(error instanceof Error ? error.message : String(error))
process.exit(1)
})
+9
View File
@@ -1,6 +1,7 @@
import type { SeedOptions } from './seed-runner.ts'
export type RunOptions = {
downloadWebBuild: boolean
forwardArgs: string[]
full: boolean
headed: boolean
@@ -18,6 +19,7 @@ const readOptionValue = (argv: string[], index: number, option: string) => {
export const parseRunOptions = (argv: string[]): RunOptions => {
let allowBlocked = false
let dryRun = false
let downloadWebBuild = false
let full = false
let headed = false
let pack = 'agent-v2'
@@ -38,6 +40,11 @@ export const parseRunOptions = (argv: string[]): RunOptions => {
continue
}
if (arg === '--download-web-build') {
downloadWebBuild = true
continue
}
if (arg === '--headed') {
headed = true
continue
@@ -89,8 +96,10 @@ export const parseRunOptions = (argv: string[]): RunOptions => {
if (!shouldSeed && (allowBlocked || dryRun || pack !== 'agent-v2'))
throw new Error('Seed options require --seed-only or --profile.')
if (dryRun && !seedOnly) throw new Error('--dry-run requires --seed-only.')
if (downloadWebBuild && !full) throw new Error('--download-web-build requires --full.')
return {
downloadWebBuild,
forwardArgs,
full,
headed,
+30
View File
@@ -0,0 +1,30 @@
import path from 'node:path'
import { rootDir } from '../scripts/common.ts'
import { startLoggedProcess } from './process.ts'
export const startWebBuildDownload = async (logDir: string) => {
const managedProcess = await startLoggedProcess({
command: process.execPath,
args: ['.github/actions/wait-e2e-web-build/download.cjs'],
cwd: rootDir,
label: 'Web build download',
logFilePath: path.join(logDir, 'cucumber-web-build.log'),
})
// Capture failures immediately, even while the runner is still starting the API.
// Resolving with the error avoids an unhandled rejection before the join point.
const completed = new Promise<Error | undefined>((resolve) => {
managedProcess.childProcess.once('error', resolve)
managedProcess.childProcess.once('close', (code) => {
resolve(
code === 0
? undefined
: new Error(
`Web build download failed (${code ?? 'signal'}). See ${managedProcess.logFilePath}.`,
),
)
})
})
return { process: managedProcess, completed }
}
+140
View File
@@ -0,0 +1,140 @@
import type { ManagedProcess } from '../support/process.ts'
import { EventEmitter } from 'node:events'
import { readFile } from 'node:fs/promises'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vite-plus/test'
import { runCommand } from '../scripts/common.ts'
import { runCucumber } from '../scripts/run-cucumber.ts'
import { resetState, startMiddleware, stopMiddleware } from '../scripts/setup.ts'
import { startLoggedProcess, stopManagedProcess, waitForUrl } from '../support/process.ts'
import { startWebBuildDownload } from '../support/web-build.ts'
import { startWebServer, stopWebServer } from '../support/web-server.ts'
vi.mock('node:fs/promises', () => ({ mkdir: vi.fn(), readFile: vi.fn(), rm: vi.fn() }))
vi.mock('../scripts/env-register.ts', () => ({}))
vi.mock('../scripts/seed-runner.ts', () => ({ runSeed: vi.fn() }))
vi.mock('../scripts/common.ts', () => ({
e2eDir: '/e2e',
isMainModule: () => false,
runCommand: vi.fn(),
}))
vi.mock('../scripts/setup.ts', () => ({
resetState: vi.fn(),
startMiddleware: vi.fn(),
stopMiddleware: vi.fn(),
}))
vi.mock('../support/process.ts', () => ({
startLoggedProcess: vi.fn(),
stopManagedProcess: vi.fn(),
waitForUrl: vi.fn(),
}))
vi.mock('../support/web-build.ts', () => ({ startWebBuildDownload: vi.fn() }))
vi.mock('../support/web-server.ts', () => ({ startWebServer: vi.fn(), stopWebServer: vi.fn() }))
vi.mock('../test-env.ts', () => ({
apiURL: 'http://localhost:5001',
baseURL: 'http://localhost:3000',
reuseExistingWebServer: false,
}))
const managedProcess = () =>
({
childProcess: Object.assign(new EventEmitter(), { exitCode: null }),
logFilePath: '/e2e/.logs/test.log',
label: 'test process',
}) as unknown as ManagedProcess
describe('core E2E preparation', () => {
beforeEach(() => {
vi.resetAllMocks()
vi.stubEnv('E2E_START_AGENT_BACKEND', '')
vi.mocked(startLoggedProcess).mockImplementation(async () => managedProcess())
vi.mocked(readFile).mockResolvedValue('{"testCaseStarted":{}}')
vi.mocked(runCommand).mockResolvedValue({ exitCode: 0, stdout: '', stderr: '' })
})
afterEach(() => {
vi.unstubAllEnvs()
process.exitCode = undefined
})
it('starts the backend before the build is ready and joins before starting Web or Cucumber', async () => {
let finishBuild!: () => void
const completed = new Promise<Error | undefined>((resolve) => {
finishBuild = () => resolve(undefined)
})
const downloadProcess = managedProcess()
vi.mocked(startWebBuildDownload).mockResolvedValue({
process: downloadProcess,
completed,
})
const run = runCucumber(['--full', '--download-web-build', '--', '--shard', '2/3'])
await vi.waitFor(() => expect(startLoggedProcess).toHaveBeenCalledTimes(2))
expect(resetState).toHaveBeenCalledBefore(vi.mocked(startWebBuildDownload))
expect(startWebBuildDownload).toHaveBeenCalledBefore(vi.mocked(startMiddleware))
expect(waitForUrl).toHaveBeenCalledWith('http://localhost:5001/health', 180_000, 1_000)
expect(startWebServer).not.toHaveBeenCalled()
expect(runCommand).not.toHaveBeenCalled()
finishBuild()
await run
expect(startWebServer).toHaveBeenCalledBefore(vi.mocked(runCommand))
expect(runCommand).toHaveBeenCalledWith(
expect.objectContaining({
args: expect.arrayContaining(['--shard', '2/3']),
}),
)
expect(stopManagedProcess).toHaveBeenCalledWith(downloadProcess)
expect(stopWebServer).toHaveBeenCalledOnce()
expect(stopMiddleware).toHaveBeenCalledOnce()
})
it('fails closed and tears down the backend when artifact preparation fails', async () => {
const error = new Error('Web build producer failed')
vi.mocked(startWebBuildDownload).mockResolvedValue({
process: managedProcess(),
completed: Promise.resolve(error),
})
await expect(runCucumber(['--full', '--download-web-build'])).rejects.toThrow(error)
expect(startWebServer).not.toHaveBeenCalled()
expect(runCommand).not.toHaveBeenCalled()
expect(stopManagedProcess).toHaveBeenCalledTimes(5)
expect(stopMiddleware).toHaveBeenCalledOnce()
})
it('stops an unfinished artifact download when backend startup fails', async () => {
const downloadProcess = managedProcess()
vi.mocked(startWebBuildDownload).mockResolvedValue({
process: downloadProcess,
completed: new Promise(() => {}),
})
vi.mocked(startMiddleware).mockRejectedValue(new Error('middleware failed'))
await expect(runCucumber(['--full', '--download-web-build'])).rejects.toThrow(
'middleware failed',
)
expect(stopManagedProcess).toHaveBeenCalledWith(downloadProcess)
expect(stopMiddleware).toHaveBeenCalledOnce()
expect(startWebServer).not.toHaveBeenCalled()
})
it('keeps local full runs independent of GitHub artifact preparation', async () => {
await runCucumber(['--full'])
expect(startWebBuildDownload).not.toHaveBeenCalled()
expect(startWebServer).toHaveBeenCalledOnce()
expect(runCommand).toHaveBeenCalledOnce()
})
it('preserves Cucumber failure and empty-selection gates after preparation', async () => {
vi.mocked(startWebBuildDownload).mockResolvedValue({
process: managedProcess(),
completed: Promise.resolve(undefined),
})
vi.mocked(runCommand).mockResolvedValueOnce({ exitCode: 1, stdout: '', stderr: '' })
await runCucumber(['--full', '--download-web-build'])
expect(process.exitCode).toBe(1)
vi.mocked(readFile).mockResolvedValue('')
await expect(runCucumber(['--full', '--download-web-build'])).rejects.toThrow('zero scenarios')
})
})
+14
View File
@@ -4,6 +4,7 @@ import { parseRunOptions, shouldStartManagedAgentBackend } from '../scripts/run-
describe('E2E run options', () => {
it('forwards Cucumber arguments without requesting seed data', () => {
expect(parseRunOptions(['--tags', '@smoke'])).toEqual({
downloadWebBuild: false,
forwardArgs: ['--tags', '@smoke'],
full: false,
headed: false,
@@ -41,6 +42,19 @@ describe('E2E run options', () => {
'--dry-run requires --seed-only.',
)
})
it('selects a shared CI build without forwarding the option to Cucumber', () => {
expect(
parseRunOptions(['--full', '--download-web-build', '--', '--shard', '2/3']),
).toMatchObject({
downloadWebBuild: true,
full: true,
forwardArgs: ['--shard', '2/3'],
})
expect(() => parseRunOptions(['--download-web-build'])).toThrow(
'--download-web-build requires --full.',
)
})
})
describe('managed Agent backend selection', () => {
+50
View File
@@ -0,0 +1,50 @@
import type { ManagedProcess } from '../support/process.ts'
import { EventEmitter } from 'node:events'
import { beforeEach, describe, expect, it, vi } from 'vite-plus/test'
import { startLoggedProcess } from '../support/process.ts'
import { startWebBuildDownload } from '../support/web-build.ts'
vi.mock('../support/process.ts', () => ({ startLoggedProcess: vi.fn() }))
describe('managed Web build download', () => {
beforeEach(() => {
vi.resetAllMocks()
})
const start = async () => {
const childProcess = new EventEmitter()
const process = {
childProcess,
logFilePath: '/logs/cucumber-web-build.log',
} as unknown as ManagedProcess
vi.mocked(startLoggedProcess).mockResolvedValue(process)
return { childProcess, download: await startWebBuildDownload('/logs'), process }
}
it('keeps the process available for cleanup and waits for successful completion', async () => {
const { childProcess, download, process } = await start()
expect(download.process).toBe(process)
childProcess.emit('close', 0)
await expect(download.completed).resolves.toBeUndefined()
})
it.each([1, null])(
'records failed or signalled completion (%s) before the runner joins',
async (code) => {
const { childProcess, download } = await start()
childProcess.emit('close', code)
await expect(download.completed).resolves.toEqual(
expect.objectContaining({
message: expect.stringContaining('/logs/cucumber-web-build.log'),
}),
)
},
)
it('captures spawn errors without emitting an unhandled rejection during backend startup', async () => {
const { childProcess, download } = await start()
const error = new Error('spawn failed')
childProcess.emit('error', error)
await expect(download.completed).resolves.toBe(error)
})
})