test(e2e): harden behavior coverage and CI gates (#39043)

Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>

e2e/scripts/env.ts

@@ -81,6 +81,7 @@ export const validateE2eEnv = () =>
       E2E_AGENT_DECISION_MODEL_TYPE: process.env.E2E_AGENT_DECISION_MODEL_TYPE,
       E2E_API_URL: process.env.E2E_API_URL,
       E2E_BASE_URL: process.env.E2E_BASE_URL,
+      E2E_BROWSER: process.env.E2E_BROWSER,
       E2E_BROKEN_MODEL_NAME: process.env.E2E_BROKEN_MODEL_NAME,
       E2E_BROKEN_MODEL_PROVIDER: process.env.E2E_BROKEN_MODEL_PROVIDER,
       E2E_BROKEN_MODEL_TYPE: process.env.E2E_BROKEN_MODEL_TYPE,
@@ -123,6 +124,7 @@ export const validateE2eEnv = () =>
       E2E_AGENT_DECISION_MODEL_TYPE: z.string().min(1).optional(),
       E2E_API_URL: z.url().optional(),
       E2E_BASE_URL: z.url().optional(),
+      E2E_BROWSER: z.enum(['chromium', 'webkit']).optional(),
       E2E_BROKEN_MODEL_NAME: z.string().min(1).optional(),
       E2E_BROKEN_MODEL_PROVIDER: z.string().min(1).optional(),
       E2E_BROKEN_MODEL_TYPE: z.string().min(1).optional(),

e2e/scripts/run-cucumber.ts

@@ -1,6 +1,13 @@
 import type { ManagedProcess } from '../support/process'
 import { mkdir, readFile, rm } from 'node:fs/promises'
 import path from 'node:path'
+import { runCleanupTasks } from '../support/cleanup'
+import {
+  assertCucumberReport,
+  formatCucumberReportSummary,
+  getCucumberReportGate,
+  readCucumberReportSummary,
+} from '../support/cucumber-report'
 import { startLoggedProcess, stopManagedProcess, waitForUrl } from '../support/process'
 import { startWebServer, stopWebServer } from '../support/web-server'
 import { apiURL, baseURL, reuseExistingWebServer } from '../test-env'
@@ -150,19 +157,17 @@ const main = async () => {
   const cleanup = async () => {
     if (!cleanupPromise) {
       cleanupPromise = (async () => {
-        await stopWebServer()
-        await stopManagedProcess(celeryProcess)
-        await stopManagedProcess(apiProcess)
-        await stopManagedProcess(difyAgentProcess)
-        await stopManagedProcess(shellctlProcess)
+        const cleanupErrors = await runCleanupTasks([
+          { label: 'Stop web server', run: stopWebServer },
+          { label: 'Stop celery worker', run: () => stopManagedProcess(celeryProcess) },
+          { label: 'Stop API server', run: () => stopManagedProcess(apiProcess) },
+          { label: 'Stop agent backend', run: () => stopManagedProcess(difyAgentProcess) },
+          { label: 'Stop shellctl sandbox', run: () => stopManagedProcess(shellctlProcess) },
+          ...(startMiddlewareForRun ? [{ label: 'Stop middleware', run: stopMiddleware }] : []),
+        ])
 
-        if (startMiddlewareForRun) {
-          try {
-            await stopMiddleware()
-          } catch {
-            // Cleanup should continue even if middleware shutdown fails.
-          }
-        }
+        if (cleanupErrors.length > 0)
+          throw new Error(`E2E teardown errors:\n${cleanupErrors.join('\n')}`)
       })()
     }
 
@@ -170,9 +175,13 @@ const main = async () => {
   }
 
   const onTerminate = () => {
-    void cleanup().finally(() => {
-      process.exit(1)
-    })
+    void cleanup()
+      .catch((error) => {
+        console.error(error instanceof Error ? error.message : String(error))
+      })
+      .finally(() => {
+        process.exit(1)
+      })
   }
 
   process.once('SIGINT', onTerminate)
@@ -265,6 +274,16 @@ const main = async () => {
       env: cucumberEnv,
     })
 
+    const reportGate = getCucumberReportGate(cucumberEnv)
+    if (reportGate) {
+      const reportPath = path.join(cucumberReportDir, 'report.json')
+      const reportSummary = await readCucumberReportSummary(reportPath)
+      console.warn(
+        `[e2e] cucumber report ${reportGate.profile}: ${formatCucumberReportSummary(reportSummary)}`,
+      )
+      assertCucumberReport(reportSummary, reportGate)
+    }
+
     process.exitCode = result.exitCode
   } finally {
     process.off('SIGINT', onTerminate)

e2e/scripts/setup.ts

@@ -55,9 +55,13 @@ const middlewareDataPaths = [
 const e2eStatePaths = [
   path.join(e2eDir, '.auth'),
   path.join(e2eDir, 'cucumber-report'),
+  path.join(e2eDir, 'cucumber-report-non-external'),
+  path.join(e2eDir, 'cucumber-report-webkit'),
   path.join(e2eDir, '.logs'),
   path.join(e2eDir, '.logs-non-external'),
+  path.join(e2eDir, '.logs-webkit'),
   path.join(e2eDir, 'playwright-report'),
+  path.join(e2eDir, 'seed-report'),
   path.join(e2eDir, 'test-results'),
 ]