diff --git a/.github/workflows/test-evals-instance-ai.yml b/.github/workflows/test-evals-instance-ai.yml index 081f32661f5..8d7ce535c0e 100644 --- a/.github/workflows/test-evals-instance-ai.yml +++ b/.github/workflows/test-evals-instance-ai.yml @@ -92,12 +92,19 @@ on: jobs: run-evals: name: 'Run Evals' - runs-on: blacksmith-4vcpu-ubuntu-2204 - timeout-minutes: 90 + # 8vcpu/32GB: full-suite N=10 baselines need ~150 min of eval time and + # did not fit the 4vcpu tier at any timeout. + runs-on: blacksmith-8vcpu-ubuntu-2204 + # Long ceiling only for high-N baseline captures; regular runs keep a + # tight guard so a wedged run fails fast. Plain comparison (not fromJSON): + # it coerces the string input to a number, and a malformed dispatch value + # (NaN) falls back to the 90-minute guard instead of erroring the job. + timeout-minutes: ${{ (inputs.iterations || '3') >= 5 && 240 || 90 }} env: # Each port hosts an independent n8n container. The eval CLI's # work-stealing allocator dispatches builds across them, capped per-lane. - LANE_PORTS: '5678,5679,5680,5681,5682,5683,5684,5685,5686,5687,5688' + # 10 lanes x 2.5 GB caps = 25 GB, leaving ~7 GB for sandbox + CLI + OS. + LANE_PORTS: '5678,5679,5680,5681,5682,5683,5684,5685,5686,5687' permissions: contents: read pull-requests: write @@ -122,6 +129,25 @@ jobs: with: cache-sha: ${{ inputs.cache-sha }} + # Host OOMs and disk exhaustion leave no application trace — sample host + # + container telemetry every 60s and ship it with the results artifact. + - name: Start host telemetry sampler + run: | + mkdir -p eval-diag + { echo "=== disk at job start ==="; df -h /; docker system df; } > eval-diag/df-at-start.log + nohup bash -c 'while true; do + { + date -u +%FT%TZ + free -m | head -2 + df -h / | tail -1 + docker stats --no-stream --format "{{.Name}} cpu={{.CPUPerc}} mem={{.MemUsage}}" + ps -eo rss=,pid=,comm= --sort=-rss | head -5 + echo + } >> eval-diag/host-samples.log 2>&1 + sleep 60 + done' > /dev/null 2>&1 & + echo $! > eval-diag/sampler.pid + - name: Start sandbox service if: ${{ inputs.sandbox-provider == 'n8n-sandbox' }} run: pnpm --filter n8n-containers services --services sandbox --network n8n-eval-net --name n8n-svc-sandbox @@ -174,8 +200,17 @@ jobs: IFS=',' read -ra PORTS <<< "$LANE_PORTS" for i in "${!PORTS[@]}"; do port="${PORTS[$i]}" + # Bounded and self-healing: a lane that exhausts its capped heap is + # restarted by docker instead of staying dead; pruning + log caps + # keep per-lane disk and memory flat over a multi-hour run. docker run -d --name "n8n-eval-$((i+1))" \ "${NETWORK_ARGS[@]}" \ + --memory 2.5g --memory-swap 2.5g \ + --restart on-failure \ + --log-opt max-size=50m --log-opt max-file=2 \ + -e NODE_OPTIONS=--max-old-space-size=2048 \ + -e EXECUTIONS_DATA_PRUNE=true \ + -e EXECUTIONS_DATA_MAX_AGE=1 \ -e E2E_TESTS=true \ -e N8N_ENABLED_MODULES=instance-ai \ -e N8N_AI_ENABLED=true \ @@ -195,7 +230,7 @@ jobs: n8nio/n8n:local done # 120s budget per port: containers booting in parallel on a shared - # 4vcpu runner contend for CPU/disk during n8n's startup (DB migrations, + # runner contend for CPU/disk during n8n's startup (DB migrations, # license init), so each takes longer than a solo boot. for port in "${PORTS[@]}"; do ready=false @@ -277,6 +312,25 @@ jobs: exit 1 fi + # Disk pressure develops during the eval (per-lane SQLite growth, + # verifier snapshots, logs), not at job start — so the runway check + # lives here, after the image load and lane startup, where "is there + # enough free disk for the next few hours" is a meaningful question. + # A run-A-style on-runner fallback build leaves tens of GB of builder + # cache that is safe to reclaim once every lane is up. + - name: Ensure disk runway for the eval run + shell: bash + run: | + free_gb=$(df -BG --output=avail / | tail -1 | tr -dc '0-9') + if [ "${free_gb:-0}" -lt 30 ]; then + echo "Only ${free_gb}GB free — reclaiming builder cache before the run accretes data" + # Best-effort: a prune failure must not skip the eval run itself. + docker builder prune -f || echo "::warning::builder prune failed — continuing with ${free_gb}GB free" + df -h / || true + else + echo "Disk runway OK: ${free_gb}GB free" + fi + - name: Run Instance AI Evals continue-on-error: true working-directory: packages/@n8n/instance-ai @@ -346,19 +400,22 @@ jobs: done # Layer 1 — accuracy filter: only surface diagnostic signals. - # `tail -100` after the filter so we get the LATEST matching lines + # `tail -1000` after the filter so we get the LATEST matching lines # (post-eval failure signal), not the earliest startup-time ones. + # -t keeps timestamps; the inspect line surfaces OOM kills/restarts, + # which leave no log line of their own. SIGNALS='sandbox|builder|sandbox-service|daytona|instance.?ai|error|warn|reject|exception|fail' for c in $(docker ps -aq --filter "name=n8n-eval-"); do name=$(docker inspect --format '{{.Name}}' "$c" | sed 's|^/||') echo "" echo "============================================================" - echo "=== $name (filtered diagnostic signals, last 100 lines) ===" + echo "=== $name (filtered diagnostic signals, last 1000 lines) ===" echo "============================================================" - docker logs "$c" 2>&1 \ + docker inspect --format 'state: status={{.State.Status}} oomkilled={{.State.OOMKilled}} exitcode={{.State.ExitCode}} restarts={{.RestartCount}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}' "$c" || true + docker logs -t "$c" 2>&1 \ | grep -ivE 'migration' \ | grep -iE "$SIGNALS" \ - | tail -100 \ + | tail -1000 \ || true done @@ -375,6 +432,7 @@ jobs: - name: Stop n8n containers if: ${{ always() }} run: | + [ -f eval-diag/sampler.pid ] && kill "$(cat eval-diag/sampler.pid)" 2>/dev/null || true mapfile -t ids < <(docker ps -aq --filter "name=n8n-eval-") if [ "${#ids[@]}" -gt 0 ]; then docker stop "${ids[@]}" 2>/dev/null || true @@ -420,4 +478,5 @@ jobs: path: | packages/@n8n/instance-ai/eval-results.json packages/@n8n/instance-ai/.data/workflow-eval-report.html + eval-diag/ retention-days: 14 diff --git a/packages/@n8n/instance-ai/evaluations/README.md b/packages/@n8n/instance-ai/evaluations/README.md index 20d90b69bd1..b63bf9909d0 100644 --- a/packages/@n8n/instance-ai/evaluations/README.md +++ b/packages/@n8n/instance-ai/evaluations/README.md @@ -136,7 +136,7 @@ dotenvx run -f ../../../.env.local -- pnpm eval:instance-ai --iterations 3 | `--filter` | — | Filter test cases by filename substring. Comma-separated values mean OR (e.g. `contact-form,deduplication`) | | `--exclude` | — | Skip test cases whose filename matches any of the substrings. Same comma-separated shape as `--filter`; applied after `--filter` | | `--prebuilt-workflows` | — | Path to a JSON manifest mapping test-case slugs to existing workflow IDs. Skips the orchestrator build for matched test cases — see [Running evals against pre-built workflows](#running-evals-against-pre-built-workflows) | -| `--keep-workflows` | `false` | Don't delete built workflows after the run. Pair with the HTML report's "view in n8n" links to inspect each scenario's canvas execution | +| `--keep-workflows` | `false` | Don't delete any build artifacts after the run — workflows, data tables, and threads (with their sandboxes) all survive. Pair with the HTML report's "view in n8n" links to inspect each scenario's canvas execution. Sandboxes have no auto-cleanup, so use with `--filter` on a few cases rather than full baselines | | `--delete-prebuilt-workflows` | `false` | With `--prebuilt-workflows`, delete successfully used manifest workflows after the eval run. Mutually exclusive with `--keep-workflows` | | `--base-url` | `http://localhost:5678` | n8n instance URL | | `--email` | E2E test owner | Override login email (or `N8N_EVAL_EMAIL`) | diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/capture-run-debug.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/capture-run-debug.test.ts index e58476b3463..33fe3ac7003 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/capture-run-debug.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/capture-run-debug.test.ts @@ -33,7 +33,8 @@ describe('captureThreadRunDebug', () => { expect(records).toHaveLength(1); expect(records[0]?.label).toBe('Build workflow'); expect(records[0]?.steps).toHaveLength(1); - expect(client.getRunDebug).toHaveBeenCalledWith('run-1'); + // Per-request timeout so a stalled call releases its socket. + expect(client.getRunDebug).toHaveBeenCalledWith('run-1', expect.any(Number)); }); it('returns an empty array when the debug API is unavailable', async () => { diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/cleanup-build.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/cleanup-build.test.ts new file mode 100644 index 00000000000..430973bf64b --- /dev/null +++ b/packages/@n8n/instance-ai/evaluations/__tests__/cleanup-build.test.ts @@ -0,0 +1,69 @@ +import { vi } from 'vitest'; +import type { Mock } from 'vitest'; + +import type { N8nClient } from '../clients/n8n-client'; +import type { EvalLogger } from '../harness/logger'; +import { cleanupBuild } from '../harness/runner'; +import type { BuildResult } from '../harness/runner'; + +/** + * Locks in the cleanupBuild contract the CLI's per-case cleanup relies on: + * the return value reports whether every deletion succeeded, so a caller can + * keep the build cached and retry a transiently failed cleanup later. + */ + +const silentLogger: EvalLogger = { + info: () => {}, + verbose: () => {}, + success: () => {}, + warn: () => {}, + error: () => {}, + isVerbose: false, +}; + +function makeClient(overrides: Partial> = {}): { + client: N8nClient; + mocks: Record; +} { + const mocks: Record = { + deleteWorkflow: vi.fn().mockResolvedValue(undefined), + deleteDataTable: vi.fn().mockResolvedValue(undefined), + getPersonalProjectId: vi.fn().mockResolvedValue('project-1'), + deleteThread: vi.fn().mockResolvedValue(undefined), + ...overrides, + }; + return { client: mocks as unknown as N8nClient, mocks }; +} + +function makeBuild(): BuildResult { + return { + success: true, + workflowJsons: [], + createdWorkflowIds: ['W1'], + createdDataTableIds: ['DT1'], + threadId: 'T1', + }; +} + +describe('cleanupBuild', () => { + it('deletes workflows, data tables and the thread, and reports clean', async () => { + const { client, mocks } = makeClient(); + + await expect(cleanupBuild(client, makeBuild(), silentLogger)).resolves.toBe(true); + + expect(mocks.deleteWorkflow).toHaveBeenCalledWith('W1'); + expect(mocks.deleteDataTable).toHaveBeenCalledWith('project-1', 'DT1'); + expect(mocks.deleteThread).toHaveBeenCalledWith('T1'); + }); + + it('reports not clean when a deletion fails, but still attempts the rest', async () => { + const { client, mocks } = makeClient({ + deleteWorkflow: vi.fn().mockRejectedValue(new Error('HTTP 502')), + }); + + await expect(cleanupBuild(client, makeBuild(), silentLogger)).resolves.toBe(false); + + expect(mocks.deleteDataTable).toHaveBeenCalledWith('project-1', 'DT1'); + expect(mocks.deleteThread).toHaveBeenCalledWith('T1'); + }); +}); diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/fetch-baseline.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/fetch-baseline.test.ts index 8c90ce61361..fa99bbe21e0 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/fetch-baseline.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/fetch-baseline.test.ts @@ -12,6 +12,7 @@ import { BUILD_ONLY_SCENARIO_NAME } from '../langsmith/dataset-sync'; interface FakeProject { name?: string; start_time?: string; + extra?: Record; } /** @@ -26,9 +27,17 @@ function clientWith(projects: FakeProject[]): { client: Client; listProjects: Mo for (const p of projects) yield p; })(), ); - return { client: { listProjects } as unknown as Client, listProjects }; + const readProject = vi.fn(async ({ projectName }: { projectName: string }) => { + const project = projects.find((p) => p.name === projectName); + if (!project) throw new Error(`not found: ${projectName}`); + return await Promise.resolve(project); + }); + return { client: { listProjects, readProject } as unknown as Client, listProjects }; } +/** Aggregate metadata written by the CLI only when a run completes. */ +const COMPLETED = { metadata: { pass_rate_per_iter: '78%' } }; + describe('findLatestBaseline', () => { it('queries with the default instance-ai baseline prefix when none is given', async () => { const { client, listProjects } = clientWith([]); @@ -75,6 +84,30 @@ describe('findLatestBaseline', () => { ]); expect(await findLatestBaseline(client, 'mcp-baseline-')).toBe('mcp-baseline-no-ts'); }); + + it('skips a newer capture that never wrote the completion marker (killed run)', async () => { + const { client } = clientWith([ + { name: 'instance-ai-baseline-done', start_time: '2026-07-08T00:00:00Z', extra: COMPLETED }, + { name: 'instance-ai-baseline-killed', start_time: '2026-07-09T00:00:00Z', extra: {} }, + ]); + expect(await findLatestBaseline(client)).toBe('instance-ai-baseline-done'); + }); + + it('picks the newest among completed captures', async () => { + const { client } = clientWith([ + { name: 'instance-ai-baseline-old', start_time: '2026-07-01T00:00:00Z', extra: COMPLETED }, + { name: 'instance-ai-baseline-new', start_time: '2026-07-08T00:00:00Z', extra: COMPLETED }, + ]); + expect(await findLatestBaseline(client)).toBe('instance-ai-baseline-new'); + }); + + it('falls back to the newest candidate when none carry the marker', async () => { + const { client } = clientWith([ + { name: 'instance-ai-baseline-a', start_time: '2026-07-01T00:00:00Z' }, + { name: 'instance-ai-baseline-b', start_time: '2026-07-08T00:00:00Z' }, + ]); + expect(await findLatestBaseline(client)).toBe('instance-ai-baseline-b'); + }); }); interface FakeRun { diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/lane-allocator.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/lane-allocator.test.ts index e7964ee915a..64781f0e973 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/lane-allocator.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/lane-allocator.test.ts @@ -91,4 +91,109 @@ describe('LaneAllocator', () => { await w1; expect(order).toEqual(['p3', 'p1']); }); + + describe('lane health', () => { + afterEach(() => { + vi.useRealTimers(); + }); + + it('quarantines a lane after consecutive transient failures and stops assigning to it', async () => { + const lanes = newLanes(2); + const onQuarantine = vi.fn(); + const a = new LaneAllocator(lanes, 4, { + probe: async () => await Promise.resolve(false), + quarantineThreshold: 3, + onQuarantine, + }); + for (let i = 0; i < 3; i++) a.reportBuildOutcome(lanes[0], 'transient-failure'); + expect(a.isQuarantined(lanes[0])).toBe(true); + expect(onQuarantine).toHaveBeenCalledWith(lanes[0]); + const l1 = await a.acquire('p1'); + const l2 = await a.acquire('p2'); + expect([l1.id, l2.id]).toEqual([1, 1]); + }); + + it('resets the consecutive-failure counter on a successful build', () => { + const lanes = newLanes(1); + const a = new LaneAllocator(lanes, 4, { + probe: async () => await Promise.resolve(false), + quarantineThreshold: 3, + }); + a.reportBuildOutcome(lanes[0], 'transient-failure'); + a.reportBuildOutcome(lanes[0], 'transient-failure'); + a.reportBuildOutcome(lanes[0], 'ok'); + a.reportBuildOutcome(lanes[0], 'transient-failure'); + a.reportBuildOutcome(lanes[0], 'transient-failure'); + expect(a.isQuarantined(lanes[0])).toBe(false); + }); + + it('re-admits a lane once the probe reports healthy and serves queued waiters', async () => { + vi.useFakeTimers(); + const lanes = newLanes(1); + let healthy = false; + const onReadmit = vi.fn(); + const a = new LaneAllocator(lanes, 4, { + probe: async () => await Promise.resolve(healthy), + probeIntervalMs: 1000, + quarantineThreshold: 1, + allQuarantinedGraceMs: 60_000, + onReadmit, + }); + a.reportBuildOutcome(lanes[0], 'transient-failure'); + expect(a.isQuarantined(lanes[0])).toBe(true); + const waiter = a.acquire('p1'); + await vi.advanceTimersByTimeAsync(1000); + expect(a.isQuarantined(lanes[0])).toBe(true); + healthy = true; + await vi.advanceTimersByTimeAsync(1000); + expect(a.isQuarantined(lanes[0])).toBe(false); + expect(onReadmit).toHaveBeenCalledWith(lanes[0]); + await expect(waiter).resolves.toBe(lanes[0]); + }); + + it('prefers a lane other than `not`, falling back to it when nothing else is free', async () => { + const lanes = newLanes(2); + const a = new LaneAllocator(lanes, 4); + const first = await a.acquire('p1'); + const retry = await a.acquire('p2', { not: first }); + expect(retry.id).not.toBe(first.id); + + const single = newLanes(1); + const b = new LaneAllocator(single, 4); + const only = await b.acquire('p1'); + const fallback = await b.acquire('p2', { not: only }); + expect(fallback).toBe(only); + }); + + it('remembers when a lane was quarantined so mid-flight failures can be attributed', () => { + const before = Date.now(); + const lanes = newLanes(1); + const a = new LaneAllocator(lanes, 4, { + probe: async () => await Promise.resolve(false), + quarantineThreshold: 1, + }); + expect(a.wasQuarantinedSince(lanes[0], before)).toBe(false); + a.reportBuildOutcome(lanes[0], 'transient-failure'); + expect(a.wasQuarantinedSince(lanes[0], before)).toBe(true); + expect(a.wasQuarantinedSince(lanes[0], Date.now() + 1000)).toBe(false); + }); + + it('aborts pending and future acquires when all lanes stay quarantined past the grace period', async () => { + vi.useFakeTimers(); + const lanes = newLanes(2); + const a = new LaneAllocator(lanes, 4, { + probe: async () => await Promise.resolve(false), + probeIntervalMs: 1000, + quarantineThreshold: 1, + allQuarantinedGraceMs: 5000, + }); + a.reportBuildOutcome(lanes[0], 'transient-failure'); + a.reportBuildOutcome(lanes[1], 'transient-failure'); + const pending = a.acquire('p1'); + const rejection = expect(pending).rejects.toThrow('All 2 lanes quarantined'); + await vi.advanceTimersByTimeAsync(5000); + await rejection; + await expect(a.acquire('p2')).rejects.toThrow('quarantined'); + }); + }); }); diff --git a/packages/@n8n/instance-ai/evaluations/cli/index.ts b/packages/@n8n/instance-ai/evaluations/cli/index.ts index ab4652a0611..b132d46a385 100644 --- a/packages/@n8n/instance-ai/evaluations/cli/index.ts +++ b/packages/@n8n/instance-ai/evaluations/cli/index.ts @@ -80,6 +80,7 @@ import { } from '../harness/runner'; import { extractErrorMessage, + isTransientNetworkError, MAX_EXEC_ATTEMPTS, shouldRetryScenarioExecution, } from '../harness/transient-error'; @@ -105,6 +106,39 @@ import { caseDisplayPrompt, conversationUserTurnsAsText } from '../utils/convers // n8n degrades above ~4 concurrent builds. const MAX_CONCURRENT_BUILDS = 4; +/** Attempts (initial + retries) for a build hitting transient network errors. */ +const MAX_BUILD_ATTEMPTS = 3; + +/** Framework-noise share above which a baseline capture gets a quality warning. */ +const BASELINE_MAX_FRAMEWORK_NOISE_RATE = 0.05; + +/** Count framework-noise trials and cases that failed on nothing but noise. */ +function assessFrameworkNoise( + evaluation: MultiRunEvaluation, + slugByTestCase?: Map, +): { frameworkTrials: number; totalTrials: number; fullyNoisyCases: string[] } { + let frameworkTrials = 0; + let totalTrials = 0; + const fullyNoisyCases: string[] = []; + for (const tc of evaluation.testCases) { + let caseFramework = 0; + let caseTotal = 0; + for (const sa of tc.executionScenarios) { + for (const run of sa.runs) { + if (run.incomplete) continue; + caseTotal++; + if (!run.success && run.failureCategory === 'framework_issue') caseFramework++; + } + } + frameworkTrials += caseFramework; + totalTrials += caseTotal; + if (caseTotal > 0 && caseFramework === caseTotal) { + fullyNoisyCases.push(slugByTestCase?.get(tc.testCase) ?? caseDisplayPrompt(tc.testCase)); + } + } + return { frameworkTrials, totalTrials, fullyNoisyCases }; +} + /** Target input shape with the iteration index we inject for multi-run. */ type TargetInputs = DatasetExampleInputs & { _iteration?: number }; @@ -458,6 +492,28 @@ async function main(): Promise { console.log( '\n' + formatComparisonTerminal(evaluation, outcome, { commitSha, slugByTestCase, gate }), ); + + // Advisory only: findLatestBaseline trusts the newest experiment by + // prefix, so surface elevated harness noise for the humans reading the log. + if (args.experimentName?.startsWith('instance-ai-baseline')) { + const { frameworkTrials, totalTrials, fullyNoisyCases } = assessFrameworkNoise( + evaluation, + slugByTestCase, + ); + const noiseRate = totalTrials > 0 ? frameworkTrials / totalTrials : 0; + if (noiseRate > BASELINE_MAX_FRAMEWORK_NOISE_RATE || fullyNoisyCases.length > 0) { + console.warn( + `Baseline quality warning: ${String(frameworkTrials)}/${String(totalTrials)} trials (${(noiseRate * 100).toFixed(1)}%) failed for harness reasons` + + ' (lane transport, seeding, timeouts) rather than agent behavior' + + (fullyNoisyCases.length > 0 + ? `; cases with only framework failures: ${fullyNoisyCases.join(', ')}` + : '') + + '. This experiment becomes the comparison target for future runs, but those scenarios will' + + ' under-count the agent — deltas against them may reflect harness noise, not regressions or' + + ' improvements. Consider fixing the noise and re-capturing.', + ); + } + } } finally { // Per-lane cleanup: each lane only holds the workflows built/fetched on it, // so delete them via that lane's own client (multi-lane MCP builds spread @@ -557,6 +613,7 @@ async function runWithLangSmith(config: RunConfig): Promise<{ > & { timeoutMs: number }; interface LaneState { runner: Lane; + laneNum: number; activeBuilds: number; inflightKeys: Set; tracedBuild: (buildArgs: BuildArgs) => Promise; @@ -574,6 +631,7 @@ async function runWithLangSmith(config: RunConfig): Promise<{ const laneTag = lanes.length > 1 ? ` [lane ${String(laneNum)}/${String(lanes.length)}]` : ''; return { runner: lane, + laneNum, activeBuilds: 0, inflightKeys: new Set(), tracedBuild: traceable( @@ -630,14 +688,49 @@ async function runWithLangSmith(config: RunConfig): Promise<{ }; }); + // Direct fetch (not N8nClient) so a hung lane can't stall the probe. + async function laneHealthy(lane: LaneState): Promise { + try { + const res = await fetch(`${lane.runner.baseUrl}/healthz/readiness`, { + signal: AbortSignal.timeout(5_000), + }); + return res.ok; + } catch { + return false; + } + } + + // A build that sat out its timeout against a dead lane reports "Run timed + // out", not "fetch failed" — so any failed build also health-probes its lane. + async function isTransportFailure(build: BuildResult, lane: LaneState): Promise { + if (build.success) return false; + if (build.error !== undefined && isTransientNetworkError(build.error)) return true; + return !(await laneHealthy(lane)); + } + // Work-stealing: each build acquires a lane that isn't already running its - // fileSlug, runs there (capped per-lane), then releases. Scenarios re-use the - // lane that built their workflow. - const allocator = new LaneAllocator(laneStates, MAX_CONCURRENT_BUILDS); + // fileSlug, runs there (capped per-lane), then releases. Scenarios re-use + // the lane that built their workflow. Health options quarantine a dead lane + // instead of letting its instant failures attract the whole queue. + const allocator = new LaneAllocator(laneStates, MAX_CONCURRENT_BUILDS, { + probe: laneHealthy, + onQuarantine: (lane) => + logger.error( + `[lane ${String(lane.laneNum)}] quarantined after consecutive transport failures; probing ${lane.runner.baseUrl} for recovery`, + ), + onReadmit: (lane) => logger.info(`[lane ${String(lane.laneNum)}] healthy again — re-admitted`), + onAllQuarantined: () => + logger.error('All lanes quarantined — builds paused pending lane recovery'), + }); const buildCache = new Map< string, Promise<{ build: BuildResult; lane: LaneState; buildDurationMs: number }> >(); + // Transport-evicted builds leave buildCache before any cleanup pass sees + // them, but their artifacts (restored workflows, data tables, thread — and + // with it the sandbox) are real. Stash them for the end-of-run drain; the + // lane may be mid-restart at eviction time, so immediate cleanup can't work. + const orphanedBuilds: Array<{ build: BuildResult; client: N8nClient }> = []; const buildDurations = new Map(); async function getOrBuild( @@ -677,6 +770,11 @@ async function runWithLangSmith(config: RunConfig): Promise<{ // holding the slot through it would idle the lane's build capacity. allocator.release(lane, fileSlug); } + { + const transient = await isTransportFailure(build, lane); + if (!build.success) build.transportFailure = transient; + allocator.reportBuildOutcome(lane, transient ? 'transient-failure' : 'ok'); + } const buildDurationMs = Date.now() - start; // Cleanup registration happens inside buildWorkflowViaMcpOnLane (as soon // as `claude` reports the id), so even a failed fetch-back is covered. @@ -728,39 +826,72 @@ async function runWithLangSmith(config: RunConfig): Promise<{ } // Orchestrator path: allocator spreads distinct fileSlugs across lanes; // the build cache dedupes scenarios within one file. - const lane = await allocator.acquire(fileSlug); const entry = testCaseByFileSlug.get(fileSlug); if (!entry) throw new Error(`No conversation found for fileSlug=${fileSlug}`); - try { - const start = Date.now(); - const timeoutMs = effectiveTimeoutMs(entry.complexity, args.timeoutMs); - if (timeoutMs !== args.timeoutMs) { - logger.info( - ` Complex case: per-iteration budget ${String(Math.round(timeoutMs / 1000))}s [${fileSlug}]`, - ); - } - const build = await lane.tracedBuild({ - conversation: entry.conversation, - messageBudget: entry.messageBudget, - credentials: entry.credentials, - seedFile: entry.seedFile, - priorConversation: entry.priorConversation, - seedThread: entry.seedThread, - executionScenarios: entry.executionScenarios, - outcomeExpectations: entry.outcomeExpectations, - timeoutMs, - }); - const buildDurationMs = Date.now() - start; - buildDurations.set(key, buildDurationMs); - stashTranscript(build); - stashBuildExpectations(key, fileSlug, build, false); - stashRunDebug(lane.runner.client, build); - return { build, lane, buildDurationMs }; - } finally { - allocator.release(lane, fileSlug); + const timeoutMs = effectiveTimeoutMs(entry.complexity, args.timeoutMs); + if (timeoutMs !== args.timeoutMs) { + logger.info( + ` Complex case: per-iteration budget ${String(Math.round(timeoutMs / 1000))}s [${fileSlug}]`, + ); } + // Transport failures are not agent verdicts — retry on a different lane + // instead of recording 0-score rows for every scenario of the case. + let lane = await allocator.acquire(fileSlug); + let build: BuildResult; + let buildDurationMs: number; + for (let attempt = 1; ; attempt++) { + const start = Date.now(); + try { + build = await lane.tracedBuild({ + conversation: entry.conversation, + messageBudget: entry.messageBudget, + credentials: entry.credentials, + seedFile: entry.seedFile, + priorConversation: entry.priorConversation, + seedThread: entry.seedThread, + executionScenarios: entry.executionScenarios, + outcomeExpectations: entry.outcomeExpectations, + timeoutMs, + }); + } finally { + allocator.release(lane, fileSlug); + } + buildDurationMs = Date.now() - start; + const transient = + (await isTransportFailure(build, lane)) || + (!build.success && allocator.wasQuarantinedSince(lane, start)); + if (!build.success) build.transportFailure = transient; + allocator.reportBuildOutcome(lane, transient ? 'transient-failure' : 'ok'); + if (!transient || attempt >= MAX_BUILD_ATTEMPTS) break; + logger.warn( + `Build ${fileSlug} attempt ${String(attempt)}/${String(MAX_BUILD_ATTEMPTS)} failed transiently on lane ${String(lane.laneNum)} (${build.error ?? 'unknown'}); retrying on another lane`, + ); + lane = await allocator.acquire(fileSlug, { not: lane }); + } + buildDurations.set(key, buildDurationMs); + stashTranscript(build); + stashBuildExpectations(key, fileSlug, build, false); + stashRunDebug(lane.runner.client, build); + logger.info( + `[lane ${String(lane.laneNum)}] built ${fileSlug} (iteration ${String(iteration)}) thread=${build.threadId ?? 'none'} success=${String(build.success)}`, + ); + // Only the pairwise flow reads captured events — drop the largest chunk + // of each BuildResult from the run-long cache. + build.events = undefined; + return { build, lane, buildDurationMs }; })(); buildCache.set(key, promise); + // Evict transport-failed builds so a later scenario rebuilds. Agent build + // failures stay cached — they are the verdict; rebuilding just multiplies cost. + void promise.then( + ({ build, lane }) => { + if (build.transportFailure) { + orphanedBuilds.push({ build, client: lane.runner.client }); + buildCache.delete(key); + } + }, + () => buildCache.delete(key), + ); return await promise; } @@ -812,8 +943,56 @@ async function runWithLangSmith(config: RunConfig): Promise<{ ); } + // Rows remaining per `iteration:fileSlug` build. When the last row of a + // build finishes, its backend artifacts (workflow, data tables, thread — and + // with the thread its sandbox) are deleted right away instead of at the end + // of the run: sandboxes have no auto-cleanup, so end-of-run-only deletion + // let the sandbox runner grow to ~15 GB over a full N=10 baseline. + // --keep-workflows deliberately keeps everything, thread/sandbox included — + // it's a debugging flag for small filtered runs, not baselines. + const remainingRowsByKey = new Map(); + + function rowsPerCase(fileSlug: string): number { + const scenarios = testCaseByFileSlug.get(fileSlug)?.executionScenarios?.length ?? 0; + return Math.max(1, scenarios); // scenario-less cases get one build-only row + } + + async function releaseCaseRow(iteration: number, fileSlug: string): Promise { + const key = `${String(iteration)}:${fileSlug}`; + const remaining = (remainingRowsByKey.get(key) ?? rowsPerCase(fileSlug)) - 1; + remainingRowsByKey.set(key, remaining); + if (remaining > 0 || args.keepWorkflows) return; + const cached = buildCache.get(key); + if (!cached) return; // evicted (transport failure) — nothing to clean + try { + const { build, lane } = await cached; + // Run-debug capture reads the thread — let it settle before deletion. + if (build.threadId) await runDebugByThreadId.get(build.threadId)?.catch(() => {}); + const clean = await cleanupBuild(lane.runner.client, build, logger); + if (!clean) { + // Leave the entry in buildCache — the end-of-run pass retries it. + logger.verbose( + ` [cleanup] ${fileSlug} (iteration ${String(iteration)}) incomplete, retrying at end of run`, + ); + return; + } + buildCache.delete(key); + logger.verbose(` [cleanup] ${fileSlug} (iteration ${String(iteration)}) artifacts deleted`); + } catch { + // Best-effort — a failed cleanup must never fail the row. + } + } + const target = async (inputs: TargetInputs): Promise => { const iteration = inputs._iteration ?? 0; + try { + return await targetRow(inputs, iteration); + } finally { + await releaseCaseRow(iteration, inputs.testCaseFile); + } + }; + + const targetRow = async (inputs: TargetInputs, iteration: number): Promise => { const scenario: ExecutionScenario = { name: inputs.scenarioName, description: inputs.scenarioDescription, @@ -872,9 +1051,10 @@ async function runWithLangSmith(config: RunConfig): Promise<{ passed: false, score: 0, reasoning: `Build failed: ${build.error ?? 'unknown'}`, - // Seeding failures are a harness setup problem, not an agent build - // failure — keep them out of the agent's build_failure bucket. - failureCategory: build.seedingFailed ? 'framework_issue' : 'build_failure', + // Seeding and transport failures are harness problems, not agent build + // failures — keep them out of the agent's build_failure bucket. + failureCategory: + build.seedingFailed || build.transportFailure ? 'framework_issue' : 'build_failure', execErrors: build.error ? [build.error] : [], buildDurationMs, execDurationMs: 0, @@ -1133,6 +1313,8 @@ async function runWithLangSmith(config: RunConfig): Promise<{ }; } finally { if (!args.keepWorkflows) { + // Entries still here had no rows run, or their per-case cleanup failed + // (releaseCaseRow leaves those cached so this pass can retry them). await Promise.all( [...buildCache.values()].map(async (promise) => { try { @@ -1143,6 +1325,15 @@ async function runWithLangSmith(config: RunConfig): Promise<{ } }), ); + await Promise.all( + orphanedBuilds.map(async ({ build, client }) => { + try { + await cleanupBuild(client, build, logger); + } catch { + // Best-effort — the lane may still be unreachable + } + }), + ); } } } diff --git a/packages/@n8n/instance-ai/evaluations/cli/lane-allocator.ts b/packages/@n8n/instance-ai/evaluations/cli/lane-allocator.ts index 4aa37366ee1..efc3e523a6c 100644 --- a/packages/@n8n/instance-ai/evaluations/cli/lane-allocator.ts +++ b/packages/@n8n/instance-ai/evaluations/cli/lane-allocator.ts @@ -1,6 +1,11 @@ // Pull-based lane allocator. Each lane caps at `maxConcurrentBuilds` and never // runs the same key twice concurrently — pairing those rules eliminates the // same-key concentration that breaks the agent under load. +// +// A dead lane fails builds in milliseconds, making it permanently the +// least-loaded lane — it would swallow the whole remaining queue. Consecutive +// transport failures therefore quarantine a lane; a health probe re-admits it +// once its backend responds; all-lanes-quarantined aborts after a grace period. export interface AllocatableLane { activeBuilds: number; @@ -10,24 +15,57 @@ export interface AllocatableLane { interface Waiter { key: string; resolve: (lane: L) => void; + reject: (error: Error) => void; } +export interface LaneHealthOptions { + /** Resolves true when the lane's backend responds healthy again. */ + probe: (lane: L) => Promise; + /** Delay between health probes of a quarantined lane. */ + probeIntervalMs?: number; + /** Consecutive transient build failures before a lane is quarantined. */ + quarantineThreshold?: number; + /** How long ALL lanes may stay quarantined before acquires abort. */ + allQuarantinedGraceMs?: number; + onQuarantine?: (lane: L) => void; + onReadmit?: (lane: L) => void; + onAllQuarantined?: () => void; +} + +const DEFAULT_PROBE_INTERVAL_MS = 30_000; +const DEFAULT_QUARANTINE_THRESHOLD = 3; +const DEFAULT_ALL_QUARANTINED_GRACE_MS = 5 * 60_000; + export class LaneAllocator { private readonly waiters: Array> = []; + private readonly consecutiveFailures = new Map(); + + private readonly quarantined = new Set(); + + private readonly lastQuarantinedAt = new Map(); + + private allQuarantinedTimer?: NodeJS.Timeout; + + private aborted?: Error; + constructor( private readonly lanes: L[], private readonly maxConcurrentBuilds: number, + private readonly health?: LaneHealthOptions, ) {} - async acquire(key: string): Promise { - const lane = this.findFree(key); + async acquire(key: string, opts?: { not?: L }): Promise { + if (this.aborted) throw this.aborted; + // Prefer a lane other than `not` (e.g. retrying a build that just failed + // there), but fall back to it rather than starving. + const lane = this.findFree(key, opts?.not) ?? this.findFree(key); if (lane) { this.markBusy(lane, key); return lane; } - return await new Promise((resolve) => { - this.waiters.push({ key, resolve }); + return await new Promise((resolve, reject) => { + this.waiters.push({ key, resolve, reject }); }); } @@ -37,19 +75,104 @@ export class LaneAllocator { this.wakeNext(lane); } - private findFree(key: string): L | undefined { + /** A completed build — even one the agent failed — is 'ok'; only + * network-level failures count toward quarantine. */ + reportBuildOutcome(lane: L, outcome: 'ok' | 'transient-failure'): void { + if (outcome === 'ok') { + this.consecutiveFailures.delete(lane); + return; + } + const failures = (this.consecutiveFailures.get(lane) ?? 0) + 1; + this.consecutiveFailures.set(lane, failures); + const threshold = this.health?.quarantineThreshold ?? DEFAULT_QUARANTINE_THRESHOLD; + if (failures >= threshold && !this.quarantined.has(lane)) this.quarantine(lane); + } + + isQuarantined(lane: L): boolean { + return this.quarantined.has(lane); + } + + /** Attributes a slow failure (e.g. a build timeout) to a lane death that + * happened mid-flight, even if the lane has since restarted. */ + wasQuarantinedSince(lane: L, sinceMs: number): boolean { + const at = this.lastQuarantinedAt.get(lane); + return at !== undefined && at >= sinceMs; + } + + private quarantine(lane: L): void { + this.quarantined.add(lane); + this.lastQuarantinedAt.set(lane, Date.now()); + this.health?.onQuarantine?.(lane); + if (this.health?.probe) this.scheduleProbe(lane); + if (this.quarantined.size === this.lanes.length) { + this.health?.onAllQuarantined?.(); + const graceMs = this.health?.allQuarantinedGraceMs ?? DEFAULT_ALL_QUARANTINED_GRACE_MS; + this.allQuarantinedTimer = setTimeout(() => { + if (this.quarantined.size === this.lanes.length) { + this.abort( + new Error( + `All ${String(this.lanes.length)} lanes quarantined for ${String(graceMs)}ms — no healthy backend to build on`, + ), + ); + } + }, graceMs); + // Deliberately referenced (no unref): with every lane dead, queued + // acquire() promises hold no live handles — an unref'd deadline would + // let the process exit 0 mid-run instead of aborting loudly. readmit() + // clears it as soon as any lane recovers. + } + } + + private scheduleProbe(lane: L): void { + const intervalMs = this.health?.probeIntervalMs ?? DEFAULT_PROBE_INTERVAL_MS; + const timer = setTimeout(() => { + if (!this.quarantined.has(lane) || this.aborted) return; + this.health + ?.probe(lane) + .then((healthy) => { + if (healthy) this.readmit(lane); + else this.scheduleProbe(lane); + }) + .catch(() => this.scheduleProbe(lane)); + }, intervalMs); + timer.unref?.(); + } + + private readmit(lane: L): void { + this.quarantined.delete(lane); + this.consecutiveFailures.delete(lane); + if (this.allQuarantinedTimer) { + clearTimeout(this.allQuarantinedTimer); + this.allQuarantinedTimer = undefined; + } + this.health?.onReadmit?.(lane); + // A re-admitted lane is idle — wake every waiter it can serve. + let woke = true; + while (woke) woke = this.wakeNext(lane); + } + + private abort(error: Error): void { + this.aborted = error; + for (const w of this.waiters.splice(0)) w.reject(error); + } + + private findFree(key: string, not?: L): L | undefined { // Least-loaded policy: spread builds evenly across lanes rather than // filling lane 0 to cap before touching lane 1. Avoids hot-spotting. let best: L | undefined; for (const lane of this.lanes) { - if (!this.canRun(lane, key)) continue; + if (lane === not || !this.canRun(lane, key)) continue; if (best === undefined || lane.activeBuilds < best.activeBuilds) best = lane; } return best; } private canRun(lane: L, key: string): boolean { - return lane.activeBuilds < this.maxConcurrentBuilds && !lane.inflightKeys.has(key); + return ( + !this.quarantined.has(lane) && + lane.activeBuilds < this.maxConcurrentBuilds && + !lane.inflightKeys.has(key) + ); } private markBusy(lane: L, key: string): void { @@ -57,7 +180,7 @@ export class LaneAllocator { lane.inflightKeys.add(key); } - private wakeNext(lane: L): void { + private wakeNext(lane: L): boolean { // Wake the first waiter this lane can now serve. FIFO ordering. for (let i = 0; i < this.waiters.length; i++) { const w = this.waiters[i]; @@ -65,8 +188,9 @@ export class LaneAllocator { this.waiters.splice(i, 1); this.markBusy(lane, w.key); w.resolve(lane); - return; + return true; } } + return false; } } diff --git a/packages/@n8n/instance-ai/evaluations/clients/n8n-client.ts b/packages/@n8n/instance-ai/evaluations/clients/n8n-client.ts index cdf67e6543a..5cf0e212830 100644 --- a/packages/@n8n/instance-ai/evaluations/clients/n8n-client.ts +++ b/packages/@n8n/instance-ai/evaluations/clients/n8n-client.ts @@ -233,9 +233,12 @@ export class N8nClient { * List captured LLM debug runs for a thread. * GET /rest/instance-ai/debug/threads/:threadId/runs */ - async listThreadDebugRuns(threadId: string): Promise { + async listThreadDebugRuns( + threadId: string, + timeoutMs?: number, + ): Promise { return this.unwrapRestData( - await this.fetch(`/rest/instance-ai/debug/threads/${threadId}/runs`), + await this.fetch(`/rest/instance-ai/debug/threads/${threadId}/runs`, { timeoutMs }), ); } @@ -243,9 +246,9 @@ export class N8nClient { * Fetch full LLM step debug for a single run. * GET /rest/instance-ai/debug/runs/:runId */ - async getRunDebug(runId: string): Promise { + async getRunDebug(runId: string, timeoutMs?: number): Promise { return this.unwrapRestData( - await this.fetch(`/rest/instance-ai/debug/runs/${runId}`), + await this.fetch(`/rest/instance-ai/debug/runs/${runId}`, { timeoutMs }), ); } diff --git a/packages/@n8n/instance-ai/evaluations/comparison/fetch-baseline.ts b/packages/@n8n/instance-ai/evaluations/comparison/fetch-baseline.ts index f77cadf2ba0..1ab05782ab8 100644 --- a/packages/@n8n/instance-ai/evaluations/comparison/fetch-baseline.ts +++ b/packages/@n8n/instance-ai/evaluations/comparison/fetch-baseline.ts @@ -62,10 +62,39 @@ const outputsSchema = z }) .passthrough(); +/** How many newest candidates to probe for the completion marker. Wide enough + * that a burst of failed captures can't hide an older completed baseline + * behind the probe window; only the no-marker-found path pays the extra + * reads. The plain-newest fallback below stays for pre-marker cohorts. */ +const MAX_COMPLETION_PROBES = 25; + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + /** - * Return the most recently created baseline experiment, or `undefined` if - * none exist. We pick by `start_time` so a re-run of an older snapshot - * doesn't displace the latest one. + * A capture killed mid-run (job timeout, runner death) leaves a partial + * experiment with the baseline prefix. The CLI writes aggregate metadata + * (`pass_rate_per_iter`) only at successful run end — treat it as the + * completion marker so partial captures never become the comparison target. + */ +async function hasCompletionMarker(client: Client, projectName: string): Promise { + try { + const project = await client.readProject({ projectName }); + const extra: unknown = project.extra; + if (!isRecord(extra) || !isRecord(extra.metadata)) return false; + return 'pass_rate_per_iter' in extra.metadata; + } catch { + return false; + } +} + +/** + * Return the most recently created COMPLETED baseline experiment, or + * `undefined` if none exist. We pick by `start_time` so a re-run of an older + * snapshot doesn't displace the latest one, and require the completion marker + * so a killed capture doesn't either (falling back to the plain newest when + * no probed candidate carries the marker, e.g. legacy cohorts). * * `prefix` defaults to the Instance AI baseline. Pass a different prefix (e.g. * `mcp-baseline-`) to scope the lookup to an isolated cohort, so an MCP run @@ -75,14 +104,18 @@ export async function findLatestBaseline( client: Client, prefix: string = BASELINE_EXPERIMENT_PREFIX, ): Promise { - let latest: { name: string; ts: number } | undefined; + const candidates: Array<{ name: string; ts: number }> = []; for await (const project of client.listProjects({ nameContains: prefix })) { const name = project.name; if (!name?.startsWith(prefix)) continue; const ts = project.start_time ? new Date(project.start_time).getTime() : 0; - if (!latest || ts > latest.ts) latest = { name, ts }; + candidates.push({ name, ts }); } - return latest?.name; + candidates.sort((a, b) => b.ts - a.ts); + for (const candidate of candidates.slice(0, MAX_COMPLETION_PROBES)) { + if (await hasCompletionMarker(client, candidate.name)) return candidate.name; + } + return candidates[0]?.name; } /** diff --git a/packages/@n8n/instance-ai/evaluations/harness/capture-run-debug.ts b/packages/@n8n/instance-ai/evaluations/harness/capture-run-debug.ts index 95aa7596c18..c767716f4e2 100644 --- a/packages/@n8n/instance-ai/evaluations/harness/capture-run-debug.ts +++ b/packages/@n8n/instance-ai/evaluations/harness/capture-run-debug.ts @@ -6,13 +6,46 @@ import type { N8nClient } from '../clients/n8n-client'; const RUN_FETCH_CONCURRENCY = 4; +/** Hard deadline for the whole capture. The per-case cleanup path awaits this + * before deleting the thread, so a stalled debug endpoint would otherwise + * block that row — and with it the run — indefinitely. */ +const CAPTURE_DEADLINE_MS = 60_000; + +/** Per-request abort so a stalled call releases its socket instead of running + * on after the outer deadline resolves the race. */ +const REQUEST_TIMEOUT_MS = 20_000; + export async function captureThreadRunDebug( client: N8nClient, threadId: string, logger?: EvalLogger, ): Promise { + let deadline: NodeJS.Timeout | undefined; + const timedOut = new Promise((resolve) => { + deadline = setTimeout(() => { + logger?.warn( + ` Run debug capture timed out for thread ${threadId} after ${String(CAPTURE_DEADLINE_MS)}ms`, + ); + resolve([]); + }, CAPTURE_DEADLINE_MS); + // Best-effort capture must never keep the process alive on its own. + deadline.unref?.(); + }); + try { - const response = await client.listThreadDebugRuns(threadId); + return await Promise.race([captureAllRuns(client, threadId, logger), timedOut]); + } finally { + if (deadline) clearTimeout(deadline); + } +} + +async function captureAllRuns( + client: N8nClient, + threadId: string, + logger?: EvalLogger, +): Promise { + try { + const response = await client.listThreadDebugRuns(threadId, REQUEST_TIMEOUT_MS); const runs = response.runs ?? []; if (runs.length === 0) { logger?.verbose(` No run debug records for thread ${threadId}`); @@ -25,7 +58,7 @@ export async function captureThreadRunDebug( async (summary) => await limit(async (): Promise => { try { - const record = await client.getRunDebug(summary.runId); + const record = await client.getRunDebug(summary.runId, REQUEST_TIMEOUT_MS); return summary.label ? { ...record, label: summary.label } : record; } catch (error: unknown) { logger?.warn( diff --git a/packages/@n8n/instance-ai/evaluations/harness/runner.ts b/packages/@n8n/instance-ai/evaluations/harness/runner.ts index dedc5978925..151adfb1f81 100644 --- a/packages/@n8n/instance-ai/evaluations/harness/runner.ts +++ b/packages/@n8n/instance-ai/evaluations/harness/runner.ts @@ -534,6 +534,9 @@ export interface BuildResult { * gone, reconstruction drift, restore failed) — a harness/framework problem, * not an agent build failure. Routed to `framework_issue`. */ seedingFailed?: boolean; + /** Transport-level failure (network error, or the lane unreachable right + * after failing — e.g. timed out against a dead lane). Routed to `framework_issue`. */ + transportFailure?: boolean; } export interface BuildWorkflowConfig { @@ -923,17 +926,21 @@ export async function executeScenario( /** * Clean up workflows and data tables created during a build. + * + * Returns false when any deletion failed so callers can retry later. */ export async function cleanupBuild( client: N8nClient, build: BuildResult, logger: EvalLogger, -): Promise { +): Promise { + let clean = true; + for (const id of build.createdWorkflowIds) { try { await client.deleteWorkflow(id); } catch { - // Best-effort cleanup + clean = false; // Best-effort cleanup } } @@ -944,14 +951,26 @@ export async function cleanupBuild( try { await client.deleteDataTable(projectId, dtId); } catch { - // Best-effort cleanup + clean = false; // Best-effort cleanup } } logger.verbose(` Cleaned up ${String(build.createdDataTableIds.length)} data table(s)`); } catch { - // Non-fatal — project ID lookup may fail + clean = false; // Non-fatal — project ID lookup may fail } } + + // Clears backend thread state (run-state registries, memory) that otherwise + // grows one entry per build for the container's lifetime. + if (build.threadId) { + try { + await client.deleteThread(build.threadId); + } catch { + clean = false; // Best-effort cleanup + } + } + + return clean; } // --------------------------------------------------------------------------- diff --git a/packages/@n8n/instance-ai/scripts/run-eval-lanes.sh b/packages/@n8n/instance-ai/scripts/run-eval-lanes.sh index d86bc8fdaa5..8518ae96333 100755 --- a/packages/@n8n/instance-ai/scripts/run-eval-lanes.sh +++ b/packages/@n8n/instance-ai/scripts/run-eval-lanes.sh @@ -335,8 +335,16 @@ for port in "${PORTS[@]}"; do docker rm -f "$name" >/dev/null 2>&1 || true fi + # Same bounds as CI (test-evals-instance-ai.yml): capped + restartable + # lanes, pruned executions. docker run -d --name "$name" \ --env-file "$ENV_FILE_PATH" \ + --memory 2.5g --memory-swap 2.5g \ + --restart on-failure \ + --log-opt max-size=50m --log-opt max-file=2 \ + -e NODE_OPTIONS=--max-old-space-size=2048 \ + -e EXECUTIONS_DATA_PRUNE=true \ + -e EXECUTIONS_DATA_MAX_AGE=1 \ -e E2E_TESTS=true \ -e N8N_USER_FOLDER=/home/node/.n8n \ -p "${port}:5678" \