From 7d6fd4ca9c6f55b7cef026d4a42e4ae9c115e153 Mon Sep 17 00:00:00 2001 From: Mutasem Aldmour <4711238+mutdmour@users.noreply.github.com> Date: Fri, 17 Jul 2026 16:23:10 +0200 Subject: [PATCH] fix: Report an unverifiable unit as notVerified, not a silent pass (#34421) Co-authored-by: Claude Opus 4.8 (1M context) --- .../evaluations/__tests__/aggregator.test.ts | 70 +++++++++++ .../__tests__/bucket-from-evaluation.test.ts | 1 + .../__tests__/comparison-format.test.ts | 5 + .../__tests__/comparison-gate.test.ts | 5 + .../fixtures/eval-results.not-verified.json | 94 ++++++++++++++ .../fixtures/eval-results.partial-abort.json | 3 + .../evaluations/__tests__/summary.test.ts | 71 ++++++++++- .../__tests__/write-eval-results.test.ts | 119 ++++++++++++++++++ .../instance-ai/evaluations/cli/aggregator.ts | 19 +++ .../@n8n/instance-ai/evaluations/cli/index.ts | 8 ++ .../@n8n/instance-ai/evaluations/summary.ts | 26 ++++ .../@n8n/instance-ai/evaluations/types.ts | 12 ++ 12 files changed, 431 insertions(+), 2 deletions(-) create mode 100644 packages/@n8n/instance-ai/evaluations/__tests__/fixtures/eval-results.not-verified.json create mode 100644 packages/@n8n/instance-ai/evaluations/__tests__/write-eval-results.test.ts diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/aggregator.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/aggregator.test.ts index cb959166ce2..6da0a08be0c 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/aggregator.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/aggregator.test.ts @@ -97,6 +97,76 @@ describe('aggregateResults — verifier-incomplete scenario runs', () => { }); }); +describe('aggregateResults — case verification status', () => { + it('marks a case notVerified when every scenario run is incomplete', () => { + const allRuns = [ + [makeRunResult({ success: false, incomplete: true })], + [makeRunResult({ success: false, incomplete: true })], + ]; + + const evaluation = aggregateResults(allRuns, 2); + + expect(evaluation.testCases[0].status).toBe('notVerified'); + }); + + it('marks a case verified when at least one scenario run was evaluated', () => { + const allRuns = [ + [makeRunResult({ success: true })], + [makeRunResult({ success: false, incomplete: true })], + ]; + + const evaluation = aggregateResults(allRuns, 2); + + expect(evaluation.testCases[0].status).toBe('verified'); + }); + + it('marks a build-failed case verified — a build failure is a verified failure, not a gap', () => { + // Build failures substitute a non-incomplete "scenario not executed" result, + // so they count as evaluated failures rather than an unverifiable gap. + const buildFailedCase: WorkflowTestCase = { + ...incompleteTestCase, + }; + const allRuns = [ + [ + { + testCase: buildFailedCase, + workflowBuildSuccess: false, + executionScenarioResults: [], + }, + ], + ]; + + const evaluation = aggregateResults(allRuns, 1); + + expect(evaluation.testCases[0].executionScenarios[0].evaluatedCount).toBe(1); + expect(evaluation.testCases[0].status).toBe('verified'); + }); + + it('marks a case notVerified when its only expectations were skipped (prebuilt process gap)', () => { + // A process-only case run without a transcript (prebuilt/MCP) judges nothing: + // the expectation is absent from every run, so nothing could be verified. + const processOnlyCase: WorkflowTestCase = { + ...incompleteTestCase, + executionScenarios: undefined, + processExpectations: ['asks before building'], + }; + const allRuns = [ + [ + { + testCase: processOnlyCase, + workflowBuildSuccess: true, + executionScenarioResults: [], + buildExpectationResults: [], + }, + ], + ]; + + const evaluation = aggregateResults(allRuns, 1); + + expect(evaluation.testCases[0].status).toBe('notVerified'); + }); +}); + describe('aggregateResults — build expectations as units', () => { const expectationCase: WorkflowTestCase = { ...incompleteTestCase, diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/bucket-from-evaluation.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/bucket-from-evaluation.test.ts index 6652c81d240..188d9199da5 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/bucket-from-evaluation.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/bucket-from-evaluation.test.ts @@ -75,6 +75,7 @@ function fixture(): { evaluation: MultiRunEvaluation; withFiles: WorkflowTestCas expectationAggregation('asks before building', 2, 3), expectationAggregation('never judged', 0, 0), ], + status: 'verified', }, ], }; diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/comparison-format.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/comparison-format.test.ts index 0affc096bc1..2d465d47d47 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/comparison-format.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/comparison-format.test.ts @@ -132,6 +132,11 @@ function evaluation( })), buildSuccessCount, buildExpectations, + status: + scenarios.some((sa) => sa.evaluatedCount > 0) || + buildExpectations.some((ea) => ea.evaluatedCount > 0) + ? ('verified' as const) + : ('notVerified' as const), }; }), }; diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/comparison-gate.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/comparison-gate.test.ts index 76860ba833a..424422dd2e0 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/comparison-gate.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/comparison-gate.test.ts @@ -97,6 +97,11 @@ function makeEval(totalRuns: number, cases: CaseSpec[]) { buildSuccessCount: totalRuns, executionScenarios: scenarioAggs, buildExpectations, + status: + scenarioAggs.some((sa) => sa.evaluatedCount > 0) || + buildExpectations.some((ea) => ea.evaluatedCount > 0) + ? ('verified' as const) + : ('notVerified' as const), }; }); const evaluation: MultiRunEvaluation = { totalRuns, testCases }; diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/fixtures/eval-results.not-verified.json b/packages/@n8n/instance-ai/evaluations/__tests__/fixtures/eval-results.not-verified.json new file mode 100644 index 00000000000..8997166eff8 --- /dev/null +++ b/packages/@n8n/instance-ai/evaluations/__tests__/fixtures/eval-results.not-verified.json @@ -0,0 +1,94 @@ +{ + "timestamp": "2026-07-17T00:00:00.000Z", + "duration": 1234, + "totalRuns": 2, + "experimentName": "exp-not-verified-fixture", + "summary": { + "testCases": 2, + "built": 2, + "scenariosTotal": 2, + "passAtK": 1, + "passHatK": 0.25, + "passRatePerIter": "100% / 0%", + "notVerified": 1 + }, + "comparisonStatus": "not_attempted", + "testCases": [ + { + "name": "build it", + "testCaseFile": "behavioral-not-verified", + "status": "notVerified", + "buildSuccessCount": 2, + "totalRuns": 2, + "workflowChecksPerRun": [null, null], + "buildExpectationResultsPerRun": [null, null], + "buildExpectations": [], + "threadIds": [null, null], + "scenarios": [ + { + "name": "happy-path", + "passCount": 0, + "evaluatedCount": 0, + "totalRuns": 2, + "passAtK": 0, + "passHatK": 0, + "runs": [ + { + "workflowId": null, + "passed": false, + "incomplete": true, + "score": 0, + "reasoning": "nope", + "execErrors": [] + }, + { + "workflowId": null, + "passed": false, + "incomplete": true, + "score": 0, + "reasoning": "nope", + "execErrors": [] + } + ] + } + ] + }, + { + "name": "build it", + "testCaseFile": "scenario-verified", + "status": "verified", + "buildSuccessCount": 2, + "totalRuns": 2, + "workflowChecksPerRun": [null, null], + "buildExpectationResultsPerRun": [null, null], + "buildExpectations": [], + "threadIds": [null, null], + "scenarios": [ + { + "name": "happy-path", + "passCount": 1, + "evaluatedCount": 2, + "totalRuns": 2, + "passAtK": 1, + "passHatK": 0.25, + "runs": [ + { + "workflowId": null, + "passed": true, + "score": 1, + "reasoning": "ok", + "execErrors": [] + }, + { + "workflowId": null, + "passed": false, + "score": 0, + "reasoning": "nope", + "execErrors": [] + } + ] + } + ] + } + ] +} diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/fixtures/eval-results.partial-abort.json b/packages/@n8n/instance-ai/evaluations/__tests__/fixtures/eval-results.partial-abort.json index 497b512ca5e..ec5d36cf3e7 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/fixtures/eval-results.partial-abort.json +++ b/packages/@n8n/instance-ai/evaluations/__tests__/fixtures/eval-results.partial-abort.json @@ -3,6 +3,7 @@ "summary": { "testCases": 2, "built": 1, + "notVerified": 0, "scenariosTotal": 2, "passAtK": 0.5, "passHatK": 0.5, @@ -12,6 +13,7 @@ "testCases": [ { "name": "build me something", + "status": "verified", "buildSuccessCount": 1, "totalRuns": 1, "workflowChecksPerRun": [null], @@ -40,6 +42,7 @@ }, { "name": "build me something", + "status": "verified", "buildSuccessCount": 0, "totalRuns": 1, "workflowChecksPerRun": [null], diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/summary.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/summary.test.ts index 53312fc7eb1..d2def4e6c10 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/summary.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/summary.test.ts @@ -1,7 +1,74 @@ -import { getCaseRunStatus, getCheckedRunCount, getRunScoredCounts } from '../summary'; -import type { TestCaseAggregation, WorkflowTestCaseResult } from '../types'; +import { + getCaseRunStatus, + getCheckedRunCount, + getRunScoredCounts, + rollupCaseVerification, +} from '../summary'; +import type { + ExecutionScenarioAggregation, + TestCaseAggregation, + WorkflowTestCaseResult, +} from '../types'; import { baseTestCase } from './fixtures'; +describe('rollupCaseVerification', () => { + function scenarioAgg( + overrides: Partial = {}, + ): ExecutionScenarioAggregation { + return { + scenario: { name: 's', description: '', dataSetup: '', successCriteria: 'ok' }, + runs: [], + evaluatedCount: 1, + passCount: 1, + passRate: 1, + passAtK: [1], + passHatK: [1], + ...overrides, + }; + } + + function caseAgg(overrides: Partial): TestCaseAggregation { + return { + testCase: baseTestCase(), + runs: [], + buildSuccessCount: 1, + executionScenarios: [], + buildExpectations: [], + status: 'verified', + ...overrides, + }; + } + + it('counts a not-verified case separately from passed and failed', () => { + const cases: TestCaseAggregation[] = [ + caseAgg({ status: 'verified', executionScenarios: [scenarioAgg({ passCount: 1 })] }), + caseAgg({ + status: 'verified', + executionScenarios: [scenarioAgg({ passCount: 0, passRate: 0 })], + }), + caseAgg({ + status: 'notVerified', + executionScenarios: [scenarioAgg({ evaluatedCount: 0, passCount: 0, passRate: 0 })], + }), + ]; + + expect(rollupCaseVerification(cases)).toEqual({ passed: 1, failed: 1, notVerified: 1 }); + }); + + it('does not count a not-verified case as passed', () => { + const cases: TestCaseAggregation[] = [ + caseAgg({ + status: 'notVerified', + executionScenarios: [scenarioAgg({ evaluatedCount: 0, passCount: 0, passRate: 0 })], + }), + ]; + + const rollup = rollupCaseVerification(cases); + expect(rollup.passed).toBe(0); + expect(rollup.notVerified).toBe(1); + }); +}); + // A scenario-less case (e.g. an agent build that saves its artifact outside the workflow path) // legitimately produces no workflow (workflowBuildSuccess: false). These lock in that such a run // is scored on its outcome expectations — which cover the rendered agent/config-eval context — diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/write-eval-results.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/write-eval-results.test.ts new file mode 100644 index 00000000000..d220d62ae74 --- /dev/null +++ b/packages/@n8n/instance-ai/evaluations/__tests__/write-eval-results.test.ts @@ -0,0 +1,119 @@ +import { mkdtempSync, readFileSync } from 'fs'; +import { jsonParse } from 'n8n-workflow'; +import { tmpdir } from 'os'; +import { join } from 'path'; + +import { aggregateResults } from '../cli/aggregator'; +import { writeEvalResults } from '../cli/index'; +import type { ExecutionScenario, WorkflowTestCase, WorkflowTestCaseResult } from '../types'; + +// The lang-tracer dispatcher reads `eval-results.json`, not the in-memory +// aggregation. These tests pin that the `notVerified` status actually survives +// serialization: per-case `testCases[].status` and top-level `summary.notVerified`. + +const scenario: ExecutionScenario = { + name: 'happy-path', + description: 'baseline', + dataSetup: 'plain', + successCriteria: 'works', +}; + +function scenarioCase(): WorkflowTestCase { + return { + conversation: [{ role: 'user', text: 'build it' }], + complexity: 'simple', + tags: [], + executionScenarios: [scenario], + datasets: ['full'], + }; +} + +function runResult( + testCase: WorkflowTestCase, + run: { success: boolean; incomplete?: boolean }, +): WorkflowTestCaseResult { + return { + testCase, + workflowBuildSuccess: true, + executionScenarioResults: [ + { + scenario, + success: run.success, + score: run.success ? 1 : 0, + reasoning: run.success ? 'ok' : 'nope', + ...(run.incomplete ? { incomplete: true } : {}), + }, + ], + }; +} + +interface SerializedResults { + summary: { notVerified: number }; + testCases: Array<{ status: string; testCaseFile?: string }>; +} + +function writeAndRead(): SerializedResults { + const notVerifiedCase = scenarioCase(); + const verifiedCase = scenarioCase(); + + // Two iterations. Case 0: every run incomplete → notVerified. + // Case 1: one pass, one fail → verified. + const evaluation = aggregateResults( + [ + [ + runResult(notVerifiedCase, { success: false, incomplete: true }), + runResult(verifiedCase, { success: true }), + ], + [ + runResult(notVerifiedCase, { success: false, incomplete: true }), + runResult(verifiedCase, { success: false }), + ], + ], + 2, + ); + + const slugByTestCase = new Map([ + [notVerifiedCase, 'behavioral-not-verified'], + [verifiedCase, 'scenario-verified'], + ]); + + const dir = mkdtempSync(join(tmpdir(), 'eval-results-test-')); + const { jsonPath } = writeEvalResults( + evaluation, + 1234, + dir, + 'exp-test', + undefined, + undefined, + slugByTestCase, + undefined, + undefined, + ); + + return jsonParse(readFileSync(jsonPath, 'utf8')); +} + +describe('writeEvalResults — notVerified serialization', () => { + it('serializes per-case status and the top-level notVerified count', () => { + const report = writeAndRead(); + + const byFile = new Map(report.testCases.map((tc) => [tc.testCaseFile, tc.status])); + expect(byFile.get('behavioral-not-verified')).toBe('notVerified'); + expect(byFile.get('scenario-verified')).toBe('verified'); + expect(report.summary.notVerified).toBe(1); + }); + + // The committed fixture is the cross-repo contract anchor consumed by the + // lang-tracer dispatcher test. Guard it so the pinned fields can't silently drift. + it('matches the committed golden fixture on the pinned fields', () => { + const golden = jsonParse( + readFileSync(join(__dirname, 'fixtures', 'eval-results.not-verified.json'), 'utf8'), + ); + + const live = writeAndRead(); + expect(golden.summary.notVerified).toBe(live.summary.notVerified); + expect(golden.testCases.map((tc) => [tc.testCaseFile, tc.status])).toEqual( + live.testCases.map((tc) => [tc.testCaseFile, tc.status]), + ); + }); +}); diff --git a/packages/@n8n/instance-ai/evaluations/cli/aggregator.ts b/packages/@n8n/instance-ai/evaluations/cli/aggregator.ts index 613608cb89c..e7676c75012 100644 --- a/packages/@n8n/instance-ai/evaluations/cli/aggregator.ts +++ b/packages/@n8n/instance-ai/evaluations/cli/aggregator.ts @@ -6,6 +6,7 @@ import type { ExecutionScenarioAggregation, BuildExpectationAggregation, BuildExpectationResult, + CaseVerificationStatus, } from '../types'; /** @@ -85,6 +86,23 @@ function aggregateUnit( }; } +/** + * A case is `notVerified` when nothing could be scored: every scenario and every + * build expectation came back with 0 evaluated runs (all incomplete, or skipped — + * e.g. process expectations with no transcript). A build FAILURE is NOT a gap: the + * aggregator substitutes a non-incomplete "scenario not executed" result, so a + * build-failed case has evaluated (failing) scenarios and stays `verified`. + */ +function computeCaseStatus( + scenarios: ExecutionScenarioAggregation[], + expectations: BuildExpectationAggregation[], +): CaseVerificationStatus { + const evaluatedUnits = + scenarios.reduce((n, sa) => n + sa.evaluatedCount, 0) + + expectations.reduce((n, ea) => n + ea.evaluatedCount, 0); + return evaluatedUnits > 0 ? 'verified' : 'notVerified'; +} + export function aggregateResults( allRunResults: WorkflowTestCaseResult[][], totalRuns: number, @@ -147,6 +165,7 @@ export function aggregateResults( buildSuccessCount, executionScenarios, buildExpectations, + status: computeCaseStatus(executionScenarios, buildExpectations), }); } diff --git a/packages/@n8n/instance-ai/evaluations/cli/index.ts b/packages/@n8n/instance-ai/evaluations/cli/index.ts index 6a0f8d427ed..9234c9e3372 100644 --- a/packages/@n8n/instance-ai/evaluations/cli/index.ts +++ b/packages/@n8n/instance-ai/evaluations/cli/index.ts @@ -95,6 +95,7 @@ import { seedMcpRegistry } from '../mcp-registry/seeder'; import { snapshotWorkflowIds } from '../outcome/workflow-discovery'; import { writeRunDebugReport } from '../report/run-debug-report'; import { writeWorkflowReport } from '../report/workflow-report'; +import { rollupCaseVerification } from '../summary'; import type { BuildExpectationResult, MultiRunEvaluation, @@ -1880,6 +1881,7 @@ export function writeEvalResults( ): { jsonPath: string; prCommentPath: string } { const { totalRuns, testCases } = evaluation; const metrics = computeAggregateMetrics(evaluation); + const verification = rollupCaseVerification(testCases); const result = outcome?.kind === 'ok' ? outcome.result : undefined; @@ -1901,6 +1903,9 @@ export function writeEvalResults( passAtK: metrics.passAtK, passHatK: metrics.passHatK, passRatePerIter: metrics.passRatePerIter, + // Cases where nothing could be scored (all units incomplete / skipped) — + // reported apart from the pass rate, never as a silent pass. + notVerified: verification.notVerified, ...(checksSummary ? { workflowChecks: checksSummary } : {}), ...(buildSpendSummary ? { mcpBuild: buildSpendSummary } : {}), }, @@ -1922,6 +1927,9 @@ export function writeEvalResults( testCases: testCases.map((tc) => ({ name: caseDisplayPrompt(tc.testCase, tc.runs[0]?.transcript).slice(0, 70), testCaseFile: slugByTestCase?.get(tc.testCase), + // `notVerified` when no scenario or expectation was scored across runs — + // consumers must not treat a zero pass rate here as a pass. + status: tc.status, buildSuccessCount: tc.buildSuccessCount, totalRuns, workflowChecksPerRun: tc.runs.map((run) => diff --git a/packages/@n8n/instance-ai/evaluations/summary.ts b/packages/@n8n/instance-ai/evaluations/summary.ts index ed24650dee4..d921fd6d4c1 100644 --- a/packages/@n8n/instance-ai/evaluations/summary.ts +++ b/packages/@n8n/instance-ai/evaluations/summary.ts @@ -73,6 +73,32 @@ export function getCaseRunStatus(result: WorkflowTestCaseResult): CaseRunStatus return 'build_failed'; } +/** Per-case verification tallies across an aggregated run. `notVerified` cases — + * where nothing could be scored — are counted apart from passed/failed so the + * reported pass rate reflects only what was actually checked. */ +export interface CaseVerificationRollup { + /** Verified cases where every evaluated unit passed. */ + passed: number; + /** Verified cases with at least one failing evaluated unit. */ + failed: number; + /** Cases where no unit could be scored (all incomplete / skipped). */ + notVerified: number; +} + +export function rollupCaseVerification(cases: TestCaseAggregation[]): CaseVerificationRollup { + const rollup: CaseVerificationRollup = { passed: 0, failed: 0, notVerified: 0 }; + for (const tc of cases) { + if (tc.status === 'notVerified') { + rollup.notVerified++; + continue; + } + const { passCount, totalCount } = countAggregatedUnitTrials(getAggregatedCaseUnits(tc)); + if (totalCount > 0 && passCount === totalCount) rollup.passed++; + else rollup.failed++; + } + return rollup; +} + export function getCaseRunStatusLabel(result: WorkflowTestCaseResult): string { switch (getCaseRunStatus(result)) { case 'checked': diff --git a/packages/@n8n/instance-ai/evaluations/types.ts b/packages/@n8n/instance-ai/evaluations/types.ts index 2e6c360ac4f..d19bd93aabb 100644 --- a/packages/@n8n/instance-ai/evaluations/types.ts +++ b/packages/@n8n/instance-ai/evaluations/types.ts @@ -426,6 +426,15 @@ export interface BuildExpectationAggregation { passHatK: number[]; } +/** + * Whether a case produced any scoreable verdict across its runs. + * - `verified` — at least one scenario or build expectation was evaluated. + * - `notVerified` — every measured unit came back incomplete / was skipped (no + * transcript for process expectations, verifier gaps for scenarios), so nothing + * could actually be checked. Such a case MUST NOT roll up as a silent pass. + */ +export type CaseVerificationStatus = 'verified' | 'notVerified'; + export interface TestCaseAggregation { testCase: WorkflowTestCase; runs: WorkflowTestCaseResult[]; @@ -433,6 +442,9 @@ export interface TestCaseAggregation { executionScenarios: ExecutionScenarioAggregation[]; /** Build expectations aggregated as measured units (counted in the pass rate). */ buildExpectations: BuildExpectationAggregation[]; + /** `notVerified` when no unit (scenario or expectation) was evaluated across + * all runs — nothing could be checked, so the case is not a pass. */ + status: CaseVerificationStatus; } export interface MultiRunEvaluation {