fix: Report an unverifiable unit as notVerified, not a silent pass (#34421)

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

packages/@n8n/instance-ai/evaluations/__tests__/aggregator.test.ts

@@ -97,6 +97,76 @@ describe('aggregateResults — verifier-incomplete scenario runs', () => {
 	});
 });
 
+describe('aggregateResults — case verification status', () => {
+	it('marks a case notVerified when every scenario run is incomplete', () => {
+		const allRuns = [
+			[makeRunResult({ success: false, incomplete: true })],
+			[makeRunResult({ success: false, incomplete: true })],
+		];
+
+		const evaluation = aggregateResults(allRuns, 2);
+
+		expect(evaluation.testCases[0].status).toBe('notVerified');
+	});
+
+	it('marks a case verified when at least one scenario run was evaluated', () => {
+		const allRuns = [
+			[makeRunResult({ success: true })],
+			[makeRunResult({ success: false, incomplete: true })],
+		];
+
+		const evaluation = aggregateResults(allRuns, 2);
+
+		expect(evaluation.testCases[0].status).toBe('verified');
+	});
+
+	it('marks a build-failed case verified — a build failure is a verified failure, not a gap', () => {
+		// Build failures substitute a non-incomplete "scenario not executed" result,
+		// so they count as evaluated failures rather than an unverifiable gap.
+		const buildFailedCase: WorkflowTestCase = {
+			...incompleteTestCase,
+		};
+		const allRuns = [
+			[
+				{
+					testCase: buildFailedCase,
+					workflowBuildSuccess: false,
+					executionScenarioResults: [],
+				},
+			],
+		];
+
+		const evaluation = aggregateResults(allRuns, 1);
+
+		expect(evaluation.testCases[0].executionScenarios[0].evaluatedCount).toBe(1);
+		expect(evaluation.testCases[0].status).toBe('verified');
+	});
+
+	it('marks a case notVerified when its only expectations were skipped (prebuilt process gap)', () => {
+		// A process-only case run without a transcript (prebuilt/MCP) judges nothing:
+		// the expectation is absent from every run, so nothing could be verified.
+		const processOnlyCase: WorkflowTestCase = {
+			...incompleteTestCase,
+			executionScenarios: undefined,
+			processExpectations: ['asks before building'],
+		};
+		const allRuns = [
+			[
+				{
+					testCase: processOnlyCase,
+					workflowBuildSuccess: true,
+					executionScenarioResults: [],
+					buildExpectationResults: [],
+				},
+			],
+		];
+
+		const evaluation = aggregateResults(allRuns, 1);
+
+		expect(evaluation.testCases[0].status).toBe('notVerified');
+	});
+});
+
 describe('aggregateResults — build expectations as units', () => {
 	const expectationCase: WorkflowTestCase = {
 		...incompleteTestCase,

packages/@n8n/instance-ai/evaluations/__tests__/bucket-from-evaluation.test.ts

@@ -75,6 +75,7 @@ function fixture(): { evaluation: MultiRunEvaluation; withFiles: WorkflowTestCas
 					expectationAggregation('asks before building', 2, 3),
 					expectationAggregation('never judged', 0, 0),
 				],
+				status: 'verified',
 			},
 		],
 	};

packages/@n8n/instance-ai/evaluations/__tests__/comparison-format.test.ts

@@ -132,6 +132,11 @@ function evaluation(
 				})),
 				buildSuccessCount,
 				buildExpectations,
+				status:
+					scenarios.some((sa) => sa.evaluatedCount > 0) ||
+					buildExpectations.some((ea) => ea.evaluatedCount > 0)
+						? ('verified' as const)
+						: ('notVerified' as const),
 			};
 		}),
 	};

packages/@n8n/instance-ai/evaluations/__tests__/comparison-gate.test.ts

@@ -97,6 +97,11 @@ function makeEval(totalRuns: number, cases: CaseSpec[]) {
 			buildSuccessCount: totalRuns,
 			executionScenarios: scenarioAggs,
 			buildExpectations,
+			status:
+				scenarioAggs.some((sa) => sa.evaluatedCount > 0) ||
+				buildExpectations.some((ea) => ea.evaluatedCount > 0)
+					? ('verified' as const)
+					: ('notVerified' as const),
 		};
 	});
 	const evaluation: MultiRunEvaluation = { totalRuns, testCases };

packages/@n8n/instance-ai/evaluations/__tests__/fixtures/eval-results.not-verified.json

@@ -0,0 +1,94 @@
+{
+	"timestamp": "2026-07-17T00:00:00.000Z",
+	"duration": 1234,
+	"totalRuns": 2,
+	"experimentName": "exp-not-verified-fixture",
+	"summary": {
+		"testCases": 2,
+		"built": 2,
+		"scenariosTotal": 2,
+		"passAtK": 1,
+		"passHatK": 0.25,
+		"passRatePerIter": "100% / 0%",
+		"notVerified": 1
+	},
+	"comparisonStatus": "not_attempted",
+	"testCases": [
+		{
+			"name": "build it",
+			"testCaseFile": "behavioral-not-verified",
+			"status": "notVerified",
+			"buildSuccessCount": 2,
+			"totalRuns": 2,
+			"workflowChecksPerRun": [null, null],
+			"buildExpectationResultsPerRun": [null, null],
+			"buildExpectations": [],
+			"threadIds": [null, null],
+			"scenarios": [
+				{
+					"name": "happy-path",
+					"passCount": 0,
+					"evaluatedCount": 0,
+					"totalRuns": 2,
+					"passAtK": 0,
+					"passHatK": 0,
+					"runs": [
+						{
+							"workflowId": null,
+							"passed": false,
+							"incomplete": true,
+							"score": 0,
+							"reasoning": "nope",
+							"execErrors": []
+						},
+						{
+							"workflowId": null,
+							"passed": false,
+							"incomplete": true,
+							"score": 0,
+							"reasoning": "nope",
+							"execErrors": []
+						}
+					]
+				}
+			]
+		},
+		{
+			"name": "build it",
+			"testCaseFile": "scenario-verified",
+			"status": "verified",
+			"buildSuccessCount": 2,
+			"totalRuns": 2,
+			"workflowChecksPerRun": [null, null],
+			"buildExpectationResultsPerRun": [null, null],
+			"buildExpectations": [],
+			"threadIds": [null, null],
+			"scenarios": [
+				{
+					"name": "happy-path",
+					"passCount": 1,
+					"evaluatedCount": 2,
+					"totalRuns": 2,
+					"passAtK": 1,
+					"passHatK": 0.25,
+					"runs": [
+						{
+							"workflowId": null,
+							"passed": true,
+							"score": 1,
+							"reasoning": "ok",
+							"execErrors": []
+						},
+						{
+							"workflowId": null,
+							"passed": false,
+							"score": 0,
+							"reasoning": "nope",
+							"execErrors": []
+						}
+					]
+				}
+			]
+		}
+	]
+}

packages/@n8n/instance-ai/evaluations/__tests__/fixtures/eval-results.partial-abort.json

@@ -3,6 +3,7 @@
 	"summary": {
 		"testCases": 2,
 		"built": 1,
+		"notVerified": 0,
 		"scenariosTotal": 2,
 		"passAtK": 0.5,
 		"passHatK": 0.5,
@@ -12,6 +13,7 @@
 	"testCases": [
 		{
 			"name": "build me something",
+			"status": "verified",
 			"buildSuccessCount": 1,
 			"totalRuns": 1,
 			"workflowChecksPerRun": [null],
@@ -40,6 +42,7 @@
 		},
 		{
 			"name": "build me something",
+			"status": "verified",
 			"buildSuccessCount": 0,
 			"totalRuns": 1,
 			"workflowChecksPerRun": [null],

packages/@n8n/instance-ai/evaluations/__tests__/summary.test.ts

@@ -1,7 +1,74 @@
-import { getCaseRunStatus, getCheckedRunCount, getRunScoredCounts } from '../summary';
-import type { TestCaseAggregation, WorkflowTestCaseResult } from '../types';
+import {
+	getCaseRunStatus,
+	getCheckedRunCount,
+	getRunScoredCounts,
+	rollupCaseVerification,
+} from '../summary';
+import type {
+	ExecutionScenarioAggregation,
+	TestCaseAggregation,
+	WorkflowTestCaseResult,
+} from '../types';
 import { baseTestCase } from './fixtures';
 
+describe('rollupCaseVerification', () => {
+	function scenarioAgg(
+		overrides: Partial<ExecutionScenarioAggregation> = {},
+	): ExecutionScenarioAggregation {
+		return {
+			scenario: { name: 's', description: '', dataSetup: '', successCriteria: 'ok' },
+			runs: [],
+			evaluatedCount: 1,
+			passCount: 1,
+			passRate: 1,
+			passAtK: [1],
+			passHatK: [1],
+			...overrides,
+		};
+	}
+
+	function caseAgg(overrides: Partial<TestCaseAggregation>): TestCaseAggregation {
+		return {
+			testCase: baseTestCase(),
+			runs: [],
+			buildSuccessCount: 1,
+			executionScenarios: [],
+			buildExpectations: [],
+			status: 'verified',
+			...overrides,
+		};
+	}
+
+	it('counts a not-verified case separately from passed and failed', () => {
+		const cases: TestCaseAggregation[] = [
+			caseAgg({ status: 'verified', executionScenarios: [scenarioAgg({ passCount: 1 })] }),
+			caseAgg({
+				status: 'verified',
+				executionScenarios: [scenarioAgg({ passCount: 0, passRate: 0 })],
+			}),
+			caseAgg({
+				status: 'notVerified',
+				executionScenarios: [scenarioAgg({ evaluatedCount: 0, passCount: 0, passRate: 0 })],
+			}),
+		];
+
+		expect(rollupCaseVerification(cases)).toEqual({ passed: 1, failed: 1, notVerified: 1 });
+	});
+
+	it('does not count a not-verified case as passed', () => {
+		const cases: TestCaseAggregation[] = [
+			caseAgg({
+				status: 'notVerified',
+				executionScenarios: [scenarioAgg({ evaluatedCount: 0, passCount: 0, passRate: 0 })],
+			}),
+		];
+
+		const rollup = rollupCaseVerification(cases);
+		expect(rollup.passed).toBe(0);
+		expect(rollup.notVerified).toBe(1);
+	});
+});
+
 // A scenario-less case (e.g. an agent build that saves its artifact outside the workflow path)
 // legitimately produces no workflow (workflowBuildSuccess: false). These lock in that such a run
 // is scored on its outcome expectations — which cover the rendered agent/config-eval context —

packages/@n8n/instance-ai/evaluations/__tests__/write-eval-results.test.ts

@@ -0,0 +1,119 @@
+import { mkdtempSync, readFileSync } from 'fs';
+import { jsonParse } from 'n8n-workflow';
+import { tmpdir } from 'os';
+import { join } from 'path';
+
+import { aggregateResults } from '../cli/aggregator';
+import { writeEvalResults } from '../cli/index';
+import type { ExecutionScenario, WorkflowTestCase, WorkflowTestCaseResult } from '../types';
+
+// The lang-tracer dispatcher reads `eval-results.json`, not the in-memory
+// aggregation. These tests pin that the `notVerified` status actually survives
+// serialization: per-case `testCases[].status` and top-level `summary.notVerified`.
+
+const scenario: ExecutionScenario = {
+	name: 'happy-path',
+	description: 'baseline',
+	dataSetup: 'plain',
+	successCriteria: 'works',
+};
+
+function scenarioCase(): WorkflowTestCase {
+	return {
+		conversation: [{ role: 'user', text: 'build it' }],
+		complexity: 'simple',
+		tags: [],
+		executionScenarios: [scenario],
+		datasets: ['full'],
+	};
+}
+
+function runResult(
+	testCase: WorkflowTestCase,
+	run: { success: boolean; incomplete?: boolean },
+): WorkflowTestCaseResult {
+	return {
+		testCase,
+		workflowBuildSuccess: true,
+		executionScenarioResults: [
+			{
+				scenario,
+				success: run.success,
+				score: run.success ? 1 : 0,
+				reasoning: run.success ? 'ok' : 'nope',
+				...(run.incomplete ? { incomplete: true } : {}),
+			},
+		],
+	};
+}
+
+interface SerializedResults {
+	summary: { notVerified: number };
+	testCases: Array<{ status: string; testCaseFile?: string }>;
+}
+
+function writeAndRead(): SerializedResults {
+	const notVerifiedCase = scenarioCase();
+	const verifiedCase = scenarioCase();
+
+	// Two iterations. Case 0: every run incomplete → notVerified.
+	// Case 1: one pass, one fail → verified.
+	const evaluation = aggregateResults(
+		[
+			[
+				runResult(notVerifiedCase, { success: false, incomplete: true }),
+				runResult(verifiedCase, { success: true }),
+			],
+			[
+				runResult(notVerifiedCase, { success: false, incomplete: true }),
+				runResult(verifiedCase, { success: false }),
+			],
+		],
+		2,
+	);
+
+	const slugByTestCase = new Map<WorkflowTestCase, string>([
+		[notVerifiedCase, 'behavioral-not-verified'],
+		[verifiedCase, 'scenario-verified'],
+	]);
+
+	const dir = mkdtempSync(join(tmpdir(), 'eval-results-test-'));
+	const { jsonPath } = writeEvalResults(
+		evaluation,
+		1234,
+		dir,
+		'exp-test',
+		undefined,
+		undefined,
+		slugByTestCase,
+		undefined,
+		undefined,
+	);
+
+	return jsonParse<SerializedResults>(readFileSync(jsonPath, 'utf8'));
+}
+
+describe('writeEvalResults — notVerified serialization', () => {
+	it('serializes per-case status and the top-level notVerified count', () => {
+		const report = writeAndRead();
+
+		const byFile = new Map(report.testCases.map((tc) => [tc.testCaseFile, tc.status]));
+		expect(byFile.get('behavioral-not-verified')).toBe('notVerified');
+		expect(byFile.get('scenario-verified')).toBe('verified');
+		expect(report.summary.notVerified).toBe(1);
+	});
+
+	// The committed fixture is the cross-repo contract anchor consumed by the
+	// lang-tracer dispatcher test. Guard it so the pinned fields can't silently drift.
+	it('matches the committed golden fixture on the pinned fields', () => {
+		const golden = jsonParse<SerializedResults>(
+			readFileSync(join(__dirname, 'fixtures', 'eval-results.not-verified.json'), 'utf8'),
+		);
+
+		const live = writeAndRead();
+		expect(golden.summary.notVerified).toBe(live.summary.notVerified);
+		expect(golden.testCases.map((tc) => [tc.testCaseFile, tc.status])).toEqual(
+			live.testCases.map((tc) => [tc.testCaseFile, tc.status]),
+		);
+	});
+});

packages/@n8n/instance-ai/evaluations/cli/aggregator.ts

@@ -6,6 +6,7 @@ import type {
 	ExecutionScenarioAggregation,
 	BuildExpectationAggregation,
 	BuildExpectationResult,
+	CaseVerificationStatus,
 } from '../types';
 
 /**
@@ -85,6 +86,23 @@ function aggregateUnit<T extends { pass: boolean; incomplete?: boolean }>(
 	};
 }
 
+/**
+ * A case is `notVerified` when nothing could be scored: every scenario and every
+ * build expectation came back with 0 evaluated runs (all incomplete, or skipped —
+ * e.g. process expectations with no transcript). A build FAILURE is NOT a gap: the
+ * aggregator substitutes a non-incomplete "scenario not executed" result, so a
+ * build-failed case has evaluated (failing) scenarios and stays `verified`.
+ */
+function computeCaseStatus(
+	scenarios: ExecutionScenarioAggregation[],
+	expectations: BuildExpectationAggregation[],
+): CaseVerificationStatus {
+	const evaluatedUnits =
+		scenarios.reduce((n, sa) => n + sa.evaluatedCount, 0) +
+		expectations.reduce((n, ea) => n + ea.evaluatedCount, 0);
+	return evaluatedUnits > 0 ? 'verified' : 'notVerified';
+}
+
 export function aggregateResults(
 	allRunResults: WorkflowTestCaseResult[][],
 	totalRuns: number,
@@ -147,6 +165,7 @@ export function aggregateResults(
 			buildSuccessCount,
 			executionScenarios,
 			buildExpectations,
+			status: computeCaseStatus(executionScenarios, buildExpectations),
 		});
 	}
 

packages/@n8n/instance-ai/evaluations/cli/index.ts

@@ -95,6 +95,7 @@ import { seedMcpRegistry } from '../mcp-registry/seeder';
 import { snapshotWorkflowIds } from '../outcome/workflow-discovery';
 import { writeRunDebugReport } from '../report/run-debug-report';
 import { writeWorkflowReport } from '../report/workflow-report';
+import { rollupCaseVerification } from '../summary';
 import type {
 	BuildExpectationResult,
 	MultiRunEvaluation,
@@ -1880,6 +1881,7 @@ export function writeEvalResults(
 ): { jsonPath: string; prCommentPath: string } {
 	const { totalRuns, testCases } = evaluation;
 	const metrics = computeAggregateMetrics(evaluation);
+	const verification = rollupCaseVerification(testCases);
 
 	const result = outcome?.kind === 'ok' ? outcome.result : undefined;
 
@@ -1901,6 +1903,9 @@ export function writeEvalResults(
 			passAtK: metrics.passAtK,
 			passHatK: metrics.passHatK,
 			passRatePerIter: metrics.passRatePerIter,
+			// Cases where nothing could be scored (all units incomplete / skipped) —
+			// reported apart from the pass rate, never as a silent pass.
+			notVerified: verification.notVerified,
 			...(checksSummary ? { workflowChecks: checksSummary } : {}),
 			...(buildSpendSummary ? { mcpBuild: buildSpendSummary } : {}),
 		},
@@ -1922,6 +1927,9 @@ export function writeEvalResults(
 		testCases: testCases.map((tc) => ({
 			name: caseDisplayPrompt(tc.testCase, tc.runs[0]?.transcript).slice(0, 70),
 			testCaseFile: slugByTestCase?.get(tc.testCase),
+			// `notVerified` when no scenario or expectation was scored across runs —
+			// consumers must not treat a zero pass rate here as a pass.
+			status: tc.status,
 			buildSuccessCount: tc.buildSuccessCount,
 			totalRuns,
 			workflowChecksPerRun: tc.runs.map((run) =>

packages/@n8n/instance-ai/evaluations/summary.ts

@@ -73,6 +73,32 @@ export function getCaseRunStatus(result: WorkflowTestCaseResult): CaseRunStatus
 	return 'build_failed';
 }
 
+/** Per-case verification tallies across an aggregated run. `notVerified` cases —
+ *  where nothing could be scored — are counted apart from passed/failed so the
+ *  reported pass rate reflects only what was actually checked. */
+export interface CaseVerificationRollup {
+	/** Verified cases where every evaluated unit passed. */
+	passed: number;
+	/** Verified cases with at least one failing evaluated unit. */
+	failed: number;
+	/** Cases where no unit could be scored (all incomplete / skipped). */
+	notVerified: number;
+}
+
+export function rollupCaseVerification(cases: TestCaseAggregation[]): CaseVerificationRollup {
+	const rollup: CaseVerificationRollup = { passed: 0, failed: 0, notVerified: 0 };
+	for (const tc of cases) {
+		if (tc.status === 'notVerified') {
+			rollup.notVerified++;
+			continue;
+		}
+		const { passCount, totalCount } = countAggregatedUnitTrials(getAggregatedCaseUnits(tc));
+		if (totalCount > 0 && passCount === totalCount) rollup.passed++;
+		else rollup.failed++;
+	}
+	return rollup;
+}
+
 export function getCaseRunStatusLabel(result: WorkflowTestCaseResult): string {
 	switch (getCaseRunStatus(result)) {
 		case 'checked':

packages/@n8n/instance-ai/evaluations/types.ts

@@ -426,6 +426,15 @@ export interface BuildExpectationAggregation {
 	passHatK: number[];
 }
 
+/**
+ * Whether a case produced any scoreable verdict across its runs.
+ * - `verified`  — at least one scenario or build expectation was evaluated.
+ * - `notVerified` — every measured unit came back incomplete / was skipped (no
+ *   transcript for process expectations, verifier gaps for scenarios), so nothing
+ *   could actually be checked. Such a case MUST NOT roll up as a silent pass.
+ */
+export type CaseVerificationStatus = 'verified' | 'notVerified';
+
 export interface TestCaseAggregation {
 	testCase: WorkflowTestCase;
 	runs: WorkflowTestCaseResult[];
@@ -433,6 +442,9 @@ export interface TestCaseAggregation {
 	executionScenarios: ExecutionScenarioAggregation[];
 	/** Build expectations aggregated as measured units (counted in the pass rate). */
 	buildExpectations: BuildExpectationAggregation[];
+	/** `notVerified` when no unit (scenario or expectation) was evaluated across
+	 *  all runs — nothing could be checked, so the case is not a pass. */
+	status: CaseVerificationStatus;
 }
 
 export interface MultiRunEvaluation {