chore(ai-builder): Typecheck the eval harness and pin its external contracts (no-changelog) (#34673)

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

packages/@n8n/instance-ai/eslint.config.mjs

@@ -22,7 +22,16 @@ const restrictedLazyRuntimeImports = [
 export default defineConfig(
 	baseConfig,
 	{
-		ignores: ['scripts/**/*.cjs', 'skills/**/*.mjs'],
+		ignores: [
+			'scripts/**/*.cjs',
+			'skills/**/*.mjs',
+			// Local eval scratch output — never linted, never committed.
+			'.data/**',
+			'evaluations/.data/**',
+			// Deep-imports ai-workflow-builder.ee's evaluations source, so it sits outside
+			// evaluations/tsconfig.json (see its exclude) and the eslint project service.
+			'evaluations/cli/pairwise.ts',
+		],
 	},
 	{
 		rules: {

packages/@n8n/instance-ai/evaluations/__tests__/artifact-config-eval-handler.test.ts

@@ -24,7 +24,14 @@ function googleSheetsConfig(workflowId: string): EvaluationConfigDto {
 				id: 'metric-2',
 				name: 'Correctness',
 				type: 'llm_judge',
-				config: { preset: 'correctness' },
+				config: {
+					preset: 'correctness',
+					provider: '@n8n/n8n-nodes-langchain.lmChatOpenAi',
+					credentialId: 'cred-judge',
+					model: 'gpt-4.1',
+					outputType: 'numeric',
+					inputs: { actualAnswer: '={{ $json.actual }}', expectedAnswer: '={{ $json.expected }}' },
+				},
 			},
 		],
 		datasetSource: 'google_sheets',

packages/@n8n/instance-ai/evaluations/__tests__/artifact-workflow-handler.test.ts

@@ -54,7 +54,7 @@ describe('workflowHandler', () => {
 		const expectedIds = extractWorkflowIdsFromMessages(messages);
 		expect(expectedIds.length).toBeGreaterThan(0);
 
-		const refs = workflowHandler.discover({ messages });
+		const refs = workflowHandler.discover({ messages, artifactRefs: [] });
 		expect(refs.map((r) => r.id)).toEqual(expectedIds);
 		for (const ref of refs) {
 			expect(ref.type).toBe('workflow');

packages/@n8n/instance-ai/evaluations/__tests__/comparison-format.test.ts

@@ -67,6 +67,7 @@ function evaluation(
 				conversation: [{ role: 'user', text: tc.userText ?? 'Test workflow prompt' }],
 				complexity: 'medium' as const,
 				tags: [],
+				datasets: ['full'],
 				executionScenarios: (tc.scenarios ?? []).map((sa) => ({
 					name: sa.name,
 					description: '',
@@ -76,7 +77,7 @@ function evaluation(
 			} as WorkflowTestCase;
 			const buildSuccessCount = tc.buildSuccessCount ?? totalRuns;
 			const scenarios = (tc.scenarios ?? []).map((sa) => ({
-				scenario: testCase.executionScenarios.find((sc) => sc.name === sa.name)!,
+				scenario: testCase.executionScenarios!.find((sc) => sc.name === sa.name)!,
 				evaluatedCount: sa.passes.length,
 				passCount: sa.passCount,
 				passRate: totalRuns > 0 ? sa.passCount / totalRuns : 0,
@@ -84,7 +85,7 @@ function evaluation(
 				passHatK: new Array(totalRuns).fill(sa.passCount === totalRuns ? 1 : 0) as number[],
 				runs: sa.passes.map(
 					(passed): ExecutionScenarioResult => ({
-						scenario: testCase.executionScenarios.find((sc) => sc.name === sa.name)!,
+						scenario: testCase.executionScenarios!.find((sc) => sc.name === sa.name)!,
 						success: passed,
 						score: passed ? 1 : 0,
 						reasoning: sa.reasoning ?? '',

packages/@n8n/instance-ai/evaluations/__tests__/comparison-gate.test.ts

@@ -42,7 +42,7 @@ function makeEval(totalRuns: number, cases: CaseSpec[]) {
 		const scenarioAggs = (c.scenarios ?? []).map((sa) => {
 			const evaluated = sa.passes.filter((p) => p !== 'incomplete');
 			const passCount = evaluated.filter((p) => p).length;
-			const scenario = testCase.executionScenarios.find((x) => x.name === sa.name)!;
+			const scenario = testCase.executionScenarios!.find((x) => x.name === sa.name)!;
 			return {
 				scenario,
 				evaluatedCount: evaluated.length,

packages/@n8n/instance-ai/evaluations/__tests__/conversation-seed.test.ts

@@ -42,6 +42,7 @@ function makeSeed(): ConversationSeed {
 			},
 		],
 		workflows: [{ id: WF_ID, name: 'Daily digest', nodes: [], connections: {} }],
+		dataTables: [],
 	};
 }
 

packages/@n8n/instance-ai/evaluations/__tests__/data-discovery.test.ts

@@ -51,6 +51,7 @@ describe('loadDiscoveryTestCasesWithFiles', () => {
 				workflowIds: [],
 				executionIds: [],
 				dataTableIds: [],
+				artifactRefs: [],
 				finalText: '',
 				toolCalls: [],
 				agentActivities: [],

packages/@n8n/instance-ai/evaluations/__tests__/data-workflows-schema.test.ts

@@ -37,7 +37,7 @@ describe('EvalTestCaseSchema', () => {
 	it('accepts a minimal valid fixture', () => {
 		const parsed = EvalTestCaseSchema.parse(validFixture());
 		expect(parsed.executionScenarios).toHaveLength(1);
-		expect(parsed.conversation[0].role).toBe('user');
+		expect(parsed.conversation![0].role).toBe('user');
 	});
 
 	it('rejects an empty conversation', () => {
@@ -49,7 +49,7 @@ describe('EvalTestCaseSchema', () => {
 			...validFixture(),
 			conversation: [{ role: 'user', text: ['line 1', 'line 2'] }],
 		});
-		expect(parsed.conversation[0].text).toBe('line 1\nline 2');
+		expect(parsed.conversation![0].text).toBe('line 1\nline 2');
 	});
 
 	it('rejects 0 execution scenarios AND 0 expectations (a case must assert something)', () => {
@@ -251,7 +251,7 @@ describe('EvalTestCaseSchema', () => {
 			requires: 'mock-server',
 		} as (typeof fixture.executionScenarios)[number];
 		const parsed = EvalTestCaseSchema.parse(fixture);
-		expect(parsed.executionScenarios[0].requires).toBe('mock-server');
+		expect(parsed.executionScenarios![0].requires).toBe('mock-server');
 	});
 });
 

packages/@n8n/instance-ai/evaluations/__tests__/data-workflows.test.ts

@@ -244,7 +244,7 @@ describe('loadWorkflowTestCasesWithFiles', () => {
 			});
 			mockedReadFile.mockImplementation((p) => {
 				const filename = String(p);
-				const parsed = jsonParse(STUB_TEST_CASE);
+				const parsed = jsonParse<Record<string, unknown>>(STUB_TEST_CASE);
 				if (filename.includes('agent-intent')) {
 					return JSON.stringify({
 						...parsed,

packages/@n8n/instance-ai/evaluations/__tests__/eval-results-dispatcher-contract.test.ts

@@ -0,0 +1,167 @@
+import type { InstanceAiEvalExecutionResult } from '@n8n/api-types';
+import { mkdtempSync, readFileSync } from 'fs';
+import { jsonParse } from 'n8n-workflow';
+import { tmpdir } from 'os';
+import { join } from 'path';
+
+import type { CheckOutcome } from '../binaryChecks/types';
+import { aggregateResults } from '../cli/aggregator';
+import { writeEvalResults } from '../cli/index';
+import type { ExecutionScenario, WorkflowTestCase, WorkflowTestCaseResult } from '../types';
+
+// Pins the `eval-results.json` fields the lang-tracer dispatcher ingests
+// (lang-tracer-dispatcher `src/lib/runner.ts`): it spawns this CLI per case in
+// direct (no-LangSmith) mode, reads the file, and projects these fields into
+// LangTracer run state. Renaming or dropping any of them breaks LangTracer
+// ingestion silently — the dispatcher tolerates absent fields by design.
+
+const scenario: ExecutionScenario = {
+	name: 'happy-path',
+	description: 'baseline',
+	dataSetup: 'plain',
+	successCriteria: 'digest arrives',
+};
+
+const testCase: WorkflowTestCase = {
+	conversation: [{ role: 'user', text: 'send me a daily digest' }],
+	complexity: 'simple',
+	tags: [],
+	datasets: ['full'],
+	executionScenarios: [scenario],
+	outcomeExpectations: ['sends a digest'],
+};
+
+const passingCheck: CheckOutcome = {
+	name: 'no-unreachable-nodes',
+	description: 'all nodes reachable',
+	kind: 'deterministic',
+	dimension: 'structure',
+	status: 'pass',
+};
+
+function iteration1(): WorkflowTestCaseResult {
+	return {
+		testCase,
+		workflowBuildSuccess: true,
+		workflowChecks: [passingCheck],
+		buildExpectationResults: [
+			{ expectation: 'sends a digest', pass: true, reason: 'digest node present' },
+		],
+		executionScenarioResults: [{ scenario, success: true, score: 1, reasoning: 'works' }],
+	};
+}
+
+function iteration2(): WorkflowTestCaseResult {
+	return {
+		testCase,
+		workflowBuildSuccess: true,
+		buildExpectationResults: [
+			{ expectation: 'sends a digest', pass: false, reason: 'digest node missing' },
+		],
+		executionScenarioResults: [
+			{
+				scenario,
+				success: false,
+				score: 0,
+				reasoning: 'no digest was produced',
+				failureCategory: 'mock_issue',
+				rootCause: 'mock returned an empty page',
+				evalResult: { errors: ['HTTP 500 from the mocked API'] } as InstanceAiEvalExecutionResult,
+			},
+		],
+	};
+}
+
+interface DispatcherView {
+	experimentName?: string;
+	testCases: Array<{
+		buildSuccessCount: number;
+		totalRuns: number;
+		workflowChecksPerRun: Array<Record<string, string> | null>;
+		buildExpectations: Array<{
+			expectation: string;
+			passCount: number;
+			evaluatedCount: number;
+		}>;
+		buildExpectationResultsPerRun: Array<Array<{
+			expectation: string;
+			pass: boolean;
+			reason: string;
+		}> | null>;
+		scenarios: Array<{
+			name: string;
+			passCount: number;
+			totalRuns: number;
+			runs: Array<{
+				passed: boolean;
+				score: number;
+				reasoning: string;
+				failureCategory?: string;
+				rootCause?: string;
+				execErrors: string[];
+			}>;
+		}>;
+	}>;
+}
+
+function writeAndRead(): DispatcherView {
+	const evaluation = aggregateResults([[iteration1()], [iteration2()]], 2);
+	const dir = mkdtempSync(join(tmpdir(), 'eval-results-contract-'));
+	const { jsonPath } = writeEvalResults(
+		evaluation,
+		1234,
+		dir,
+		'exp-dispatcher-contract',
+		undefined,
+		undefined,
+		new Map([[testCase, 'daily-digest']]),
+		undefined,
+		undefined,
+	);
+	return jsonParse<DispatcherView>(readFileSync(jsonPath, 'utf8'));
+}
+
+describe('eval-results.json — dispatcher contract', () => {
+	it('serializes every field the dispatcher projects into run state', () => {
+		const report = writeAndRead();
+
+		expect(report.experimentName).toBe('exp-dispatcher-contract');
+		expect(report.testCases).toHaveLength(1);
+
+		const tc = report.testCases[0];
+		expect(tc.buildSuccessCount).toBe(2);
+		expect(tc.totalRuns).toBe(2);
+
+		// Per-iteration build signals. Checks serialize as a name→status map (an
+		// iteration without checks serializes as null, not as a hole).
+		expect(tc.workflowChecksPerRun).toEqual([{ 'no-unreachable-nodes': 'pass' }, null]);
+		expect(tc.buildExpectations).toHaveLength(1);
+		expect(tc.buildExpectations[0]).toMatchObject({
+			expectation: 'sends a digest',
+			passCount: 1,
+			evaluatedCount: 2,
+		});
+		expect(tc.buildExpectationResultsPerRun).toEqual([
+			[{ expectation: 'sends a digest', pass: true, reason: 'digest node present' }],
+			[{ expectation: 'sends a digest', pass: false, reason: 'digest node missing' }],
+		]);
+
+		// Scenario blocks serialize under the flat `scenarios` key with a flat
+		// `name` — the shape the dispatcher's fallback reader consumes today.
+		expect(tc.scenarios).toHaveLength(1);
+		const sc = tc.scenarios[0];
+		expect(sc.name).toBe('happy-path');
+		expect(sc.passCount).toBe(1);
+		expect(sc.totalRuns).toBe(2);
+		expect(sc.runs).toHaveLength(2);
+		expect(sc.runs[0]).toMatchObject({ passed: true, score: 1, reasoning: 'works' });
+		expect(sc.runs[1]).toMatchObject({
+			passed: false,
+			score: 0,
+			reasoning: 'no digest was produced',
+			failureCategory: 'mock_issue',
+			rootCause: 'mock returned an empty page',
+			execErrors: ['HTTP 500 from the mocked API'],
+		});
+	});
+});

packages/@n8n/instance-ai/evaluations/__tests__/fixtures.ts

@@ -56,7 +56,19 @@ export function dataTableConfig(workflowId: string, dataTableId: string): Evalua
 		startNodeName: 'Start',
 		endNodeName: 'End',
 		metrics: [
-			{ id: 'metric-1', name: 'Correctness', type: 'llm_judge', config: { preset: 'correctness' } },
+			{
+				id: 'metric-1',
+				name: 'Correctness',
+				type: 'llm_judge',
+				config: {
+					preset: 'correctness',
+					provider: '@n8n/n8n-nodes-langchain.lmChatOpenAi',
+					credentialId: 'cred-judge',
+					model: 'gpt-4.1',
+					outputType: 'numeric',
+					inputs: { actualAnswer: '={{ $json.actual }}', expectedAnswer: '={{ $json.expected }}' },
+				},
+			},
 		],
 		datasetSource: 'data_table',
 		datasetRef: { dataTableId },

packages/@n8n/instance-ai/evaluations/__tests__/partial-results-abort.test.ts

@@ -45,6 +45,7 @@ function makeTestCase(scenarios: ExecutionScenario[]): WorkflowTestCase {
 		conversation: [{ role: 'user', text: 'build me something' }],
 		complexity: 'complex',
 		tags: ['test'],
+		datasets: ['full'],
 		executionScenarios: scenarios,
 	};
 }

packages/@n8n/instance-ai/evaluations/__tests__/runner-prebuilt.test.ts

@@ -53,6 +53,7 @@ function makeTestCase(): WorkflowTestCase {
 		conversation: [{ role: 'user', text: 'build me something' }],
 		complexity: 'simple',
 		tags: ['test'],
+		datasets: ['full'],
 		executionScenarios: [],
 	};
 }

packages/@n8n/instance-ai/evaluations/__tests__/subagent-runner.test.ts

@@ -34,7 +34,7 @@ function makeClient(): N8nClient {
 describe('runWorkflowBuildEval', () => {
 	beforeEach(() => {
 		vi.clearAllMocks();
-		mockedCleanupBuild.mockResolvedValue(undefined);
+		mockedCleanupBuild.mockResolvedValue(true);
 		mockedRunBinaryChecks.mockResolvedValue({
 			feedback: [
 				{

packages/@n8n/instance-ai/evaluations/__tests__/summary.test.ts

@@ -136,6 +136,7 @@ describe('summary — scenario-less cases (no workflow)', () => {
 				buildSuccessCount: 0,
 				executionScenarios: [],
 				buildExpectations: [],
+				status: 'verified',
 			};
 			// Only the first run judged an expectation.
 			expect(getCheckedRunCount(tc)).toBe(1);

packages/@n8n/instance-ai/evaluations/__tests__/verification-artifact.test.ts

@@ -602,6 +602,7 @@ describe('buildVerificationArtifact', () => {
 				interceptedRequests: Array.from({ length: 30 }, (_, i) => ({
 					method: 'GET',
 					url: `https://api.example.com/page/${i}`,
+					nodeType: 'n8n-nodes-base.httpRequest',
 					mockResponse: { page: i },
 				})),
 				iterationCount: 1,

packages/@n8n/instance-ai/evaluations/__tests__/workflow-checks-applicability.test.ts

@@ -6,6 +6,7 @@ function workflowWebhookOnly(): WorkflowResponse {
 		id: 'wf-1',
 		name: 'Webhook only',
 		active: false,
+		versionId: 'test-version',
 		nodes: [
 			{ name: 'Webhook', type: 'n8n-nodes-base.webhook', parameters: {} },
 			{

packages/@n8n/instance-ai/evaluations/__tests__/workflow-discovery.test.ts

@@ -9,6 +9,7 @@ const emptyEventOutcome: EventOutcome = {
 	workflowIds: [],
 	executionIds: [],
 	dataTableIds: [],
+	artifactRefs: [],
 	finalText: '',
 	toolCalls: [],
 	agentActivities: [],

packages/@n8n/instance-ai/evaluations/__tests__/write-partial-results.test.ts

@@ -67,6 +67,7 @@ function makeTestCase(scenarios: ExecutionScenario[]): WorkflowTestCase {
 		conversation: [{ role: 'user', text: 'build me something' }],
 		complexity: 'complex',
 		tags: ['test'],
+		datasets: ['full'],
 		executionScenarios: scenarios,
 	};
 }

packages/@n8n/instance-ai/evaluations/binaryChecks/checks/code-node-no-http-requests.test.ts

@@ -6,6 +6,7 @@ function workflowWithCodeNode(parameters: Record<string, unknown>): WorkflowResp
 		id: 'wf-1',
 		name: 'Code HTTP test',
 		active: false,
+		versionId: 'test-version',
 		nodes: [
 			{
 				name: 'Manual Trigger',
@@ -152,6 +153,7 @@ describe('codeNodeNoHttpRequests', () => {
 			id: 'wf-2',
 			name: 'No code',
 			active: false,
+			versionId: 'test-version',
 			nodes: [
 				{
 					name: 'Manual Trigger',

packages/@n8n/instance-ai/evaluations/binaryChecks/checks/error-routes-consistent.test.ts

@@ -10,6 +10,7 @@ function reminderWorkflow(overrides: {
 		id: 'wf-1',
 		name: 'Reminder sender',
 		active: false,
+		versionId: 'test-version',
 		nodes: [
 			{ name: 'Daily', type: 'n8n-nodes-base.scheduleTrigger', parameters: {} },
 			{
@@ -73,6 +74,7 @@ describe('errorRoutesConsistent', () => {
 			id: 'wf-2',
 			name: 'Branching',
 			active: false,
+			versionId: 'test-version',
 			nodes: [
 				{ name: 'Start', type: 'n8n-nodes-base.manualTrigger', parameters: {} },
 				{ name: 'IF', type: 'n8n-nodes-base.if', parameters: {} },
@@ -97,6 +99,7 @@ describe('errorRoutesConsistent', () => {
 			id: 'wf-3',
 			name: 'Empty',
 			active: false,
+			versionId: 'test-version',
 			nodes: [{ name: 'Start', type: 'n8n-nodes-base.manualTrigger', parameters: {} }],
 			connections: {},
 		};

packages/@n8n/instance-ai/evaluations/binaryChecks/checks/google-sheets-rlc-default-mode.test.ts

@@ -23,8 +23,8 @@ function sheetWorkflow(documentId: unknown, sheetName?: unknown): WorkflowRespon
 }
 
 describe('googleSheetsRlcDefaultMode', () => {
-	it('fails when documentId defaults to an empty By ID field (INS-631 shape)', () => {
-		const result = googleSheetsRlcDefaultMode.run(
+	it('fails when documentId defaults to an empty By ID field (INS-631 shape)', async () => {
+		const result = await googleSheetsRlcDefaultMode.run(
 			sheetWorkflow({
 				__rl: true,
 				mode: 'id',
@@ -38,8 +38,8 @@ describe('googleSheetsRlcDefaultMode', () => {
 		expect(result.comment).toContain('Append Row.documentId');
 	});
 
-	it('fails when documentId is By ID mode with only a placeholder value', () => {
-		const result = googleSheetsRlcDefaultMode.run(
+	it('fails when documentId is By ID mode with only a placeholder value', async () => {
+		const result = await googleSheetsRlcDefaultMode.run(
 			sheetWorkflow({
 				__rl: true,
 				mode: 'id',
@@ -51,8 +51,8 @@ describe('googleSheetsRlcDefaultMode', () => {
 		expect(result.pass).toBe(false);
 	});
 
-	it('fails when documentId is By ID mode with a fabricated ID absent from the prompt (real-build shape)', () => {
-		const result = googleSheetsRlcDefaultMode.run(
+	it('fails when documentId is By ID mode with a fabricated ID absent from the prompt (real-build shape)', async () => {
+		const result = await googleSheetsRlcDefaultMode.run(
 			sheetWorkflow({
 				__rl: true,
 				mode: 'id',
@@ -66,8 +66,8 @@ describe('googleSheetsRlcDefaultMode', () => {
 		expect(result.comment).toContain('Append Row.documentId');
 	});
 
-	it('passes when documentId uses the From list picker mode', () => {
-		const result = googleSheetsRlcDefaultMode.run(
+	it('passes when documentId uses the From list picker mode', async () => {
+		const result = await googleSheetsRlcDefaultMode.run(
 			sheetWorkflow(
 				{ __rl: true, mode: 'list', value: '', cachedResultName: 'SmartAssist Bookings' },
 				{ __rl: true, mode: 'list', value: 0, cachedResultName: 'Sheet1' },
@@ -78,8 +78,8 @@ describe('googleSheetsRlcDefaultMode', () => {
 		expect(result.pass).toBe(true);
 	});
 
-	it('passes when the user supplied a concrete spreadsheet ID', () => {
-		const result = googleSheetsRlcDefaultMode.run(
+	it('passes when the user supplied a concrete spreadsheet ID', async () => {
+		const result = await googleSheetsRlcDefaultMode.run(
 			sheetWorkflow({
 				__rl: true,
 				mode: 'id',
@@ -91,8 +91,8 @@ describe('googleSheetsRlcDefaultMode', () => {
 		expect(result.pass).toBe(true);
 	});
 
-	it('is not applicable when there are no Google Sheets nodes', () => {
-		const result = googleSheetsRlcDefaultMode.run(
+	it('is not applicable when there are no Google Sheets nodes', async () => {
+		const result = await googleSheetsRlcDefaultMode.run(
 			{
 				id: 'wf',
 				name: 'x',

packages/@n8n/instance-ai/evaluations/binaryChecks/checks/http-generic-auth-type-matches-prompt.test.ts

@@ -6,6 +6,7 @@ function workflowWithHttpRequest(parameters: Record<string, unknown>): WorkflowR
 		id: 'wf-1',
 		name: 'HTTP auth test',
 		active: false,
+		versionId: 'test-version',
 		nodes: [
 			{
 				name: 'Manual Trigger',
@@ -112,6 +113,7 @@ describe('httpGenericAuthTypeMatchesPrompt', () => {
 			id: 'wf-2',
 			name: 'Slack only',
 			active: false,
+			versionId: 'test-version',
 			nodes: [
 				{
 					name: 'Manual Trigger',

packages/@n8n/instance-ai/evaluations/binaryChecks/checks/memory-session-key-expression.test.ts

@@ -6,6 +6,7 @@ function createWorkflow(memoryParameters: Record<string, unknown>): WorkflowResp
 		id: 'workflow-1',
 		name: 'Memory expression test',
 		active: false,
+		versionId: 'test-version',
 		nodes: [
 			{
 				name: 'Telegram Trigger',

packages/@n8n/instance-ai/evaluations/cli/langtracer-push.ts

@@ -171,9 +171,12 @@ async function main() {
 		throw new Error(`suite "${args.suite}" not found. Available: ${known || '(none)'}.`);
 	}
 
-	// Select disk cases: loader applies --filter/--exclude/--tier; then narrow to the
-	// exact slugs from positional args + --changed (if either was given).
-	const all = loadWorkflowTestCasesWithFiles(args.filter, args.exclude, args.tier);
+	// Select disk cases: loader applies --filter/--exclude, --tier narrows by the
+	// case's datasets (mirrors data/source.ts); then narrow to the exact slugs
+	// from positional args + --changed (if either was given).
+	const loaded = loadWorkflowTestCasesWithFiles(args.filter, args.exclude);
+	const tier = args.tier;
+	const all = tier ? loaded.filter((c) => c.testCase.datasets.includes(tier)) : loaded;
 	const exactSlugs = new Set([...args.slugs, ...(args.changed ? gitChangedSlugs() : [])]);
 	const selected = exactSlugs.size > 0 ? all.filter((c) => exactSlugs.has(c.fileSlug)) : all;
 

packages/@n8n/instance-ai/evaluations/clients/n8n-client.ts

@@ -71,9 +71,11 @@ export type GatewayStatus = z.infer<typeof GatewayStatusSchema>;
 
 /** A node as returned by the n8n REST API — the fields eval code reads. */
 export interface WorkflowNodeResponse {
+	id?: string;
 	name: string;
 	type: string;
 	typeVersion?: number;
+	position?: [number, number];
 	parameters?: Record<string, unknown>;
 	executeOnce?: boolean;
 	onError?: 'stopWorkflow' | 'continueRegularOutput' | 'continueErrorOutput';

packages/@n8n/instance-ai/evaluations/computer-use/__tests__/graders-llm.test.ts

@@ -45,7 +45,13 @@ function trace(overrides: Partial<ScenarioTrace> = {}): ScenarioTrace {
 		confirmations: [],
 		finalText: '',
 		durationMs: 0,
-		tokens: { totalInput: 0, totalOutput: 0, byTool: [] },
+		tokens: {
+			perCall: [],
+			totalArgsEst: 0,
+			totalResultsEst: 0,
+			largestResultEst: 0,
+			estimated: true,
+		},
 		threadId: 'thread-1',
 		...overrides,
 	};
@@ -62,7 +68,7 @@ function call(toolName: string, args: Record<string, unknown> = {}): CapturedToo
 
 const grader: LlmTaskCompletedGrader = { type: 'llm.taskCompleted' };
 const userPrompt = 'Set up a Slack OAuth credential';
-const category: ScenarioCategory = 'credential-setup';
+const category: ScenarioCategory = 'browser';
 
 // ---------------------------------------------------------------------------
 // Setup
@@ -143,7 +149,7 @@ describe('llm.taskCompleted', () => {
 			);
 
 			const sent = captureUserMessage();
-			expect(sent).toContain('credential-setup');
+			expect(sent).toContain('browser');
 			expect(sent).toContain('Set up a Slack OAuth credential');
 			expect(sent).toContain('Final response from the agent.');
 		});

packages/@n8n/instance-ai/evaluations/computer-use/graders/index.ts

@@ -55,7 +55,7 @@ export async function applyGrader(grader: Grader, ctx: GradeContext): Promise<Gr
 		case 'security.noSecretLeak':
 			return gradeNoSecretLeak(ctx.trace, grader);
 		case 'llm.taskCompleted': {
-			const { gradeTaskCompleted } = await import('./llm');
+			const { gradeTaskCompleted } = await import('./llm.js');
 			return await gradeTaskCompleted(ctx.trace, ctx.userPrompt, ctx.scenarioCategory, grader);
 		}
 	}

packages/@n8n/instance-ai/evaluations/computer-use/types.ts

@@ -11,6 +11,8 @@
 import type { CapturedEvent, CapturedToolCall } from '../types';
 import type { TokenStats } from './tokens';
 
+export type { CapturedToolCall };
+
 // ---------------------------------------------------------------------------
 // Scenario specification (JSON)
 // ---------------------------------------------------------------------------

packages/@n8n/instance-ai/evaluations/discovery/__tests__/expected-tools-invoked.test.ts

@@ -10,6 +10,7 @@ function makeOutcome(opts: {
 		workflowIds: [],
 		executionIds: [],
 		dataTableIds: [],
+		artifactRefs: [],
 		finalText: '',
 		toolCalls: (opts.toolCalls ?? []).map((tc, i) => ({
 			toolCallId: `call-${i}`,

packages/@n8n/instance-ai/evaluations/discovery/cli.ts

@@ -27,7 +27,8 @@ interface CliArgs {
 	trials: number;
 	passThreshold: number;
 	timeoutMs: number;
-	maxSteps: number;
+	/** Optional iteration cap; default uncapped — the wall-clock timeout bounds a run. */
+	maxSteps?: number;
 	modelId: string;
 	concurrency: number;
 	nodesJsonPath?: string;
@@ -85,7 +86,6 @@ function parseArgs(argv: string[]): CliArgs {
 		trials: 3,
 		passThreshold: 2 / 3,
 		timeoutMs: 60_000,
-		maxSteps: 5,
 		modelId: DEFAULT_MODEL,
 		concurrency: 3,
 		failOnZeroPass: false,

packages/@n8n/instance-ai/evaluations/discovery/runner.ts

@@ -10,11 +10,11 @@
 // What's tested: the orchestrator's first dispatch decision. Tools are NOT
 // stubbed — when the orchestrator loads a runtime skill or reaches for a
 // Computer Use browser tool, the tool-call event fires before any downstream
-// failure, so the discovery check still sees the dispatch intent. maxSteps caps
-// the loop so an erroring tool can't drive API spend.
+// failure, so the discovery check still sees the dispatch intent. The wall-clock
+// timeout bounds the loop so an erroring tool can't drive API spend; scenarios
+// or --max-steps can additionally opt into an iteration cap.
 // ---------------------------------------------------------------------------
 
-import { Memory } from '@n8n/agents';
 import type { InstanceAiEvent, TaskList } from '@n8n/api-types';
 import { nanoid } from 'nanoid';
 
@@ -76,7 +76,10 @@ export async function runDiscoveryScenario(
 	options: DiscoveryRunOptions,
 ): Promise<DiscoveryRunResult> {
 	const started = Date.now();
-	const maxSteps = options.scenario?.maxSteps ?? options.maxSteps ?? 5;
+	// Uncapped by default, matching live behavior: today's orchestrator legitimately
+	// explores past any small fixed cap (data-table-workflow needs >8 iterations), and
+	// the wall-clock timeout below bounds runaway runs. Scenarios/CLI opt in to a cap.
+	const maxSteps = options.scenario?.maxSteps ?? options.maxSteps;
 	const timeoutMs = options.timeoutMs ?? 60_000;
 	const nodesJsonPath = options.nodesJsonPath ?? defaultNodesJsonPath();
 
@@ -93,7 +96,6 @@ export async function runDiscoveryScenario(
 		const context = applyInstanceState(services.context, options.scenario);
 
 		const mcpManager = new McpClientManager();
-		const memory = new Memory().build();
 		const threadId = 'discovery-thread-' + nanoid(6);
 		const runId = 'discovery-run-' + nanoid(6);
 
@@ -128,7 +130,7 @@ export async function runDiscoveryScenario(
 			context,
 			orchestrationContext,
 			mcpManager,
-			memory,
+			// No memory: discovery measures stateless first-step tool dispatch.
 			memoryConfig: {},
 			// Eager tool loading — discovery measures dispatch given the full toolset,
 			// not whether the orchestrator can find a tool through search.
@@ -138,7 +140,7 @@ export async function runDiscoveryScenario(
 
 		const streamSource = normalizeStreamSource(
 			await agent.stream(options.scenario.userMessage, {
-				maxSteps,
+				maxIterations: maxSteps,
 				abortSignal: abortController.signal,
 				providerOptions: {
 					anthropic: { cacheControl: { type: 'ephemeral' as const } },

packages/@n8n/instance-ai/evaluations/harness/in-memory-event-bus.ts

@@ -29,7 +29,9 @@ export function createInMemoryEventBus(): InstanceAiEventBus {
 			};
 		},
 		getEventsAfter(threadId, afterId) {
-			return (storeByThread.get(threadId) ?? []).filter((event) => event.id > afterId);
+			return (storeByThread.get(threadId) ?? []).filter(
+				(event) => event.id !== undefined && event.id > afterId,
+			);
 		},
 		getEventsForRun(threadId, runId) {
 			return (storeByThread.get(threadId) ?? [])

packages/@n8n/instance-ai/evaluations/harness/langsmith-seed.ts

@@ -347,8 +347,10 @@ async function reconstructWithClient(
 		);
 	}
 
+	// `?? NaN` keeps the SDK's optional start_time behavior-identical: an absent
+	// value still yields NaN comparisons, never a valid epoch-0 date.
 	const byStartTime = (a: Run, b: Run) =>
-		new Date(a.start_time).getTime() - new Date(b.start_time).getTime();
+		new Date(a.start_time ?? NaN).getTime() - new Date(b.start_time ?? NaN).getTime();
 	const rootRuns = runs.filter((r) => r.run_type === 'chain' && !r.parent_run_id).sort(byStartTime);
 	// Real agent tool calls only — the compiled-workflow bookkeeping event is
 	// excluded BY NAME (it must never become a tool-call block in the rebuilt
@@ -386,7 +388,7 @@ async function reconstructWithClient(
 		);
 	}
 	const liveTurnRun = userTurns[liveIndex];
-	const boundaryMs = new Date(liveTurnRun.start_time).getTime();
+	const boundaryMs = new Date(liveTurnRun.start_time ?? NaN).getTime();
 	const liveTurn = userMessageOf(liveTurnRun)!;
 
 	const messages = buildSeedMessages(rootRuns, toolRuns, boundaryMs);
@@ -432,7 +434,7 @@ function buildSeedMessages(
 	const messages: Array<Record<string, unknown>> = [];
 
 	for (const root of rootRuns) {
-		if (new Date(root.start_time).getTime() >= boundaryMs) break;
+		if (new Date(root.start_time ?? NaN).getTime() >= boundaryMs) break;
 
 		const userText = userMessageOf(root);
 		if (userText) {
@@ -440,7 +442,7 @@ function buildSeedMessages(
 				id: `${root.id}-user`,
 				role: 'user',
 				type: 'llm',
-				createdAt: new Date(root.start_time).toISOString(),
+				createdAt: new Date(root.start_time ?? NaN).toISOString(),
 				content: [{ type: 'text', text: userText }],
 			});
 		}
@@ -478,7 +480,7 @@ function buildSeedMessages(
 				role: 'assistant',
 				type: 'llm',
 				// +1ms so the assistant reply orders after its user turn.
-				createdAt: new Date(new Date(root.start_time).getTime() + 1).toISOString(),
+				createdAt: new Date(new Date(root.start_time ?? NaN).getTime() + 1).toISOString(),
 				content,
 			});
 		}

packages/@n8n/instance-ai/evaluations/harness/normalize-workflow.ts

@@ -15,8 +15,11 @@
 
 import type { WorkflowJSON } from '@n8n/workflow-sdk';
 import { deepCopy } from 'n8n-workflow';
+import type { IWorkflowBase } from 'n8n-workflow';
 
-import type { SimpleWorkflow } from '../../../ai-workflow-builder.ee/src/types/workflow';
+/** Same computed type as ai-workflow-builder.ee's `SimpleWorkflow`, declared locally so
+ *  the eval typecheck program doesn't pull the builder package's source tree in. */
+export type SimpleWorkflow = Pick<IWorkflowBase, 'name' | 'nodes' | 'connections'>;
 
 type NodeRaw = WorkflowJSON['nodes'][number];
 

packages/@n8n/instance-ai/evaluations/harness/stub-services.ts

@@ -243,6 +243,18 @@ export async function createStubServices(
 				returned: { from: 0, to: 0 },
 			};
 		},
+		async getResolvedNodeParameters(_executionId: string, nodeName: string) {
+			return {
+				nodeName,
+				runIndex: 0,
+				itemIndex: 0,
+				parameters: null,
+				resolved: null,
+				failedExpressions: [],
+				emptyResolutions: [],
+				suppressed: 'parameter-values-disabled' as const,
+			};
+		},
 	};
 
 	const dataTableService: InstanceAiDataTableService = {
@@ -298,6 +310,7 @@ export async function createStubServices(
 
 	const context: InstanceAiContext = {
 		userId: options.userId ?? 'eval-user',
+		logger: { info() {}, warn() {}, error() {}, debug() {} },
 		workflowService,
 		executionService,
 		credentialService,

packages/@n8n/instance-ai/evaluations/langtracer/config.ts

@@ -10,10 +10,10 @@ export function resolveLangTracerConfig(env: NodeJS.ProcessEnv = process.env): L
 	const baseUrl = env.LANGTRACER_URL?.trim();
 	const apiKey = env.LANGTRACER_API_KEY?.trim();
 
-	const missing: string[] = [];
-	if (!baseUrl) missing.push('LANGTRACER_URL');
-	if (!apiKey) missing.push('LANGTRACER_API_KEY');
-	if (missing.length > 0) {
+	if (!baseUrl || !apiKey) {
+		const missing: string[] = [];
+		if (!baseUrl) missing.push('LANGTRACER_URL');
+		if (!apiKey) missing.push('LANGTRACER_API_KEY');
 		throw new Error(
 			`--source langtracer needs ${missing.join(' and ')} in the environment (set them in .env.local). ` +
 				'LANGTRACER_URL is the lang-tracer base URL; LANGTRACER_API_KEY is an MCP bearer key (lt_…), ' +

packages/@n8n/instance-ai/evaluations/tsconfig.json

@@ -1,14 +1,25 @@
 {
 	"extends": [
-		"@n8n/typescript-config/tsconfig.common.json",
-		"@n8n/typescript-config/tsconfig.backend.json"
+		"@n8n/typescript-config/tsconfig.common.go.json",
+		"@n8n/typescript-config/tsconfig.backend.go.json"
 	],
 	"compilerOptions": {
 		"target": "es2023",
 		"lib": ["es2023"],
 		"emitDecoratorMetadata": true,
 		"experimentalDecorators": true,
-		"types": ["node", "vitest/globals"]
+		"types": ["node", "vitest/globals"],
+		"noEmit": true,
+		// tsgo is fast enough to skip incremental state; avoids a stray tsbuildinfo in dist/.
+		"incremental": false,
+		// evaluations/ itself never uses `@/`; this maps it for the ../src files that
+		// enter this program through direct imports, mirroring the package root config.
+		"paths": {
+			"@/*": ["../src/*"]
+		}
 	},
-	"include": ["**/*.ts"]
+	"include": ["**/*.ts"],
+	// cli/pairwise.ts deep-imports ai-workflow-builder.ee's evaluations source (its own
+	// `@/` alias can't resolve here); it's a leaf entrypoint, still linted by eslint.
+	"exclude": ["dist/**", ".data/**", "computer-use/data/**", "cli/pairwise.ts"]
 }

packages/@n8n/instance-ai/evaluations/utils/user-proxy/agent.ts

@@ -1,5 +1,5 @@
-import { SYSTEM_PROMPT, TOOL_DESCRIPTIONS } from './prompts';
-import { decisionSchema, type Decision } from './tools';
+import { SYSTEM_PROMPT } from './prompts';
+import { decisionSchema, TOOL_DESCRIPTIONS, type Decision } from './tools';
 import { createEvalAgent } from '../../../src/utils/eval-agents';
 import type { EvalLogger } from '../../harness/logger';
 

packages/@n8n/instance-ai/package.json

@@ -3,7 +3,7 @@
   "version": "1.17.0",
   "scripts": {
     "clean": "rimraf dist .turbo",
-    "typecheck": "tsc --noEmit",
+    "typecheck": "tsc --noEmit && tsc -p evaluations/tsconfig.json",
     "build": "tsc -p ./tsconfig.build.json && tsc-alias -p tsconfig.build.json",
     "build:unchecked": "tsc -p ./tsconfig.build.json --noCheck && tsc-alias -p tsconfig.build.json",
     "format": "biome format --write src",