fix(ai-builder): Fix AI-node eval mock-execution timeouts (no-changelog) (#33323)

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-08-29 01:39:24 +08:00 · 2026-07-01 09:33:51 +01:00
parent a5ac4259b8
commit 2ad33a1303
12 changed files with 665 additions and 20 deletions
@@ -29,6 +29,7 @@ All Instance AI configuration is done via environment variables.
 | Variable | Type | Default | Description |
 |----------|------|---------|-------------|
 | `N8N_INSTANCE_AI_RUN_DEBUG_ENABLED` | boolean | `false` | Capture orchestrator LLM steps and workflow code snapshots for the dev debug panel and eval LLM debug reports. |
+| `N8N_INSTANCE_AI_EVAL_TIMING` | boolean | `false` | When `true`, logs a per-execution `[EvalMock][timing]` phase breakdown (hints / bypass-pin / http-mock / ai-turn) for the eval mock-execution path, to attribute mocked-execution latency. A no-op otherwise. |

 ### Memory

@@ -16,8 +16,14 @@ import type {
 	InstanceAiEvalSeedDataTable,
 	InstanceAiEvalSeedWorkflow,
 } from '@n8n/api-types';
+import { Agent, setGlobalDispatcher } from 'undici';
 import { z } from 'zod';

+// Disable undici's 300s timeouts — mocked eval runs take minutes; the per-request
+// AbortSignal is the real bound. This is process-global: only ever imported by the
+// eval CLI harness — never import into the n8n server or shared runtime code.
+setGlobalDispatcher(new Agent({ headersTimeout: 0, bodyTimeout: 0 }));
+
 // -- Conversation seeding response shapes -------------------------------------

 const RestoreThreadEnvelope = z.object({
@@ -28,14 +28,14 @@
 		{
 			"name": "happy-path-multi-candle-array",
 			"description": "Binance klines HTTP Request returns a multi-row candle array; normalization handles every row and a non-HOLD signal reaches Telegram.",
-			"dataSetup": "Assume Binance, Telegram, and any AI-model credentials are present (mocked). The HTTP Request to GET https://api.binance.com/api/v3/klines returns Binance's real shape: a JSON array of candle rows, each row an array like [1700000000000, '42000.00', '42500.00', '41800.00', '42300.00', '120.5', ...], totaling 250 rows with a clear upward close progression toward the end. The Telegram sendMessage call returns { ok: true, result: { message_id: 9001 } }.",
-			"successCriteria": "Judge primarily on the Normalize OHLCV Code node's output: it must consume all 250 candle rows from the HTTP Request array (e.g. via $input.all() or iterating the response rows), NOT just the first row. Its output must contain a candle series of ~250 rows with non-null numeric open/high/low/close/volume and a non-null current price (not NaN/null). If the workflow runs to completion, the Telegram sendMessage message text additionally contains a concrete signal value (e.g. 'BUY', 'SELL', or 'HOLD') derived from the computed indicators; treat a missing or null signal as a failure."
+			"dataSetup": "Assume Binance, Telegram, and any AI-model credentials are present (mocked). The HTTP Request to GET https://api.binance.com/api/v3/klines returns Binance's real shape: a small JSON array of candle rows, each row an array like [1700000000000, '42000.00', '42500.00', '41800.00', '42300.00', '120.5', ...], totaling exactly 15 rows (no more) with a clear upward close progression toward the end. The Telegram sendMessage call returns { ok: true, result: { message_id: 9001 } }.",
+			"successCriteria": "Judge primarily on the Normalize OHLCV Code node's output: it must consume all 15 candle rows from the HTTP Request array (e.g. via $input.all() or iterating the response rows), NOT just the first row. Its output must contain a candle series of ~15 rows with non-null numeric open/high/low/close/volume and a non-null current price (not NaN/null). If the workflow runs to completion, the Telegram sendMessage message text additionally contains a concrete signal value (e.g. 'BUY', 'SELL', or 'HOLD') derived from the computed indicators; treat a missing or null signal as a failure."
 		},
 		{
 			"name": "array-shape-hidden-by-single-item-mock",
 			"description": "LOAD-BEARING: the array-vs-single bug must surface under a real multi-row response even though a single-item fixture would have passed verification.",
-			"dataSetup": "Assume credentials are mocked. The HTTP Request to the Binance klines endpoint returns the REAL array shape: a JSON array of 250 candle rows (each row an array of OHLCV fields). Critically do NOT replay a single-item fixture — inject the full multi-row array so any $input.first()/single-item assumption is exercised. Telegram sendMessage returns { ok: true, result: { message_id: 9002 } }.",
-			"successCriteria": "Judge on the Normalize OHLCV node's output when the 250-row Binance array is fed in: its candle series must have ~250 rows with populated (non-null) OHLCV values and a non-null current price — it must NOT collapse to a single row, a handful of rows, or NaN/null values, which is what happens when the node maps over $input.first().json instead of $input.all(). A collapsed or null series is a failure even if the workflow otherwise completes. If indicators are computed and reach Telegram, the message must reflect the multi-row data rather than a perpetual HOLD forced by null inputs."
+			"dataSetup": "Assume credentials are mocked. The HTTP Request to the Binance klines endpoint returns the REAL array shape: a small JSON array of exactly 15 candle rows (each row an array of OHLCV fields). Critically do NOT replay a single-item fixture — inject the full 15-row array so any $input.first()/single-item assumption is exercised. Telegram sendMessage returns { ok: true, result: { message_id: 9002 } }.",
+			"successCriteria": "Judge on the Normalize OHLCV node's output when the 15-row Binance array is fed in: its candle series must have ~15 rows with populated (non-null) OHLCV values and a non-null current price — it must NOT collapse to a single row or NaN/null values, which is what happens when the node maps over $input.first().json instead of $input.all(). A collapsed or null series is a failure even if the workflow otherwise completes. If indicators are computed and reach Telegram, the message must reflect the multi-row data rather than a perpetual HOLD forced by null inputs."
 		},
 		{
 			"name": "empty-klines-response",
@@ -78,6 +78,7 @@
    "psl": "1.9.0",
    "source-map-support": "catalog:",
    "turndown": "catalog:",
+    "undici": "catalog:undici-v7",
    "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.2/xlsx-0.20.2.tgz",
    "zod": "catalog:",
    "zod-from-json-schema-v3": "npm:zod-from-json-schema@^0.0.5"
@@ -0,0 +1,80 @@
+import { EvalTimings } from '../eval-timings';
+
+function makeLogger() {
+	return { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() };
+}
+
+describe('EvalTimings', () => {
+	const original = process.env.N8N_INSTANCE_AI_EVAL_TIMING;
+
+	afterEach(() => {
+		if (original === undefined) delete process.env.N8N_INSTANCE_AI_EVAL_TIMING;
+		else process.env.N8N_INSTANCE_AI_EVAL_TIMING = original;
+	});
+
+	describe('when disabled', () => {
+		beforeEach(() => {
+			delete process.env.N8N_INSTANCE_AI_EVAL_TIMING;
+		});
+
+		it('runs the fn, returns its value, and records nothing', async () => {
+			const timings = new EvalTimings();
+			const logger = makeLogger();
+
+			await expect(timings.time('hints', undefined, async () => 42)).resolves.toBe(42);
+
+			timings.summary(logger as never);
+			expect(logger.info).not.toHaveBeenCalled();
+		});
+	});
+
+	describe('when enabled', () => {
+		beforeEach(() => {
+			process.env.N8N_INSTANCE_AI_EVAL_TIMING = 'true';
+		});
+
+		it('returns the wrapped fn value', async () => {
+			const timings = new EvalTimings();
+			await expect(timings.time('hints', undefined, async () => 'result')).resolves.toBe('result');
+		});
+
+		it('records a sample even when the wrapped fn throws', async () => {
+			const timings = new EvalTimings();
+			const logger = makeLogger();
+
+			await expect(
+				timings.time('http-mock', 'Node A', async () => {
+					throw new Error('boom');
+				}),
+			).rejects.toThrow('boom');
+
+			timings.summary(logger as never);
+			const lines = logger.info.mock.calls.map((c) => String(c[0]));
+			expect(lines.some((l) => l.includes('phase=http-mock') && l.includes('calls=1'))).toBe(true);
+		});
+
+		it('aggregates per-phase counts and a grand total', async () => {
+			const timings = new EvalTimings();
+			const logger = makeLogger();
+
+			await timings.time('hints', undefined, async () => undefined);
+			await timings.time('http-mock', 'a', async () => undefined);
+			await timings.time('http-mock', 'b', async () => undefined);
+
+			timings.summary(logger as never);
+			const lines = logger.info.mock.calls.map((c) => String(c[0]));
+			expect(lines.some((l) => l.includes('phase=hints') && l.includes('calls=1'))).toBe(true);
+			expect(lines.some((l) => l.includes('phase=http-mock') && l.includes('calls=2'))).toBe(true);
+			expect(lines.some((l) => l.includes('SUMMARY') && l.includes('llmCalls=3'))).toBe(true);
+		});
+
+		it('logs only the summary line when nothing was timed', () => {
+			const timings = new EvalTimings();
+			const logger = makeLogger();
+
+			timings.summary(logger as never);
+			expect(logger.info).toHaveBeenCalledTimes(1);
+			expect(String(logger.info.mock.calls[0][0])).toContain('SUMMARY');
+		});
+	});
+});
@@ -0,0 +1,197 @@
+interface CompletionStep {
+	kind?: 'tool_call' | 'final';
+	toolName?: string;
+	toolArguments?: Record<string, unknown>;
+	content?: string;
+}
+
+// Hoisted so the `vi.mock('@n8n/instance-ai')` factory below can resolve them.
+const { submitQueue, submitCapture, promptCapture, mockGenerate, mockAgent, mockExtractText } =
+	vi.hoisted(() => {
+		const submitQueue: CompletionStep[] = [];
+		const submitCapture: { handler?: (input: CompletionStep) => Promise<unknown> } = {};
+		const promptCapture: { prompt?: string } = {};
+
+		const mockGenerate = vi.fn(async (prompt: string) => {
+			promptCapture.prompt = prompt;
+			const next = submitQueue.shift();
+			if (next && submitCapture.handler) await submitCapture.handler(next);
+			return { messages: [], _text: '' };
+		});
+
+		const mockAgent = {
+			tool: vi.fn(function (this: unknown, builtTool: { _name?: string; _handler?: unknown }) {
+				if (builtTool._name === 'submit_agent_step') {
+					submitCapture.handler = builtTool._handler as (input: CompletionStep) => Promise<unknown>;
+				}
+				return this;
+			}),
+			generate: mockGenerate,
+		};
+
+		const mockExtractText = vi.fn((result: { _text?: string }) => result._text ?? '');
+
+		return { submitQueue, submitCapture, promptCapture, mockGenerate, mockAgent, mockExtractText };
+	});
+
+const mockLogger = { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() };
+
+function toolBuilderMock(name: string) {
+	const built: { _name: string; _handler?: unknown } = { _name: name };
+	return {
+		description: vi.fn().mockReturnThis(),
+		input: vi.fn().mockReturnThis(),
+		handler: vi.fn(function (this: unknown, h: unknown) {
+			built._handler = h;
+			return this;
+		}),
+		build: vi.fn(() => built),
+	};
+}
+
+vi.mock('@n8n/instance-ai', () => ({
+	createEvalAgent: vi.fn(() => mockAgent),
+	extractText: mockExtractText,
+	Tool: vi.fn().mockImplementation(toolBuilderMock),
+}));
+
+vi.mock('@n8n/di', () => ({
+	Container: { get: vi.fn(() => mockLogger) },
+	Service: () => (target: unknown) => target,
+}));
+
+import { Container } from '@n8n/di';
+import { createEvalAgent, Tool } from '@n8n/instance-ai';
+import type { IHttpRequestOptions, INode } from 'n8n-workflow';
+
+import { createLlmCompletionMockHandler } from '../llm-completion-mock';
+
+// `restoreMocks: true` wipes factory `.mockImplementation`s before each test.
+function reapplyMockImplementations() {
+	vi.mocked(Container.get).mockReturnValue(mockLogger as never);
+	vi.mocked(createEvalAgent).mockReturnValue(mockAgent as never);
+	vi.mocked(Tool).mockImplementation(toolBuilderMock as never);
+	mockAgent.tool.mockImplementation(function (
+		this: unknown,
+		builtTool: { _name?: string; _handler?: unknown },
+	) {
+		if (builtTool._name === 'submit_agent_step') {
+			submitCapture.handler = builtTool._handler as (input: CompletionStep) => Promise<unknown>;
+		}
+		return this;
+	});
+	mockGenerate.mockImplementation(async (prompt: string) => {
+		promptCapture.prompt = prompt;
+		const next = submitQueue.shift();
+		if (next && submitCapture.handler) await submitCapture.handler(next);
+		return { messages: [], _text: '' };
+	});
+	mockExtractText.mockImplementation((result: { _text?: string }) => result._text ?? '');
+}
+
+const node = {
+	name: 'My Agent',
+	type: '@n8n/n8n-nodes-langchain.lmChatOpenAi',
+} as unknown as INode;
+
+function chatRequest(body: unknown): IHttpRequestOptions {
+	return {
+		url: 'http://127.0.0.1/eval/My%20Agent/v1/chat/completions',
+		method: 'POST',
+		body,
+	} as unknown as IHttpRequestOptions;
+}
+
+beforeEach(() => {
+	submitQueue.length = 0;
+	submitCapture.handler = undefined;
+	promptCapture.prompt = undefined;
+	reapplyMockImplementations();
+});
+
+describe('createLlmCompletionMockHandler', () => {
+	it('emits a tool_calls shorthand when the model calls a tool', async () => {
+		submitQueue.push({ kind: 'tool_call', toolName: 'search_web', toolArguments: { q: 'x' } });
+		const handler = createLlmCompletionMockHandler();
+
+		const res = await handler(
+			chatRequest({
+				messages: [{ role: 'user', content: 'hi' }],
+				tools: [{ type: 'function', function: { name: 'search_web', parameters: {} } }],
+			}),
+			node,
+		);
+
+		expect(res?.statusCode).toBe(200);
+		expect(res?.body).toEqual({ tool_calls: [{ name: 'search_web', arguments: { q: 'x' } }] });
+	});
+
+	it('lets a named tool win even when the step is labeled "final"', async () => {
+		submitQueue.push({
+			kind: 'final',
+			toolName: 'format_response',
+			toolArguments: { text: 'x' },
+			content: 'ignored',
+		});
+		const handler = createLlmCompletionMockHandler();
+
+		const res = await handler(
+			chatRequest({
+				messages: [{ role: 'user', content: 'hi' }],
+				tools: [{ type: 'function', function: { name: 'format_response', parameters: {} } }],
+			}),
+			node,
+		);
+
+		expect(res?.body).toEqual({
+			tool_calls: [{ name: 'format_response', arguments: { text: 'x' } }],
+		});
+	});
+
+	it('emits content for a final answer', async () => {
+		submitQueue.push({ kind: 'final', content: 'done' });
+		const handler = createLlmCompletionMockHandler();
+
+		const res = await handler(chatRequest({ messages: [{ role: 'user', content: 'hi' }] }), node);
+
+		expect(res?.body).toEqual({ content: 'done' });
+	});
+
+	it('falls back to raw text (and warns) when the model never submits a step', async () => {
+		mockGenerate.mockImplementation(async (prompt: string) => {
+			promptCapture.prompt = prompt;
+			return { messages: [], _text: 'raw fallback text' };
+		});
+		const handler = createLlmCompletionMockHandler();
+
+		const res = await handler(chatRequest({ messages: [{ role: 'user', content: 'hi' }] }), node);
+
+		expect(res?.body).toEqual({ content: 'raw fallback text' });
+		expect(mockLogger.warn).toHaveBeenCalled();
+	});
+
+	it('feeds the available tools and conversation (incl. tool-result state) into the prompt', async () => {
+		submitQueue.push({ kind: 'final', content: 'ok' });
+		const handler = createLlmCompletionMockHandler();
+
+		await handler(
+			chatRequest({
+				messages: [
+					{ role: 'user', content: 'analyze this' },
+					{ role: 'tool', content: 'result data' },
+				],
+				tools: [
+					{
+						type: 'function',
+						function: { name: 'fetch_data', description: 'gets data', parameters: {} },
+					},
+				],
+			}),
+			node,
+		);
+
+		expect(promptCapture.prompt).toContain('fetch_data');
+		expect(promptCapture.prompt).toContain('analyze this');
+		expect(promptCapture.prompt).toContain('Tool results ARE present');
+	});
+});
@@ -0,0 +1,74 @@
+import type { Logger } from '@n8n/backend-common';
+
+/** LLM-call phases of a mocked eval execution. */
+type EvalTimingPhase = 'hints' | 'bypass-pin' | 'http-mock' | 'ai-turn';
+
+const PHASE_ORDER: EvalTimingPhase[] = ['hints', 'bypass-pin', 'http-mock', 'ai-turn'];
+
+interface LlmCallSample {
+	phase: EvalTimingPhase;
+	durationMs: number;
+	label?: string;
+}
+
+/**
+ * Per-execution LLM-call timing accumulator, gated by
+ * `N8N_INSTANCE_AI_EVAL_TIMING=true` (a no-op otherwise).
+ */
+export class EvalTimings {
+	private readonly samples: LlmCallSample[] = [];
+
+	private readonly startedAt = Date.now();
+
+	readonly enabled = process.env.N8N_INSTANCE_AI_EVAL_TIMING === 'true';
+
+	/** Time one mock/hint LLM call and record a sample. Always awaits `fn`. */
+	async time<T>(
+		phase: EvalTimingPhase,
+		label: string | undefined,
+		fn: () => Promise<T>,
+	): Promise<T> {
+		if (!this.enabled) return await fn();
+		const start = Date.now();
+		try {
+			return await fn();
+		} finally {
+			this.samples.push({ phase, label, durationMs: Date.now() - start });
+		}
+	}
+
+	/** Log a per-phase + grand-total breakdown. No-op when disabled. */
+	summary(logger: Logger): void {
+		if (!this.enabled) return;
+
+		for (const phase of PHASE_ORDER) {
+			const durations = this.samples.filter((s) => s.phase === phase).map((s) => s.durationMs);
+			if (durations.length === 0) continue;
+			const total = durations.reduce((sum, d) => sum + d, 0);
+			logger.info(
+				`[EvalMock][timing] phase=${phase} calls=${durations.length} total=${fmt(total)} p50=${fmt(percentile(durations, 50))} max=${fmt(Math.max(...durations))}`,
+			);
+		}
+
+		const llmTotal = this.samples.reduce((sum, s) => sum + s.durationMs, 0);
+		// Display-only; mirrors getModelId() in eval-agents.ts (SONNET_MODEL default).
+		const model =
+			process.env.N8N_INSTANCE_AI_EVAL_MODEL ??
+			process.env.N8N_INSTANCE_AI_MODEL ??
+			'anthropic/claude-sonnet-4-6';
+		logger.info(
+			`[EvalMock][timing] SUMMARY wall=${fmt(Date.now() - this.startedAt)} llmCalls=${this.samples.length} llmTotal=${fmt(llmTotal)} model=${model}`,
+		);
+	}
+}
+
+function fmt(ms: number): string {
+	return ms >= 1000 ? `${(ms / 1000).toFixed(1)}s` : `${ms}ms`;
+}
+
+function percentile(values: number[], p: number): number {
+	if (values.length === 0) return 0;
+	const sorted = [...values].sort((a, b) => a - b);
+	const index = Math.min(sorted.length - 1, Math.floor((p / 100) * sorted.length));
+	return sorted[index];
+}
@@ -45,7 +45,9 @@ import { PostHogClient } from '@/posthog';
 import { WorkflowRunner } from '@/workflow-runner';
 import { WorkflowFinderService } from '@/workflows/workflow-finder.service';

+import { createLlmCompletionMockHandler } from './llm-completion-mock';
 import { EvalMockedCredentialsHelper } from './eval-mocked-credentials-helper';
+import { EvalTimings } from './eval-timings';
 import { type InterceptedTurn, LlmWireServer } from './llm-wire-server';
 import { createLlmMockHandler } from './mock-handler';
 import { generatePinData } from './pin-data-generator';
@@ -156,7 +158,13 @@ export class EvalExecutionService {
 		}

 		const unpinSet = unpinNodes.length > 0 ? new Set(unpinNodes) : undefined;
-		const hints = await this.analyzeWorkflow(workflowEntity, options.scenarioHints, unpinSet);
+		const timings = new EvalTimings();
+		const hints = await this.analyzeWorkflow(
+			workflowEntity,
+			timings,
+			options.scenarioHints,
+			unpinSet,
+		);
 		const vendorLlmRouting = interceptionEnabled
 			? buildVendorLlmRouting(workflowEntity, unpinNodes)
 			: undefined;
@@ -165,6 +173,7 @@ export class EvalExecutionService {
 			workflowEntity,
 			user,
 			hints,
+			timings,
 			options.scenarioHints,
 			interceptionEnabled,
 			vendorLlmRouting,
@@ -188,6 +197,7 @@ export class EvalExecutionService {

 	private async analyzeWorkflow(
 		workflowEntity: IWorkflowBase,
+		timings: EvalTimings,
 		scenarioHints?: string,
 		unpinSet?: Set<string>,
 	): Promise<MockHints> {
@@ -199,11 +209,16 @@ export class EvalExecutionService {
 			`[EvalMock] Generating hints for ${nodeNames.length} nodes: ${nodeNames.join(', ')}`,
 		);

-		const hints = await generateMockHints({
-			workflow: workflowEntity,
-			nodeNames,
-			scenarioHints,
-		});
+		const hints = await timings.time(
+			'hints',
+			undefined,
+			async () =>
+				await generateMockHints({
+					workflow: workflowEntity,
+					nodeNames,
+					scenarioHints,
+				}),
+		);

 		if (!hints.globalContext && nodeNames.length > 0) {
 			this.logger.warn(
@@ -227,6 +242,7 @@ export class EvalExecutionService {
 				workflowEntity,
 				bypassNodeNames,
 				hints.globalContext,
+				timings,
 				scenarioHints,
 			);
 			this.logger.debug(
@@ -248,17 +264,23 @@ export class EvalExecutionService {
 		workflowEntity: IWorkflowBase,
 		bypassNodeNames: string[],
 		globalContext: string,
+		timings: EvalTimings,
 		scenarioHints?: string,
 	): Promise<IPinData> {
 		if (bypassNodeNames.length === 0) return {};

 		try {
 			const dataDescription = [globalContext, scenarioHints].filter(Boolean).join('\n\n');
-			const result = await generatePinData({
-				workflow: workflowEntity as unknown as WorkflowJSON,
-				nodeNames: bypassNodeNames,
-				instructions: dataDescription ? { dataDescription } : undefined,
-			});
+			const result = await timings.time(
+				'bypass-pin',
+				undefined,
+				async () =>
+					await generatePinData({
+						workflow: workflowEntity as unknown as WorkflowJSON,
+						nodeNames: bypassNodeNames,
+						instructions: dataDescription ? { dataDescription } : undefined,
+					}),
+			);

 			return normalizePinData(result as unknown as IPinData);
 		} catch (error) {
@@ -278,6 +300,7 @@ export class EvalExecutionService {
 		workflowEntity: IWorkflowBase,
 		user: User,
 		hints: MockHints,
+		timings: EvalTimings,
 		scenarioHints?: string,
 		interceptionEnabled = false,
 		vendorLlmRouting?: VendorLlmRouting,
@@ -328,8 +351,20 @@ export class EvalExecutionService {
 		try {
 			let serverUrl: string | undefined;
 			if (interceptionEnabled) {
+				// Wire server mocks the agent's own model turns (not HTTP APIs) → LLM completion mock.
+				const llmCompletionMockHandler = createLlmCompletionMockHandler({
+					scenarioHints,
+					globalContext: hints.globalContext,
+					nodeHints: hints.nodeHints,
+				});
+				const timedAiTurnHandler: EvalLlmMockHandler = async (request, node) =>
+					await timings.time(
+						'ai-turn',
+						node.type,
+						async () => await llmCompletionMockHandler(request, node),
+					);
 				wireServer = new LlmWireServer({
-					mockHandler,
+					mockHandler: timedAiTurnHandler,
 					rootToSubNode: vendorLlmRouting?.rootToSubNode,
 					onIntercept: (turn) => this.recordWireServerTurn(turn, nodeResults),
 					logger: this.logger,
@@ -356,6 +391,7 @@ export class EvalExecutionService {
 					additionalData.evalLlmMockHandler = this.createInterceptingHandler(
 						mockHandler,
 						nodeResults,
+						timings,
 					);
 				},
 			};
@@ -399,6 +435,7 @@ export class EvalExecutionService {
 					});
 				}
 			}
+			timings.summary(this.logger);
 		}
 	}

@@ -618,6 +655,7 @@ export class EvalExecutionService {
 	private createInterceptingHandler(
 		mockHandler: EvalLlmMockHandler,
 		nodeResults: Record<string, InstanceAiEvalNodeResult>,
+		timings: EvalTimings,
 	): EvalLlmMockHandler {
 		return async (
 			requestOptions: IHttpRequestOptions,
@@ -633,7 +671,11 @@ export class EvalExecutionService {
 				executionMode: 'mocked',
 			});
 			entry.executionMode = 'mocked';
-			const response = await mockHandler(requestOptions, node);
+			const response = await timings.time(
+				'http-mock',
+				node.name,
+				async () => await mockHandler(requestOptions, node),
+			);

 			entry.interceptedRequests.push({
 				url: requestOptions.url,
@@ -0,0 +1,241 @@
+/**
+ * Completion mock for the eval wire server: simulates ONE turn of an AI agent's
+ * model, returning the adapter shorthand (`{ tool_calls }` or `{ content }`).
+ * The HTTP-API mock is wrong here — its "return the full resource" prompt yields
+ * a provider envelope the adapter can't read a tool call from.
+ */
+
+import { Logger } from '@n8n/backend-common';
+import { Container } from '@n8n/di';
+import { createEvalAgent, extractText, Tool } from '@n8n/instance-ai';
+import type { EvalLlmMockHandler, EvalMockHttpResponse } from 'n8n-core';
+import { z } from 'zod';
+
+const COMPLETION_MOCK_PROMPT = `You simulate ONE response from the LLM that powers an AI agent inside an n8n workflow under evaluation. The agent runs a tool-calling loop and calls you once per turn. Decide the agent's NEXT step and submit it via submit_agent_step.
+
+This is a MOCK whose only purpose is to exercise the workflow's wiring and data flow — NOT to produce a realistic deliverable. Keep everything MINIMAL and schema-valid. Never write long documents, full HTML pages, or essays; a short stub string that satisfies the schema is ideal.
+
+You are given the conversation so far, whether any tool results have come back, and the tools the agent may call (each with its JSON input schema).
+
+How to decide:
+- If NO tool results are present yet and the agent's instructions describe gathering/processing steps, advance the loop: pick the SINGLE most appropriate NON-final tool and call it. Do not jump to the final answer on the first turn.
+- Once tool results are present (or only a "final answer" / "format response" style tool remains sensible), finalize: if such a structured-output tool exists, call it; otherwise return a short final answer.
+
+Rules:
+- kind="tool_call": set toolName to one of the available tools and toolArguments to an object matching THAT tool's input schema EXACTLY (same keys, nesting, and types). Use the SMALLEST valid value for every field — a one-line string, a single-element array — never a long document.
+- kind="final": set content to a short final answer string.
+- Call exactly ONE tool per step. Never fabricate a whole multi-step result in a single turn.
+- If the scenario, node hint, or data context states a specific value, reproduce it; otherwise keep values minimal.`;
+
+interface ParsedTool {
+	name: string;
+	description?: string;
+	schema: unknown;
+}
+
+interface CompletionStep {
+	kind?: 'tool_call' | 'final';
+	toolName?: string;
+	toolArguments?: Record<string, unknown>;
+	content?: string;
+}
+
+export interface LlmCompletionMockOptions {
+	scenarioHints?: string;
+	globalContext?: string;
+	nodeHints?: Record<string, string>;
+}
+
+const submitStepSchema = z.object({
+	kind: z
+		.enum(['tool_call', 'final'])
+		.describe('"tool_call" to advance the agent loop; "final" to give the agent\'s final answer.'),
+	toolName: z
+		.string()
+		.optional()
+		.describe('Required for kind="tool_call": the exact name of the tool to call.'),
+	toolArguments: z
+		.record(z.unknown())
+		.optional()
+		.describe(
+			'Required for kind="tool_call": an object matching the named tool\'s input schema exactly. Use minimal valid values.',
+		),
+	content: z
+		.string()
+		.optional()
+		.describe('Required for kind="final": a short final answer string.'),
+});
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+	return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+
+function asString(value: unknown): string | undefined {
+	return typeof value === 'string' && value.length > 0 ? value : undefined;
+}
+
+const TRANSCRIPT_ITEM_MAX = 500;
+
+function truncate(text: string): string {
+	return text.length > TRANSCRIPT_ITEM_MAX ? `${text.slice(0, TRANSCRIPT_ITEM_MAX)}…` : text;
+}
+
+/** Flatten chat/responses message content (string or content-part array) to text. */
+function contentToString(content: unknown): string {
+	if (typeof content === 'string') return content;
+	if (Array.isArray(content)) {
+		const parts: string[] = [];
+		for (const part of content) {
+			if (typeof part === 'string') parts.push(part);
+			else if (isRecord(part) && typeof part.text === 'string') parts.push(part.text);
+		}
+		return parts.join(' ');
+	}
+	if (content === undefined || content === null) return '';
+	return JSON.stringify(content);
+}
+
+/** Tools live under `tools[].function` (chat-completions) or flat on the item (responses). */
+function extractTools(body: unknown): ParsedTool[] {
+	if (!isRecord(body) || !Array.isArray(body.tools)) return [];
+	const out: ParsedTool[] = [];
+	for (const entry of body.tools) {
+		if (!isRecord(entry)) continue;
+		const fn = isRecord(entry.function) ? entry.function : entry;
+		const name = asString(fn.name);
+		if (!name) continue;
+		out.push({ name, description: asString(fn.description), schema: fn.parameters });
+	}
+	return out;
+}
+
+interface ConversationSummary {
+	transcript: string;
+	hasToolResults: boolean;
+}
+
+/** Summarize the conversation and detect whether any tool has already returned. */
+function summarizeConversation(body: unknown): ConversationSummary {
+	if (!isRecord(body)) return { transcript: '', hasToolResults: false };
+	const items = Array.isArray(body.messages)
+		? body.messages
+		: Array.isArray(body.input)
+			? body.input
+			: [];
+
+	const lines: string[] = [];
+	let hasToolResults = false;
+	for (const item of items) {
+		if (!isRecord(item)) continue;
+		const type = asString(item.type);
+		const role = asString(item.role);
+
+		if (type === 'function_call_output' || role === 'tool') {
+			hasToolResults = true;
+			const payload = type === 'function_call_output' ? item.output : item.content;
+			lines.push(`tool_result: ${truncate(contentToString(payload))}`);
+		} else if (type === 'function_call') {
+			lines.push(`assistant called tool: ${asString(item.name) ?? '?'}`);
+		} else if (role) {
+			lines.push(`${role}: ${truncate(contentToString(item.content))}`);
+		}
+	}
+	return { transcript: lines.join('\n'), hasToolResults };
+}
+
+function buildUserPrompt(
+	tools: ParsedTool[],
+	summary: ConversationSummary,
+	options: LlmCompletionMockOptions | undefined,
+	nodeHint: string | undefined,
+): string {
+	const toolList =
+		tools
+			.map(
+				(t) =>
+					`- ${t.name}${t.description ? `: ${t.description}` : ''}\n  input schema: ${JSON.stringify(t.schema)}`,
+			)
+			.join('\n') || '(no tools — return a final answer)';
+
+	const sections: string[] = [
+		'## Conversation so far',
+		summary.transcript || '(empty — this is the first turn)',
+		'',
+		'## Tools the agent may call',
+		toolList,
+		'',
+		'## State',
+		summary.hasToolResults
+			? 'Tool results ARE present — you likely have enough to finalize.'
+			: 'No tool results yet — advance the loop by calling a non-final tool unless none exist.',
+	];
+	if (options?.globalContext) sections.push('', '## Data context', options.globalContext);
+	if (nodeHint) sections.push('', '## Node hint', nodeHint);
+	if (options?.scenarioHints) sections.push('', '## Scenario', options.scenarioHints);
+	return sections.join('\n');
+}
+
+function jsonResponse(body: Record<string, unknown>): EvalMockHttpResponse {
+	return { body, headers: { 'content-type': 'application/json' }, statusCode: 200 };
+}
+
+/** LLM-completion mock handler for the wire server — same signature as the HTTP mock. */
+export function createLlmCompletionMockHandler(
+	options?: LlmCompletionMockOptions,
+): EvalLlmMockHandler {
+	return async (requestOptions, node) => {
+		const body = requestOptions.body;
+		const tools = extractTools(body);
+		const summary = summarizeConversation(body);
+		const userPrompt = buildUserPrompt(tools, summary, options, options?.nodeHints?.[node.name]);
+
+		const capture: CompletionStep = {};
+		const agent = createEvalAgent('eval-llm-completion-mock', {
+			instructions: COMPLETION_MOCK_PROMPT,
+			cache: true,
+		}).tool(
+			new Tool('submit_agent_step')
+				.description("Submit the agent's next step: one tool call, or a final answer.")
+				.input(submitStepSchema)
+				.handler(async (input: CompletionStep) => {
+					// A named tool always wins, even on a step labeled "final" — structured-output
+					// agents finalize by calling the tool. Capture it regardless of `kind`.
+					if (input.toolName) {
+						capture.kind = 'tool_call';
+						capture.toolName = input.toolName;
+						capture.toolArguments = input.toolArguments ?? {};
+						return 'Accepted.';
+					}
+					if (input.kind === 'tool_call') {
+						return 'Invalid: kind="tool_call" requires toolName. Call submit_agent_step again.';
+					}
+					capture.kind = 'final';
+					capture.content = input.content ?? '';
+					return 'Accepted.';
+				})
+				.build(),
+		);
+
+		const result = await agent.generate(userPrompt);
+
+		let responseBody: Record<string, unknown>;
+		if (capture.toolName) {
+			// A named tool always wins — structured-output agents finalize by calling
+			// the tool even while labeling the step "final".
+			responseBody = {
+				tool_calls: [{ name: capture.toolName, arguments: capture.toolArguments ?? {} }],
+			};
+		} else if (capture.kind === 'final') {
+			responseBody = { content: capture.content ?? '' };
+		} else {
+			// Agent never submitted — fall back to its raw text so the turn isn't empty.
+			const fallback = extractText(result).trim();
+			Container.get(Logger).warn(
+				`[EvalMock] llm-completion-mock produced no submit_agent_step for "${node.name}"; using raw text fallback`,
+			);
+			responseBody = { content: fallback || '[eval completion-mock: empty response]' };
+		}
+
+		return jsonResponse(responseBody);
+	};
+}
@@ -645,9 +645,8 @@ export async function generateMockHints(options: GenerateMockHintsOptions): Prom
 				instructions: SYSTEM_PROMPT,
 			});

-			const result = await agent.generate(userPrompt, {
-				providerOptions: { anthropic: { maxTokens: 4096 } },
-			});
+			// No maxTokens cap — a low ceiling truncates hints for large workflows.
+			const result = await agent.generate(userPrompt);

 			const text = extractText(result)
 				.replace(/^```(?:json)?\s*\n?/i, '')
@@ -695,6 +695,7 @@ export class InstanceAiController {

 	// ── Evaluation endpoints ──────────────────────────────────────────────────

+	// Runs for minutes; the eval client (N8nClient) disables undici's 300s timeout for it.
 	@Post('/eval/execute-with-llm-mock/:workflowId')
 	@GlobalScope('instanceAi:eval')
 	async executeWithLlmMock(
@@ -2395,6 +2395,9 @@ importers:
      turndown:
        specifier: 'catalog:'
        version: 7.2.2
+      undici:
+        specifier: catalog:undici-v7
+        version: 7.28.0
      xlsx:
        specifier: https://cdn.sheetjs.com/xlsx-0.20.2/xlsx-0.20.2.tgz
        version: https://cdn.sheetjs.com/xlsx-0.20.2/xlsx-0.20.2.tgz