fix(ai-builder): Remove pairwise multi-gen evals and improve logs (no-changelog) (#24270)

Signed-off-by: Oleg Ivaniv <me@olegivaniv.com>

packages/@n8n/ai-workflow-builder.ee/evaluations/README.md

@@ -207,28 +207,37 @@ The runner flushes pending trace batches before returning, so traces/results rel
 
 ```typescript
 // Inside runLangsmith():
-// IMPORTANT: Do NOT wrap target with traceable() - see "LangSmith Tracing" section below
+
+// IMPORTANT: Create traceable wrapper ONCE outside the target function
+// to avoid context leaking in concurrent scenarios. Pass params explicitly.
+const traceableGenerateWorkflow = traceable(
+  async (args: { prompt: string; genFn: Function }) => {
+    return await args.genFn(args.prompt);
+  },
+  { name: 'workflow_generation', run_type: 'chain', client: lsClient }
+);
+
 const target = async (inputs) => {
-  // Wrap inner operations with traceable() for child traces
-  const traceableGenerate = traceable(
-    async () => await generateWorkflow(prompt),
-    { name: 'workflow_generation', run_type: 'chain', client: lsClient }
-  );
-  const workflow = await traceableGenerate();
+  const { prompt } = inputs;
+  // Call the pre-created wrapper with explicit params (no closures)
+  const workflow = await traceableGenerateWorkflow({
+    prompt,
+    genFn: generateWorkflow,
+  });
   const feedback = await evaluateWithPlugins(workflow, evaluators);
   return { workflow, prompt, feedback };  // Pre-computed!
 };
 
-	// LangSmith evaluator converts internal `{ evaluator, metric }` into `{ key, score, comment? }`:
-	const feedbackExtractor = (run) => run.outputs.feedback.map(toLangsmithEvaluationResult);
+// LangSmith evaluator converts internal `{ evaluator, metric }` into `{ key, score, comment? }`:
+const feedbackExtractor = (run) => run.outputs.feedback.map(toLangsmithEvaluationResult);
 ```
 
-## LangSmith Tracing (Critical)
+## LangSmith Tracing
 
-Do this in LangSmith mode:
-
-1. **Do not** wrap the top-level `target` with `traceable()` — `evaluate()` will wrap it and wire up the run correctly.
-2. **Do** wrap inner operations with `traceable()` for child traces, and pass `client: lsClient` so child runs attach to the same project/run tree.
+- **Do not** wrap the `target` function with `traceable()` — `evaluate()` handles that automatically
+- **Do** create `traceable` wrappers **once** outside the target function (not inside concurrent code)
+- **Do** pass all parameters explicitly to avoid closure-based context leaking
+- **Do** use `getTracingCallbacks()` to bridge traceable context to LangChain calls (pass callbacks to `agent.chat()` or chain's `invoke()`)
 
 ## Available Evaluators
 
@@ -260,11 +269,10 @@ const evaluator = createPairwiseEvaluator(llm, { numJudges: 3 });
 
 **Evaluator:** `pairwise`
 
-**Metrics (v1-compatible):**
-- Single-gen: `pairwise_primary`, `pairwise_diagnostic`, `pairwise_judges_passed`, `pairwise_total_passes`, `pairwise_total_violations`
-- Multi-gen: `pairwise_generation_correctness`, `pairwise_aggregated_diagnostic`, `pairwise_generations_passed`, `pairwise_total_judge_calls`
+**Metrics:**
+`pairwise_primary`, `pairwise_diagnostic`, `pairwise_judges_passed`, `pairwise_total_passes`, `pairwise_total_violations`
 
-Additional per-judge/per-generation details may also be emitted (e.g. `judge1`, `gen1.majorityPass`).
+Additional per-judge details may also be emitted (e.g. `judge1`, `judge2`).
 
 **Context required:** `{ dos?: string, donts?: string }`
 

packages/@n8n/ai-workflow-builder.ee/evaluations/__tests__/cli.test.ts

@@ -88,7 +88,6 @@ function createMockArgs(overrides: Record<string, unknown> = {}) {
 		dos: undefined,
 		donts: undefined,
 		numJudges: 3,
-		numGenerations: 1,
 		experimentName: undefined,
 		repetitions: 1,
 		concurrency: 4,
@@ -276,9 +275,9 @@ describe('CLI', () => {
 				// Verify numJudges was passed correctly
 				const callArgs = mockCreatePairwiseEvaluator.mock.calls[0] as [
 					unknown,
-					{ numJudges: number; numGenerations: number },
+					{ numJudges: number },
 				];
-				expect(callArgs[1]).toEqual({ numJudges: 5, numGenerations: 1 });
+				expect(callArgs[1]).toEqual({ numJudges: 5 });
 				expect(mockCreateProgrammaticEvaluator).toHaveBeenCalled();
 				expect(mockCreateLLMJudgeEvaluator).not.toHaveBeenCalled();
 			});
@@ -388,7 +387,7 @@ describe('CLI', () => {
 		});
 
 		describe('exit codes', () => {
-			it('should exit with 0 when pass rate >= 70%', async () => {
+			it('should always exit with 0 on successful completion (pass/fail is informational)', async () => {
 				mockRunEvaluation.mockResolvedValue(createMockSummary({ totalExamples: 10, passed: 7 }));
 
 				const { runV2Evaluation } = await import('../cli');
@@ -396,28 +395,20 @@ describe('CLI', () => {
 				await expect(runV2Evaluation()).rejects.toThrow('process.exit(0)');
 			});
 
-			it('should exit with 1 when pass rate < 70%', async () => {
+			it('should exit with 0 even when pass rate is low', async () => {
 				mockRunEvaluation.mockResolvedValue(createMockSummary({ totalExamples: 10, passed: 5 }));
 
 				const { runV2Evaluation } = await import('../cli');
 
-				await expect(runV2Evaluation()).rejects.toThrow('process.exit(1)');
-			});
-
-			it('should exit with 0 when pass rate is exactly 70%', async () => {
-				mockRunEvaluation.mockResolvedValue(createMockSummary({ totalExamples: 10, passed: 7 }));
-
-				const { runV2Evaluation } = await import('../cli');
-
 				await expect(runV2Evaluation()).rejects.toThrow('process.exit(0)');
 			});
 
-			it('should exit with 1 when no examples', async () => {
+			it('should exit with 0 even when no examples', async () => {
 				mockRunEvaluation.mockResolvedValue(createMockSummary({ totalExamples: 0, passed: 0 }));
 
 				const { runV2Evaluation } = await import('../cli');
 
-				await expect(runV2Evaluation()).rejects.toThrow('process.exit(1)');
+				await expect(runV2Evaluation()).rejects.toThrow('process.exit(0)');
 			});
 		});
 

packages/@n8n/ai-workflow-builder.ee/evaluations/__tests__/evaluators/pairwise.test.ts

@@ -7,7 +7,6 @@
 
 import type { BaseChatModel } from '@langchain/core/language_models/chat_models';
 import { mock } from 'jest-mock-extended';
-import pLimit from 'p-limit';
 
 import type { SimpleWorkflow } from '@/types/workflow';
 
@@ -288,253 +287,4 @@ describe('Pairwise Evaluator', () => {
 			);
 		});
 	});
-
-	describe('createPairwiseEvaluator() multi-gen', () => {
-		it('should use single-gen behavior when numGenerations is 1', async () => {
-			mockRunJudgePanel.mockResolvedValue(createMockPanelResult());
-
-			const { createPairwiseEvaluator } = await import('../../evaluators/pairwise');
-			const evaluator = createPairwiseEvaluator(mockLlm, { numGenerations: 1 });
-
-			const workflow = createMockWorkflow();
-			const feedback = await evaluator.evaluate(workflow, { prompt: 'Test prompt' });
-
-			// Should have single-gen feedback keys
-			expect(findFeedback(feedback, PAIRWISE_METRICS.PAIRWISE_PRIMARY)).toBeDefined();
-			expect(findFeedback(feedback, PAIRWISE_METRICS.PAIRWISE_DIAGNOSTIC)).toBeDefined();
-			// Should NOT have multi-gen keys
-			expect(
-				findFeedback(feedback, PAIRWISE_METRICS.PAIRWISE_GENERATION_CORRECTNESS),
-			).toBeUndefined();
-		});
-
-		it('should generate multiple workflows when numGenerations > 1', async () => {
-			mockRunJudgePanel.mockResolvedValue(createMockPanelResult());
-
-			const { createPairwiseEvaluator } = await import('../../evaluators/pairwise');
-			const evaluator = createPairwiseEvaluator(mockLlm, { numJudges: 3, numGenerations: 3 });
-
-			const workflow = createMockWorkflow();
-			const mockGenerateWorkflow = jest.fn().mockResolvedValue(createMockWorkflow());
-
-			await evaluator.evaluate(workflow, {
-				dos: 'Test',
-				donts: 'Test',
-				prompt: 'Create a workflow',
-				generateWorkflow: mockGenerateWorkflow,
-			});
-
-			// Should have called generateWorkflow 3 times
-			expect(mockGenerateWorkflow).toHaveBeenCalledTimes(3);
-			// Should have called runJudgePanel 3 times (once per generation)
-			expect(mockRunJudgePanel).toHaveBeenCalledTimes(3);
-		});
-
-		it('should limit generation concurrency when llmCallLimiter is provided', async () => {
-			mockRunJudgePanel.mockResolvedValue(createMockPanelResult());
-
-			const { createPairwiseEvaluator } = await import('../../evaluators/pairwise');
-			const evaluator = createPairwiseEvaluator(mockLlm, { numJudges: 3, numGenerations: 3 });
-
-			const workflow = createMockWorkflow();
-
-			let active = 0;
-			let maxActive = 0;
-			const mockGenerateWorkflow = jest.fn().mockImplementation(async () => {
-				active++;
-				maxActive = Math.max(maxActive, active);
-				await new Promise((r) => setTimeout(r, 25));
-				active--;
-				return createMockWorkflow();
-			});
-
-			await evaluator.evaluate(workflow, {
-				prompt: 'Create a workflow',
-				generateWorkflow: mockGenerateWorkflow,
-				llmCallLimiter: pLimit(1),
-			});
-
-			expect(mockGenerateWorkflow).toHaveBeenCalledTimes(3);
-			expect(maxActive).toBe(1);
-		});
-
-		it('should aggregate results across generations', async () => {
-			// First two generations pass, third fails
-			mockRunJudgePanel
-				.mockResolvedValueOnce(
-					createMockPanelResult({ majorityPass: true, avgDiagnosticScore: 0.9 }),
-				)
-				.mockResolvedValueOnce(
-					createMockPanelResult({ majorityPass: true, avgDiagnosticScore: 0.8 }),
-				)
-				.mockResolvedValueOnce(
-					createMockPanelResult({ majorityPass: false, avgDiagnosticScore: 0.5 }),
-				);
-
-			const { createPairwiseEvaluator } = await import('../../evaluators/pairwise');
-			const evaluator = createPairwiseEvaluator(mockLlm, { numJudges: 3, numGenerations: 3 });
-
-			const workflow = createMockWorkflow();
-			const mockGenerateWorkflow = jest.fn().mockResolvedValue(createMockWorkflow());
-
-			const feedback = await evaluator.evaluate(workflow, {
-				prompt: 'Create a workflow',
-				generateWorkflow: mockGenerateWorkflow,
-			});
-
-			// Should have multi-gen feedback
-			const correctness = findFeedback(feedback, PAIRWISE_METRICS.PAIRWISE_GENERATION_CORRECTNESS);
-			expect(correctness?.score).toBeCloseTo(2 / 3); // 2 of 3 passed
-			expect(correctness?.comment).toBe('2/3 generations passed');
-
-			const diagnostic = findFeedback(feedback, PAIRWISE_METRICS.PAIRWISE_AGGREGATED_DIAGNOSTIC);
-			expect(diagnostic?.score).toBeCloseTo((0.9 + 0.8 + 0.5) / 3);
-		});
-
-		it('should return per-generation feedback', async () => {
-			mockRunJudgePanel
-				.mockResolvedValueOnce(
-					createMockPanelResult({
-						majorityPass: true,
-						avgDiagnosticScore: 0.9,
-						primaryPasses: 3,
-						judgeResults: [
-							{
-								primaryPass: true,
-								diagnosticScore: 0.9,
-								violations: [],
-								passes: [{ rule: 'Has trigger', justification: 'OK' }],
-							},
-							{
-								primaryPass: true,
-								diagnosticScore: 0.9,
-								violations: [],
-								passes: [{ rule: 'Has trigger', justification: 'OK' }],
-							},
-							{
-								primaryPass: true,
-								diagnosticScore: 0.9,
-								violations: [],
-								passes: [{ rule: 'Has trigger', justification: 'OK' }],
-							},
-						],
-					}),
-				)
-				.mockResolvedValueOnce(
-					createMockPanelResult({
-						majorityPass: false,
-						avgDiagnosticScore: 0.4,
-						primaryPasses: 1,
-						judgeResults: [
-							{
-								primaryPass: true,
-								diagnosticScore: 0.5,
-								violations: [],
-								passes: [{ rule: 'Has trigger', justification: 'OK' }],
-							},
-							{
-								primaryPass: false,
-								diagnosticScore: 0.3,
-								violations: [{ rule: 'Missing Notion', justification: 'No Notion node found' }],
-								passes: [],
-							},
-							{
-								primaryPass: false,
-								diagnosticScore: 0.4,
-								violations: [{ rule: 'No HTTP', justification: 'Contains HTTP Request node' }],
-								passes: [],
-							},
-						],
-					}),
-				);
-
-			const { createPairwiseEvaluator } = await import('../../evaluators/pairwise');
-			const evaluator = createPairwiseEvaluator(mockLlm, { numJudges: 3, numGenerations: 2 });
-
-			const workflow = createMockWorkflow();
-			const mockGenerateWorkflow = jest.fn().mockResolvedValue(createMockWorkflow());
-
-			const feedback = await evaluator.evaluate(workflow, {
-				prompt: 'Create a workflow',
-				generateWorkflow: mockGenerateWorkflow,
-			});
-
-			// Gen 1 feedback
-			const gen1Pass = findFeedback(feedback, 'gen1.majorityPass');
-			expect(gen1Pass?.score).toBe(1);
-			expect(gen1Pass?.comment).toBe('3/3 judges');
-
-			const gen1Diag = findFeedback(feedback, 'gen1.diagnosticScore');
-			expect(gen1Diag?.score).toBe(0.9);
-
-			// Gen 2 feedback
-			const gen2Pass = findFeedback(feedback, 'gen2.majorityPass');
-			expect(gen2Pass?.score).toBe(0);
-			expect(gen2Pass?.comment).toBe('1/3 judges');
-
-			const gen2Diag = findFeedback(feedback, 'gen2.diagnosticScore');
-			expect(gen2Diag?.score).toBe(0.4);
-
-			const gen2Judge2 = findFeedback(feedback, 'gen2.judge2');
-			expect(gen2Judge2?.score).toBe(0);
-			expect(gen2Judge2?.comment).toContain('[Missing Notion] No Notion node found');
-
-			const gen2Judge3 = findFeedback(feedback, 'gen2.judge3');
-			expect(gen2Judge3?.score).toBe(0);
-			expect(gen2Judge3?.comment).toContain('[No HTTP] Contains HTTP Request node');
-		});
-
-		it('should throw if generateWorkflow missing in context for multi-gen', async () => {
-			const { createPairwiseEvaluator } = await import('../../evaluators/pairwise');
-			const evaluator = createPairwiseEvaluator(mockLlm, { numGenerations: 3 });
-
-			const workflow = createMockWorkflow();
-
-			await expect(
-				evaluator.evaluate(workflow, {
-					prompt: 'Create a workflow',
-					// Missing generateWorkflow
-				}),
-			).rejects.toThrow('Multi-gen requires generateWorkflow and prompt in context');
-		});
-
-		it('should throw if prompt missing in context for multi-gen', async () => {
-			const { createPairwiseEvaluator } = await import('../../evaluators/pairwise');
-			const evaluator = createPairwiseEvaluator(mockLlm, { numGenerations: 3 });
-
-			const workflow = createMockWorkflow();
-			const mockGenerateWorkflow = jest.fn().mockResolvedValue(createMockWorkflow());
-
-			await expect(
-				// @ts-expect-error Intentionally missing required prompt to verify runtime validation
-				evaluator.evaluate(workflow, {
-					generateWorkflow: mockGenerateWorkflow,
-					// Missing prompt
-				}),
-			).rejects.toThrow('Multi-gen requires generateWorkflow and prompt in context');
-		});
-
-		it('should run all generations in parallel', async () => {
-			// Track call order with timestamps
-			const callTimes: number[] = [];
-			mockRunJudgePanel.mockImplementation(async () => {
-				callTimes.push(Date.now());
-				await new Promise((r) => setTimeout(r, 50)); // Simulate async work
-				return createMockPanelResult();
-			});
-
-			const { createPairwiseEvaluator } = await import('../../evaluators/pairwise');
-			const evaluator = createPairwiseEvaluator(mockLlm, { numJudges: 3, numGenerations: 3 });
-
-			const workflow = createMockWorkflow();
-			const mockGenerateWorkflow = jest.fn().mockResolvedValue(createMockWorkflow());
-
-			await evaluator.evaluate(workflow, {
-				prompt: 'Create a workflow',
-				generateWorkflow: mockGenerateWorkflow,
-			});
-
-			expect(callTimes).toHaveLength(3);
-		});
-	});
 });

packages/@n8n/ai-workflow-builder.ee/evaluations/__tests__/lifecycle.test.ts

@@ -276,14 +276,14 @@ describe('Console Lifecycle', () => {
 				feedback: [
 					{
 						evaluator: 'pairwise',
-						metric: 'pairwise_generation_correctness',
-						score: 0.5,
-						comment: '1/2 generations passed',
+						metric: 'pairwise_primary',
+						score: 0,
+						comment: '1/3 judges passed',
 						kind: 'score',
 					},
 					{
 						evaluator: 'pairwise',
-						metric: 'gen2.judge2',
+						metric: 'judge2',
 						score: 0,
 						comment: '[No HTTP] Contains HTTP Request node',
 						kind: 'detail',
@@ -297,7 +297,7 @@ describe('Console Lifecycle', () => {
 			expect(mockConsole.log).toHaveBeenCalled();
 			const logOutput = mockConsole.log.mock.calls.flat().join(' ');
 			expect(logOutput).toContain('issues');
-			expect(logOutput).toContain('gen2.judge2');
+			expect(logOutput).toContain('judge2');
 			expect(logOutput).toContain('Contains HTTP Request node');
 		});
 

packages/@n8n/ai-workflow-builder.ee/evaluations/__tests__/multi-gen.test.ts

@@ -1,159 +0,0 @@
-/**
- * Tests for multi-generation utilities.
- *
- * These utilities support aggregating results across multiple workflow
- * generations in pairwise evaluation.
- */
-
-import type { SimpleWorkflow } from '@/types/workflow';
-
-import {
-	getMajorityThreshold,
-	aggregateGenerations,
-	type GenerationDetail,
-} from '../harness/multi-gen';
-
-/** Helper to create a mock workflow */
-function createMockWorkflow(name = 'Test Workflow'): SimpleWorkflow {
-	return { name, nodes: [], connections: {} };
-}
-
-/** Helper to create a generation detail */
-function createGenerationDetail(overrides: Partial<GenerationDetail> = {}): GenerationDetail {
-	return {
-		workflow: createMockWorkflow(),
-		majorityPass: true,
-		diagnosticScore: 0.8,
-		primaryPasses: 2,
-		numJudges: 3,
-		...overrides,
-	};
-}
-
-describe('Multi-Generation Utilities', () => {
-	describe('getMajorityThreshold()', () => {
-		it('should throw for 0 judges', () => {
-			expect(() => getMajorityThreshold(0)).toThrow('numJudges must be >= 1');
-		});
-
-		it('should return 1 for 1 judge', () => {
-			expect(getMajorityThreshold(1)).toBe(1);
-		});
-
-		it('should return 2 for 3 judges', () => {
-			expect(getMajorityThreshold(3)).toBe(2);
-		});
-
-		it('should return 3 for 5 judges', () => {
-			expect(getMajorityThreshold(5)).toBe(3);
-		});
-
-		it('should return 1 for 2 judges (tie-goes-to-pass)', () => {
-			expect(getMajorityThreshold(2)).toBe(1);
-		});
-
-		it('should return 2 for 4 judges (tie-goes-to-pass)', () => {
-			expect(getMajorityThreshold(4)).toBe(2);
-		});
-	});
-
-	describe('aggregateGenerations()', () => {
-		it('should calculate generation correctness', () => {
-			const details = [
-				createGenerationDetail({ majorityPass: true }),
-				createGenerationDetail({ majorityPass: true }),
-				createGenerationDetail({ majorityPass: false }),
-			];
-
-			const result = aggregateGenerations(details);
-
-			expect(result.generationCorrectness).toBeCloseTo(2 / 3);
-			expect(result.passingGenerations).toBe(2);
-			expect(result.totalGenerations).toBe(3);
-		});
-
-		it('should calculate aggregated diagnostic score', () => {
-			const details = [
-				createGenerationDetail({ diagnosticScore: 0.9 }),
-				createGenerationDetail({ diagnosticScore: 0.8 }),
-				createGenerationDetail({ diagnosticScore: 0.7 }),
-			];
-
-			const result = aggregateGenerations(details);
-
-			expect(result.aggregatedDiagnosticScore).toBeCloseTo(0.8);
-		});
-
-		it('should count passing generations', () => {
-			const details = [
-				createGenerationDetail({ majorityPass: true }),
-				createGenerationDetail({ majorityPass: false }),
-				createGenerationDetail({ majorityPass: true }),
-				createGenerationDetail({ majorityPass: true }),
-				createGenerationDetail({ majorityPass: false }),
-			];
-
-			const result = aggregateGenerations(details);
-
-			expect(result.passingGenerations).toBe(3);
-			expect(result.totalGenerations).toBe(5);
-		});
-
-		it('should handle all passing', () => {
-			const details = [
-				createGenerationDetail({ majorityPass: true, diagnosticScore: 1.0 }),
-				createGenerationDetail({ majorityPass: true, diagnosticScore: 1.0 }),
-				createGenerationDetail({ majorityPass: true, diagnosticScore: 1.0 }),
-			];
-
-			const result = aggregateGenerations(details);
-
-			expect(result.generationCorrectness).toBe(1);
-			expect(result.aggregatedDiagnosticScore).toBe(1);
-			expect(result.passingGenerations).toBe(3);
-		});
-
-		it('should handle all failing', () => {
-			const details = [
-				createGenerationDetail({ majorityPass: false, diagnosticScore: 0.3 }),
-				createGenerationDetail({ majorityPass: false, diagnosticScore: 0.2 }),
-				createGenerationDetail({ majorityPass: false, diagnosticScore: 0.1 }),
-			];
-
-			const result = aggregateGenerations(details);
-
-			expect(result.generationCorrectness).toBe(0);
-			expect(result.aggregatedDiagnosticScore).toBeCloseTo(0.2);
-			expect(result.passingGenerations).toBe(0);
-		});
-
-		it('should handle single generation', () => {
-			const details = [createGenerationDetail({ majorityPass: true, diagnosticScore: 0.95 })];
-
-			const result = aggregateGenerations(details);
-
-			expect(result.generationCorrectness).toBe(1);
-			expect(result.aggregatedDiagnosticScore).toBe(0.95);
-			expect(result.passingGenerations).toBe(1);
-			expect(result.totalGenerations).toBe(1);
-		});
-
-		it('should preserve generation details', () => {
-			const workflow1 = createMockWorkflow('Workflow 1');
-			const workflow2 = createMockWorkflow('Workflow 2');
-
-			const details = [
-				createGenerationDetail({ workflow: workflow1, primaryPasses: 3 }),
-				createGenerationDetail({ workflow: workflow2, primaryPasses: 1 }),
-			];
-
-			const result = aggregateGenerations(details);
-
-			expect(result.generationDetails).toHaveLength(2);
-			expect(result.generationDetails[0].workflow.name).toBe('Workflow 1');
-			expect(result.generationDetails[0].primaryPasses).toBe(3);
-			expect(result.generationDetails[1].workflow.name).toBe('Workflow 2');
-			expect(result.generationDetails[1].primaryPasses).toBe(1);
-		});
-	});
-});

packages/@n8n/ai-workflow-builder.ee/evaluations/cli/argument-parser.ts

@@ -30,7 +30,6 @@ export interface EvaluationArgs {
 	donts?: string;
 
 	numJudges: number;
-	numGenerations: number;
 
 	featureFlags?: BuilderFeatureFlags;
 }
@@ -63,7 +62,6 @@ const cliSchema = z
 		donts: z.string().min(1).optional(),
 
 		numJudges: z.coerce.number().int().positive().default(DEFAULTS.NUM_JUDGES),
-		numGenerations: z.coerce.number().int().positive().max(10).default(DEFAULTS.NUM_GENERATIONS),
 
 		langsmith: z.boolean().optional(),
 		templateExamples: z.boolean().default(false),
@@ -189,12 +187,6 @@ const FLAG_DEFS: Record<string, FlagDef> = {
 
 	// Advanced
 	'--judges': { key: 'numJudges', kind: 'string', group: 'advanced', desc: 'Number of LLM judges' },
-	'--generations': {
-		key: 'numGenerations',
-		kind: 'string',
-		group: 'advanced',
-		desc: 'Workflow generations per prompt',
-	},
 };
 
 // Aliases (not shown in help)
@@ -438,7 +430,6 @@ export function parseEvaluationArgs(argv: string[] = process.argv.slice(2)): Eva
 		dos: parsed.dos,
 		donts: parsed.donts,
 		numJudges: parsed.numJudges,
-		numGenerations: parsed.numGenerations,
 		featureFlags,
 	};
 }

packages/@n8n/ai-workflow-builder.ee/evaluations/cli/index.ts

@@ -34,7 +34,11 @@ import {
 	loadDefaultTestCases,
 	getDefaultTestCaseIds,
 } from './csv-prompt-loader';
-import { consumeGenerator, getChatPayload } from '../harness/evaluation-helpers';
+import {
+	consumeGenerator,
+	getChatPayload,
+	getTracingCallbacks,
+} from '../harness/evaluation-helpers';
 import { createLogger } from '../harness/logger';
 import { generateRunId, isWorkflowStateValues } from '../langsmith/types';
 import { EVAL_TYPES, EVAL_USERS } from '../support/constants';
@@ -42,7 +46,8 @@ import { setupTestEnvironment, createAgent } from '../support/environment';
 
 /**
  * Create a workflow generator function.
- * NOTE: Don't pass a tracer - LangSmith tracing is handled via traceable() in the runner.
+ * LangSmith tracing is handled via traceable() in the runner.
+ * We bridge the trace context to LangChain via getTracingCallbacks().
  */
 function createWorkflowGenerator(
 	parsedNodeTypes: INodeTypeDescription[],
@@ -51,6 +56,7 @@ function createWorkflowGenerator(
 ): (prompt: string) => Promise<SimpleWorkflow> {
 	return async (prompt: string): Promise<SimpleWorkflow> => {
 		const runId = generateRunId();
+		const callbacks = await getTracingCallbacks();
 
 		const agent = createAgent({
 			parsedNodeTypes,
@@ -67,6 +73,8 @@ function createWorkflowGenerator(
 					featureFlags,
 				}),
 				EVAL_USERS.LANGSMITH,
+				undefined, // abortSignal
+				callbacks,
 			),
 		);
 
@@ -166,7 +174,6 @@ export async function runV2Evaluation(): Promise<void> {
 			evaluators.push(
 				createPairwiseEvaluator(env.llm, {
 					numJudges: args.numJudges,
-					numGenerations: args.numGenerations,
 				}),
 			);
 			evaluators.push(createProgrammaticEvaluator(env.parsedNodeTypes));
@@ -179,8 +186,6 @@ export async function runV2Evaluation(): Promise<void> {
 			break;
 	}
 
-	// Build context - include generateWorkflow for multi-gen pairwise
-	const isMultiGen = args.suite === 'pairwise' && args.numGenerations > 1;
 	const llmCallLimiter = pLimit(args.concurrency);
 
 	const baseConfig = {
@@ -190,7 +195,7 @@ export async function runV2Evaluation(): Promise<void> {
 		logger,
 		outputDir: args.outputDir,
 		timeoutMs: args.timeoutMs,
-		context: isMultiGen ? { generateWorkflow, llmCallLimiter } : { llmCallLimiter },
+		context: { llmCallLimiter },
 	};
 
 	const config: RunConfig =
@@ -210,9 +215,7 @@ export async function runV2Evaluation(): Promise<void> {
 							args.suite === 'pairwise'
 								? {
 										numJudges: args.numJudges,
-										numGenerations: args.numGenerations,
-										scoringMethod:
-											args.numGenerations > 1 ? 'hierarchical-multi-generation' : 'hierarchical',
+										scoringMethod: 'hierarchical',
 									}
 								: undefined,
 					},
@@ -224,12 +227,10 @@ export async function runV2Evaluation(): Promise<void> {
 				};
 
 	// Run evaluation
-	const summary = await runEvaluation(config);
+	await runEvaluation(config);
 
-	// Exit with appropriate code
-	// Check pass rate
-	const passRate = summary.totalExamples > 0 ? summary.passed / summary.totalExamples : 0;
-	process.exit(passRate >= 0.7 ? 0 : 1);
+	// Always exit 0 on successful completion - pass/fail is informational, not an error
+	process.exit(0);
 }
 
 // Run if called directly

packages/@n8n/ai-workflow-builder.ee/evaluations/evaluators/pairwise/index.ts

@@ -4,19 +4,12 @@ import type { SimpleWorkflow } from '@/types/workflow';
 
 import { runJudgePanel, type EvalCriteria } from './judge-panel';
 import { PAIRWISE_METRICS } from './metrics';
-import { runWithOptionalLimiter, withTimeout } from '../../harness/evaluation-helpers';
-import type { EvaluationContext, Evaluator, Feedback } from '../../harness/harness-types';
-import { aggregateGenerations, type GenerationDetail } from '../../harness/multi-gen';
-
-type MultiGenContext = EvaluationContext & {
-	generateWorkflow: (prompt: string) => Promise<SimpleWorkflow>;
-};
-
-function assertMultiGenContext(ctx: EvaluationContext): asserts ctx is MultiGenContext {
-	if (!ctx.generateWorkflow || !ctx.prompt) {
-		throw new Error('Multi-gen requires generateWorkflow and prompt in context');
-	}
-}
+import type {
+	DisplayLine,
+	EvaluationContext,
+	Evaluator,
+	Feedback,
+} from '../../harness/harness-types';
 
 /**
  * Options for creating a pairwise evaluator.
@@ -24,247 +17,108 @@ function assertMultiGenContext(ctx: EvaluationContext): asserts ctx is MultiGenC
 export interface PairwiseEvaluatorOptions {
 	/** Number of judges to run (default: 3) */
 	numJudges?: number;
-	/** Number of workflow generations (default: 1, no multi-gen) */
-	numGenerations?: number;
-}
-
-/**
- * Evaluate a single workflow with a judge panel.
- * Returns feedback with majority pass, diagnostic score, and per-judge results.
- */
-async function evaluateSingleGeneration(
-	llm: BaseChatModel,
-	workflow: SimpleWorkflow,
-	ctx: EvaluationContext,
-	numJudges: number,
-): Promise<Feedback[]> {
-	const evalCriteria: EvalCriteria = {
-		dos: ctx?.dos,
-		donts: ctx?.donts,
-	};
-
-	const result = await runJudgePanel(llm, workflow, evalCriteria, numJudges, {
-		llmCallLimiter: ctx.llmCallLimiter,
-		timeoutMs: ctx.timeoutMs,
-	});
-
-	const feedback: Feedback[] = [];
-
-	const totalViolations = result.judgeResults.reduce((sum, r) => sum + r.violations.length, 0);
-	const totalPasses = result.judgeResults.reduce((sum, r) => sum + r.passes.length, 0);
-
-	// v1-compatible single-generation metrics
-	feedback.push({
-		evaluator: 'pairwise',
-		metric: PAIRWISE_METRICS.PAIRWISE_PRIMARY,
-		score: result.majorityPass ? 1 : 0,
-		kind: 'score',
-		comment: `${result.primaryPasses}/${numJudges} judges passed`,
-	});
-	feedback.push({
-		evaluator: 'pairwise',
-		metric: PAIRWISE_METRICS.PAIRWISE_DIAGNOSTIC,
-		score: result.avgDiagnosticScore,
-		kind: 'metric',
-	});
-	feedback.push({
-		evaluator: 'pairwise',
-		metric: PAIRWISE_METRICS.PAIRWISE_JUDGES_PASSED,
-		score: result.primaryPasses,
-		kind: 'detail',
-	});
-	feedback.push({
-		evaluator: 'pairwise',
-		metric: PAIRWISE_METRICS.PAIRWISE_TOTAL_PASSES,
-		score: totalPasses,
-		kind: 'detail',
-	});
-	feedback.push({
-		evaluator: 'pairwise',
-		metric: PAIRWISE_METRICS.PAIRWISE_TOTAL_VIOLATIONS,
-		score: totalViolations,
-		kind: 'detail',
-	});
-
-	// Individual judge results
-	for (let i = 0; i < result.judgeResults.length; i++) {
-		const judge = result.judgeResults[i];
-		// Build comment from violations
-		const violationSummary =
-			judge.violations.length > 0
-				? judge.violations.map((v) => `[${v.rule}] ${v.justification}`).join('; ')
-				: undefined;
-		feedback.push({
-			evaluator: 'pairwise',
-			metric: `judge${i + 1}`,
-			score: judge.primaryPass ? 1 : 0,
-			kind: 'detail',
-			comment: violationSummary,
-		});
-	}
-
-	return feedback;
-}
-
-/**
- * Evaluate multiple workflow generations with a judge panel each.
- * Returns aggregated feedback with generation correctness and per-generation details.
- */
-async function evaluateMultiGeneration(
-	llm: BaseChatModel,
-	ctx: MultiGenContext,
-	numJudges: number,
-	numGenerations: number,
-): Promise<Feedback[]> {
-	const evalCriteria: EvalCriteria = {
-		dos: ctx?.dos,
-		donts: ctx?.donts,
-	};
-
-	// Generate all workflows and evaluate in parallel
-	const generationRuns = await Promise.all(
-		Array.from({ length: numGenerations }, async (_, i) => {
-			const workflow = await runWithOptionalLimiter(async () => {
-				return await withTimeout({
-					promise: ctx.generateWorkflow(ctx.prompt),
-					timeoutMs: ctx.timeoutMs,
-					label: 'pairwise:workflow_generation',
-				});
-			}, ctx.llmCallLimiter);
-			const result = await runJudgePanel(llm, workflow, evalCriteria, numJudges, {
-				generationIndex: i + 1,
-				llmCallLimiter: ctx.llmCallLimiter,
-				timeoutMs: ctx.timeoutMs,
-			});
-			return { workflow, result };
-		}),
-	);
-
-	const generationDetails: GenerationDetail[] = generationRuns.map(({ workflow, result }) => ({
-		workflow,
-		majorityPass: result.majorityPass,
-		diagnosticScore: result.avgDiagnosticScore,
-		primaryPasses: result.primaryPasses,
-		numJudges,
-	}));
-
-	// Aggregate results
-	const aggregation = aggregateGenerations(generationDetails);
-
-	// Build feedback
-	const feedback: Feedback[] = [
-		{
-			evaluator: 'pairwise',
-			metric: PAIRWISE_METRICS.PAIRWISE_GENERATION_CORRECTNESS,
-			score: aggregation.generationCorrectness,
-			kind: 'score',
-			comment: `${aggregation.passingGenerations}/${aggregation.totalGenerations} generations passed`,
-		},
-		{
-			evaluator: 'pairwise',
-			metric: PAIRWISE_METRICS.PAIRWISE_AGGREGATED_DIAGNOSTIC,
-			score: aggregation.aggregatedDiagnosticScore,
-			kind: 'metric',
-		},
-		{
-			evaluator: 'pairwise',
-			metric: PAIRWISE_METRICS.PAIRWISE_GENERATIONS_PASSED,
-			score: aggregation.passingGenerations,
-			kind: 'detail',
-		},
-		{
-			evaluator: 'pairwise',
-			metric: PAIRWISE_METRICS.PAIRWISE_TOTAL_JUDGE_CALLS,
-			score: aggregation.totalGenerations * numJudges,
-			kind: 'detail',
-		},
-	];
-
-	// Per-judge details (one set per generation)
-	generationRuns.forEach(({ result }, genIndex) => {
-		for (let i = 0; i < result.judgeResults.length; i++) {
-			const judge = result.judgeResults[i];
-			const violationSummary =
-				judge.violations.length > 0
-					? judge.violations.map((v) => `[${v.rule}] ${v.justification}`).join('; ')
-					: undefined;
-
-			feedback.push({
-				evaluator: 'pairwise',
-				metric: `gen${genIndex + 1}.judge${i + 1}`,
-				score: judge.primaryPass ? 1 : 0,
-				kind: 'detail',
-				comment: violationSummary,
-			});
-		}
-	});
-
-	// Per-generation details
-	aggregation.generationDetails.forEach((gen, i) => {
-		feedback.push({
-			evaluator: 'pairwise',
-			metric: `gen${i + 1}.majorityPass`,
-			score: gen.majorityPass ? 1 : 0,
-			kind: 'detail',
-			comment: `${gen.primaryPasses}/${gen.numJudges} judges`,
-		});
-		feedback.push({
-			evaluator: 'pairwise',
-			metric: `gen${i + 1}.diagnosticScore`,
-			score: gen.diagnosticScore,
-			kind: 'detail',
-		});
-	});
-
-	return feedback;
 }
 
 /**
  * Create a pairwise evaluator that uses a panel of judges.
  * Each judge evaluates the workflow against dos/donts criteria.
  *
- * For single generation (default): evaluates the provided workflow.
- * For multi-generation: generates N workflows and aggregates results.
- *
  * @param llm - Language model for evaluation
  * @param options - Configuration options
  * @returns An evaluator that produces feedback from pairwise evaluation
  *
- * @example Single generation
+ * @example
  * ```typescript
  * const evaluator = createPairwiseEvaluator(llm, { numJudges: 3 });
  * const feedback = await evaluator.evaluate(workflow, { dos, donts });
  * ```
- *
- * @example Multi-generation
- * ```typescript
- * const evaluator = createPairwiseEvaluator(llm, { numJudges: 3, numGenerations: 5 });
- * const feedback = await evaluator.evaluate(workflow, {
- *   dos, donts,
- *   prompt: "Create a workflow...",
- *   generateWorkflow: async (p) => agent.generate(p),
- * });
- * ```
  */
 export function createPairwiseEvaluator(
 	llm: BaseChatModel,
 	options?: PairwiseEvaluatorOptions,
 ): Evaluator<EvaluationContext> {
 	const numJudges = options?.numJudges ?? 3;
-	const numGenerations = options?.numGenerations ?? 1;
 
 	return {
 		name: 'pairwise',
 
 		async evaluate(workflow: SimpleWorkflow, ctx: EvaluationContext): Promise<Feedback[]> {
-			// Single generation (existing behavior)
-			if (numGenerations === 1) {
-				return await evaluateSingleGeneration(llm, workflow, ctx, numJudges);
+			const evalCriteria: EvalCriteria = {
+				dos: ctx?.dos,
+				donts: ctx?.donts,
+			};
+
+			const result = await runJudgePanel(llm, workflow, evalCriteria, numJudges, {
+				llmCallLimiter: ctx.llmCallLimiter,
+				timeoutMs: ctx.timeoutMs,
+			});
+
+			const feedback: Feedback[] = [];
+
+			const totalViolations = result.judgeResults.reduce((sum, r) => sum + r.violations.length, 0);
+			const totalPasses = result.judgeResults.reduce((sum, r) => sum + r.passes.length, 0);
+
+			// Primary metrics
+			feedback.push({
+				evaluator: 'pairwise',
+				metric: PAIRWISE_METRICS.PAIRWISE_PRIMARY,
+				score: result.majorityPass ? 1 : 0,
+				kind: 'score',
+				comment: `${result.primaryPasses}/${numJudges} judges passed`,
+			});
+			feedback.push({
+				evaluator: 'pairwise',
+				metric: PAIRWISE_METRICS.PAIRWISE_DIAGNOSTIC,
+				score: result.avgDiagnosticScore,
+				kind: 'metric',
+			});
+			feedback.push({
+				evaluator: 'pairwise',
+				metric: PAIRWISE_METRICS.PAIRWISE_JUDGES_PASSED,
+				score: result.primaryPasses,
+				kind: 'detail',
+			});
+			feedback.push({
+				evaluator: 'pairwise',
+				metric: PAIRWISE_METRICS.PAIRWISE_TOTAL_PASSES,
+				score: totalPasses,
+				kind: 'detail',
+			});
+			feedback.push({
+				evaluator: 'pairwise',
+				metric: PAIRWISE_METRICS.PAIRWISE_TOTAL_VIOLATIONS,
+				score: totalViolations,
+				kind: 'detail',
+			});
+
+			// Individual judge results
+			for (let i = 0; i < result.judgeResults.length; i++) {
+				const judge = result.judgeResults[i];
+				const violationSummary =
+					judge.violations.length > 0
+						? judge.violations.map((v) => `[${v.rule}] ${v.justification}`).join('; ')
+						: undefined;
+
+				// Pre-format display lines for verbose logging
+				const displayLines: DisplayLine[] = [];
+				if (judge.violations.length > 0) {
+					for (const v of judge.violations) {
+						displayLines.push({ text: `[${v.rule}]`, color: 'yellow' });
+						displayLines.push({ text: v.justification, color: 'red' });
+					}
+				}
+
+				feedback.push({
+					evaluator: 'pairwise',
+					metric: `judge${i + 1}`,
+					score: judge.primaryPass ? 1 : 0,
+					kind: 'detail',
+					comment: violationSummary,
+					details: displayLines.length > 0 ? { displayLines } : undefined,
+				});
 			}
 
-			// Multi-generation - validate required context
-			assertMultiGenContext(ctx);
-			return await evaluateMultiGeneration(llm, ctx, numJudges, numGenerations);
+			return feedback;
 		},
 	};
 }

packages/@n8n/ai-workflow-builder.ee/evaluations/evaluators/pairwise/judge-panel.ts

@@ -1,8 +1,13 @@
 import type { BaseChatModel } from '@langchain/core/language_models/chat_models';
+import type { RunnableConfig } from '@langchain/core/runnables';
 
 import { evaluateWorkflowPairwise, type PairwiseEvaluationResult } from './judge-chain';
 import type { SimpleWorkflow } from '../../../src/types/workflow';
-import { runWithOptionalLimiter, withTimeout } from '../../harness/evaluation-helpers';
+import {
+	getTracingCallbacks,
+	runWithOptionalLimiter,
+	withTimeout,
+} from '../../harness/evaluation-helpers';
 import type { EvaluationContext } from '../../harness/harness-types';
 
 // ============================================================================
@@ -31,23 +36,6 @@ export interface JudgePanelResult {
 	timing?: JudgePanelTiming;
 }
 
-export interface GenerationResult extends JudgePanelResult {
-	workflow: SimpleWorkflow;
-}
-
-export interface MultiGenerationAggregation {
-	/** Generation correctness: (# passing generations) / total generations */
-	generationCorrectness: number;
-	/** Average diagnostic score across all generations */
-	aggregatedDiagnosticScore: number;
-	/** Number of generations that passed majority vote */
-	passingGenerations: number;
-	/** Total number of generations run */
-	totalGenerations: number;
-	/** Detailed results for each generation */
-	generationDetails: GenerationResult[];
-}
-
 // ============================================================================
 // Helpers
 // ============================================================================
@@ -69,8 +57,6 @@ export function getMajorityThreshold(numJudges: number): number {
 // ============================================================================
 
 export interface JudgePanelOptions {
-	/** Generation index (1-based) for metadata */
-	generationIndex?: number;
 	/** Experiment name for metadata */
 	experimentName?: string;
 	/** Optional limiter for LLM calls (shared across harness) */
@@ -97,27 +83,30 @@ export async function runJudgePanel(
 	numJudges: number,
 	options?: JudgePanelOptions,
 ): Promise<JudgePanelResult> {
-	const { generationIndex, experimentName, llmCallLimiter, timeoutMs } = options ?? {};
+	const { experimentName, llmCallLimiter, timeoutMs } = options ?? {};
 	const panelStartTime = Date.now();
 
+	// Bridge LangSmith traceable context to LangChain callbacks
+	const callbacks = await getTracingCallbacks();
+
 	// Run all judges in parallel, tracking timing for each
 	const judgeTimings: number[] = [];
 	const judgeResults = await Promise.all(
 		Array.from({ length: numJudges }, async (_, judgeIndex) => {
 			const runJudge = async (): Promise<PairwiseEvaluationResult> => {
 				const judgeStartTime = Date.now();
+
+				// Build config with callbacks for proper trace context propagation
+				const config: RunnableConfig = {
+					runName: `judge_${judgeIndex + 1}`,
+					metadata: {
+						...(experimentName && { experiment_name: experimentName }),
+					},
+					callbacks,
+				};
+
 				const result = await withTimeout({
-					promise: evaluateWorkflowPairwise(
-						llm,
-						{ workflowJSON: workflow, evalCriteria },
-						{
-							runName: `judge_${judgeIndex + 1}`,
-							metadata: {
-								...(experimentName && { experiment_name: experimentName }),
-								...(generationIndex && { evaluating_generation: `generation_${generationIndex}` }),
-							},
-						},
-					),
+					promise: evaluateWorkflowPairwise(llm, { workflowJSON: workflow, evalCriteria }, config),
 					timeoutMs,
 					label: `pairwise:judge${judgeIndex + 1}`,
 				});
@@ -160,28 +149,3 @@ export function aggregateJudgeResults(
 		avgDiagnosticScore,
 	};
 }
-
-// ============================================================================
-// Multi-Generation Aggregation
-// ============================================================================
-
-/**
- * Aggregate results across multiple generations.
- */
-export function aggregateGenerations(
-	generationResults: GenerationResult[],
-): MultiGenerationAggregation {
-	const totalGenerations = generationResults.length;
-	const passingGenerations = generationResults.filter((g) => g.majorityPass).length;
-	const generationCorrectness = passingGenerations / totalGenerations;
-	const aggregatedDiagnosticScore =
-		generationResults.reduce((sum, g) => sum + g.avgDiagnosticScore, 0) / totalGenerations;
-
-	return {
-		generationCorrectness,
-		aggregatedDiagnosticScore,
-		passingGenerations,
-		totalGenerations,
-		generationDetails: generationResults,
-	};
-}

packages/@n8n/ai-workflow-builder.ee/evaluations/evaluators/pairwise/metrics.ts

@@ -1,14 +1,7 @@
 export const PAIRWISE_METRICS = {
-	// Single generation metrics
 	PAIRWISE_DIAGNOSTIC: 'pairwise_diagnostic',
 	PAIRWISE_JUDGES_PASSED: 'pairwise_judges_passed',
 	PAIRWISE_PRIMARY: 'pairwise_primary',
 	PAIRWISE_TOTAL_PASSES: 'pairwise_total_passes',
 	PAIRWISE_TOTAL_VIOLATIONS: 'pairwise_total_violations',
-
-	// Multi-generation metrics
-	PAIRWISE_AGGREGATED_DIAGNOSTIC: 'pairwise_aggregated_diagnostic',
-	PAIRWISE_GENERATION_CORRECTNESS: 'pairwise_generation_correctness',
-	PAIRWISE_GENERATIONS_PASSED: 'pairwise_generations_passed',
-	PAIRWISE_TOTAL_JUDGE_CALLS: 'pairwise_total_judge_calls',
 } as const;

packages/@n8n/ai-workflow-builder.ee/evaluations/harness/evaluation-helpers.ts

@@ -1,9 +1,23 @@
+import type { Callbacks } from '@langchain/core/callbacks/manager';
+import { getLangchainCallbacks } from 'langsmith/langchain';
 import { v4 as uuid } from 'uuid';
 
 import type { LlmCallLimiter } from './harness-types';
 import type { BuilderFeatureFlags, ChatPayload } from '../../src/workflow-builder-agent';
 import { DEFAULTS } from '../support/constants';
 
+/**
+ * Get LangChain callbacks that bridge the current traceable context.
+ * Returns undefined if not in a traceable context.
+ */
+export async function getTracingCallbacks(): Promise<Callbacks | undefined> {
+	try {
+		return await getLangchainCallbacks();
+	} catch {
+		return undefined;
+	}
+}
+
 export async function consumeGenerator<T>(gen: AsyncGenerator<T>) {
 	for await (const _ of gen) {
 		/* consume all */

packages/@n8n/ai-workflow-builder.ee/evaluations/harness/harness-types.ts

@@ -21,11 +21,6 @@ export interface EvaluationContext {
 	donts?: string;
 	/** Optional reference workflows for similarity-based checks (best match wins) */
 	referenceWorkflows?: SimpleWorkflow[];
-	/**
-	 * Optional generator for multi-generation evaluations.
-	 * When present, pairwise evaluator can generate multiple workflows from the same prompt.
-	 */
-	generateWorkflow?: (prompt: string) => Promise<SimpleWorkflow>;
 	/**
 	 * Optional limiter for LLM-bound work (generation + evaluators).
 	 * When provided, treat it as the global knob for overall parallel LLM calls.
@@ -44,6 +39,15 @@ export type TestCaseContext = Omit<Partial<EvaluationContext>, 'prompt'>;
 /** Global context attached to a run (prompt is provided per test case). */
 export type GlobalRunContext = Omit<Partial<EvaluationContext>, 'prompt'>;
 
+/**
+ * A styled line for verbose display output.
+ * Evaluators can provide these in `details.displayLines` for custom formatting.
+ */
+export interface DisplayLine {
+	text: string;
+	color?: 'yellow' | 'red' | 'dim';
+}
+
 /**
  * What evaluators return - a single piece of feedback.
  */
@@ -62,6 +66,12 @@ export interface Feedback {
 	 * - `detail`: unstable/verbose metrics that should not affect scoring
 	 */
 	kind: 'score' | 'metric' | 'detail';
+	/**
+	 * Optional structured details for rich display.
+	 * Evaluators can provide structured data here for better logging.
+	 * The `comment` field remains the primary text for LangSmith.
+	 */
+	details?: { displayLines?: DisplayLine[] } & Record<string, unknown>;
 }
 
 /**

packages/@n8n/ai-workflow-builder.ee/evaluations/harness/lifecycle.ts

@@ -1,6 +1,7 @@
 import pc from 'picocolors';
 
 import type {
+	DisplayLine,
 	EvaluationLifecycle,
 	RunConfig,
 	Feedback,
@@ -64,10 +65,6 @@ const DISPLAY_METRICS_BY_EVALUATOR: Record<string, string[]> = {
 		'pairwise_judges_passed',
 		'pairwise_total_passes',
 		'pairwise_total_violations',
-		'pairwise_generation_correctness',
-		'pairwise_aggregated_diagnostic',
-		'pairwise_generations_passed',
-		'pairwise_total_judge_calls',
 	],
 };
 
@@ -75,10 +72,36 @@ const PAIRWISE_COUNT_METRICS = new Set([
 	'pairwise_judges_passed',
 	'pairwise_total_passes',
 	'pairwise_total_violations',
-	'pairwise_generations_passed',
-	'pairwise_total_judge_calls',
 ]);
 
+const PAIRWISE_DISPLAY_NAMES: Record<string, string> = {
+	pairwise_primary: 'primary',
+	pairwise_diagnostic: 'diagnostic',
+	pairwise_judges_passed: 'judges_passed',
+	pairwise_total_passes: 'total_passes',
+	pairwise_total_violations: 'total_violations',
+};
+
+function getDisplayMetricName(evaluator: string, metric: string): string {
+	if (evaluator === 'pairwise') {
+		return PAIRWISE_DISPLAY_NAMES[metric] ?? metric;
+	}
+	return metric;
+}
+
+function isDisplayLine(item: unknown): item is DisplayLine {
+	if (typeof item !== 'object' || item === null) return false;
+	if (!('text' in item)) return false;
+	return typeof item.text === 'string';
+}
+
+function getDisplayLines(details?: Feedback['details']): DisplayLine[] | undefined {
+	if (!details?.displayLines || !Array.isArray(details.displayLines)) return undefined;
+	// Validate each item matches DisplayLine shape
+	if (!details.displayLines.every(isDisplayLine)) return undefined;
+	return details.displayLines;
+}
+
 function formatMetricValue(evaluator: string, metric: string, score: number): string {
 	if (evaluator === 'pairwise' && PAIRWISE_COUNT_METRICS.has(metric)) {
 		if (!Number.isFinite(score)) return 'N/A';
@@ -198,7 +221,8 @@ function formatEvaluatorLines(args: {
 		const metricsLine = picked
 			.map((f) => {
 				const color = scoreColor(f.score);
-				return `${f.metric}: ${color(formatMetricValue(evaluatorName, f.metric, f.score))}`;
+				const displayName = getDisplayMetricName(evaluatorName, f.metric);
+				return `${displayName}: ${color(formatMetricValue(evaluatorName, f.metric, f.score))}`;
 			})
 			.join(pc.dim(' | '));
 		lines.push(pc.dim('    ') + metricsLine);
@@ -221,8 +245,20 @@ function formatEvaluatorLines(args: {
 		const top = issues.slice(0, 3);
 		lines.push(pc.dim(`    issues(top=${top.length}):`));
 		for (const issue of top) {
-			const comment = truncateForSingleLine(issue.comment ?? '', 320);
-			lines.push(pc.dim(`      - [${issue.metric}] `) + pc.red(comment));
+			const displayMetric = getDisplayMetricName(evaluatorName, issue.metric);
+			const displayLines = getDisplayLines(issue.details);
+			if (displayLines && displayLines.length > 0) {
+				// Evaluator provided custom display lines with optional color
+				lines.push(pc.dim(`      - [${displayMetric}]`));
+				for (const dl of displayLines) {
+					const truncated = truncateForSingleLine(dl.text, 300);
+					const colorFn = dl.color === 'yellow' ? pc.yellow : dl.color === 'dim' ? pc.dim : pc.red;
+					lines.push(pc.dim('          ') + colorFn(truncated));
+				}
+			} else {
+				const comment = truncateForSingleLine(issue.comment ?? '', 320);
+				lines.push(pc.dim(`      - [${displayMetric}] `) + pc.red(comment));
+			}
 		}
 		if (issues.length > top.length) {
 			lines.push(pc.dim(`      ... and ${issues.length - top.length} more`));

packages/@n8n/ai-workflow-builder.ee/evaluations/harness/multi-gen.ts

@@ -1,97 +0,0 @@
-/**
- * Multi-generation utilities for pairwise evaluation.
- *
- * These utilities support generating multiple workflows from a single prompt
- * and aggregating evaluation results across all generations.
- */
-
-import type { SimpleWorkflow } from '@/types/workflow';
-
-/**
- * Details for a single generation in multi-gen evaluation.
- */
-export interface GenerationDetail {
-	/** The generated workflow */
-	workflow: SimpleWorkflow;
-	/** Whether majority of judges passed this generation */
-	majorityPass: boolean;
-	/** Average diagnostic score across judges */
-	diagnosticScore: number;
-	/** Number of judges that passed */
-	primaryPasses: number;
-	/** Total number of judges */
-	numJudges: number;
-}
-
-/**
- * Aggregated result across multiple generations.
- */
-export interface MultiGenerationResult {
-	/** Ratio of passing generations (0-1) */
-	generationCorrectness: number;
-	/** Average diagnostic score across all generations */
-	aggregatedDiagnosticScore: number;
-	/** Count of generations that passed */
-	passingGenerations: number;
-	/** Total number of generations */
-	totalGenerations: number;
-	/** Detailed results for each generation */
-	generationDetails: GenerationDetail[];
-}
-
-/**
- * Calculate the majority threshold for a given number of judges.
- *
- * @param numJudges - Number of judges in the panel
- * @returns Minimum number of passes needed for majority
- *
- * @example
- * getMajorityThreshold(3) // returns 2
- * getMajorityThreshold(5) // returns 3
- */
-export function getMajorityThreshold(numJudges: number): number {
-	if (!Number.isFinite(numJudges) || numJudges < 1) {
-		throw new Error(`numJudges must be >= 1 (received ${String(numJudges)})`);
-	}
-	return Math.ceil(numJudges / 2);
-}
-
-/**
- * Aggregate results across multiple workflow generations.
- *
- * Calculates:
- * - Generation correctness: passing generations / total generations
- * - Aggregated diagnostic: average diagnostic score across all generations
- *
- * @param details - Array of generation details to aggregate
- * @returns Aggregated multi-generation result
- *
- * @example
- * ```typescript
- * const result = aggregateGenerations([
- *   { majorityPass: true, diagnosticScore: 0.9, ... },
- *   { majorityPass: false, diagnosticScore: 0.6, ... },
- *   { majorityPass: true, diagnosticScore: 0.85, ... },
- * ]);
- * // result.generationCorrectness = 0.67 (2/3)
- * // result.aggregatedDiagnosticScore = 0.78
- * ```
- */
-export function aggregateGenerations(details: GenerationDetail[]): MultiGenerationResult {
-	const totalGenerations = details.length;
-	const passingGenerations = details.filter((d) => d.majorityPass).length;
-	const generationCorrectness = totalGenerations > 0 ? passingGenerations / totalGenerations : 0;
-
-	const aggregatedDiagnosticScore =
-		totalGenerations > 0
-			? details.reduce((sum, d) => sum + d.diagnosticScore, 0) / totalGenerations
-			: 0;
-
-	return {
-		generationCorrectness,
-		aggregatedDiagnosticScore,
-		passingGenerations,
-		totalGenerations,
-		generationDetails: details,
-	};
-}

packages/@n8n/ai-workflow-builder.ee/evaluations/harness/runner.ts

@@ -20,6 +20,7 @@ import type {
 	RunSummary,
 	EvaluationLifecycle,
 	LangsmithExampleFilters,
+	LlmCallLimiter,
 } from './harness-types.js';
 import type { EvalLogger } from './logger';
 import { createArtifactSaver, type ArtifactSaver } from './output';
@@ -745,7 +746,6 @@ async function runLangsmithEvaluateAndFlush(params: {
 
 /**
  * Run evaluation in LangSmith mode.
- * This wraps generation + evaluation in a traceable function.
  */
 async function runLangsmith(config: LangsmithRunConfig): Promise<RunSummary> {
 	const {
@@ -776,10 +776,32 @@ async function runLangsmith(config: LangsmithRunConfig): Promise<RunSummary> {
 	const artifactSaver = createArtifactSaverIfRequested({ outputDir, logger });
 	const capturedResults: ExampleResult[] | null = artifactSaver ? [] : null;
 
+	// Create traceable wrapper ONCE outside target function to avoid context leaking
+	// when running concurrent evaluations. Pass all parameters explicitly (no closures).
+	const traceableGenerateWorkflow = traceable(
+		async (args: {
+			prompt: string;
+			genFn: (prompt: string) => Promise<SimpleWorkflow>;
+			limiter?: LlmCallLimiter;
+			genTimeoutMs?: number;
+		}): Promise<SimpleWorkflow> => {
+			return await runWithOptionalLimiter(async () => {
+				return await withTimeout({
+					promise: args.genFn(args.prompt),
+					timeoutMs: args.genTimeoutMs,
+					label: 'workflow_generation',
+				});
+			}, args.limiter);
+		},
+		{
+			name: 'workflow_generation',
+			run_type: 'chain',
+			client: lsClient,
+		},
+	);
+
 	// Create target function that does ALL work (generation + evaluation)
-	// NOTE: Do NOT wrap target with traceable() - evaluate() handles that automatically
-	// and applies critical options (on_end callback, reference_example_id, client).
-	// Only wrap inner operations with traceable() for child traces.
+	// NOTE: Do NOT wrap target with traceable() - evaluate() handles tracing automatically.
 	let targetCallCount = 0;
 	let totalExamples = 0;
 	const stats = {
@@ -802,25 +824,12 @@ async function runLangsmith(config: LangsmithRunConfig): Promise<RunSummary> {
 		const genStart = Date.now();
 
 		try {
-			const limiter = effectiveGlobalContext.llmCallLimiter;
-
-			// Generate workflow - wrapped in traceable for proper child trace visibility
-			const traceableGenerate = traceable(
-				async () =>
-					await runWithOptionalLimiter(async () => {
-						return await withTimeout({
-							promise: generateWorkflow(prompt),
-							timeoutMs,
-							label: 'workflow_generation',
-						});
-					}, limiter),
-				{
-					name: 'workflow_generation',
-					run_type: 'chain',
-					client: lsClient,
-				},
-			);
-			const workflow = await traceableGenerate();
+			const workflow = await traceableGenerateWorkflow({
+				prompt,
+				genFn: generateWorkflow,
+				limiter: effectiveGlobalContext.llmCallLimiter,
+				genTimeoutMs: timeoutMs,
+			});
 			const genDurationMs = Date.now() - genStart;
 			lifecycle?.onWorkflowGenerated?.(workflow, genDurationMs);
 

packages/@n8n/ai-workflow-builder.ee/evaluations/index.ts

@@ -54,14 +54,6 @@ export {
 	type ArtifactSaverOptions,
 } from './harness/output';
 
-// Multi-generation utilities
-export {
-	getMajorityThreshold,
-	aggregateGenerations,
-	type GenerationDetail,
-	type MultiGenerationResult,
-} from './harness/multi-gen';
-
 // Trace filtering (re-exported from v1 for convenience)
 export {
 	createTraceFilters,

packages/@n8n/ai-workflow-builder.ee/evaluations/support/constants.ts

@@ -24,7 +24,6 @@ export const TRACEABLE_NAMES = {
 
 export const DEFAULTS = {
 	NUM_JUDGES: 3,
-	NUM_GENERATIONS: 1,
 	EXPERIMENT_NAME: 'pairwise-evals',
 	LLM_JUDGE_EXPERIMENT_NAME: 'workflow-builder-evaluation',
 	CONCURRENCY: 5,