feat(ai-builder): Validate the eval conversation-seed message envelope (no-changelog) (#35138)

Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>

packages/@n8n/instance-ai/evaluations/README.md

@@ -818,6 +818,8 @@ For a **synthetic, sanitized** seed you want pinned in git (never a real user's
 
 Schema in `harness/conversation-seed.ts` — `messages` plus optional `workflows` and `dataTables`. Two constraints worth knowing: a workflow `id` must be ≥8 characters (`remapSeedWorkflowIds` refuses to rewrite shorter ids safely), and a seeded `build-workflow` tool call's `output.workflowId` must match the seeded workflow's `id`, or the remap separates them and the agent can't find the workflow it's meant to act on.
 
+**Each message must carry the envelope** — `id`, `role` (`user` or `assistant`), `type` (`llm`, `custom`, …), `createdAt` (a parseable timestamp; ordering before the live turn depends on it), and `content` as an array of blocks each with a `type`. Only the envelope is validated: **unknown block types are accepted**, because block shapes belong to the agent's message store rather than to the harness, and unknown keys are preserved rather than stripped. A `type: 'custom'` message is the one exception — it's stored but never rendered, so it may omit `role` and carry any `content` shape. The envelope is checked because a malformed message would otherwise be stored verbatim *and* skipped by `transcriptPrefixFromSeed`, leaving the case graded against a transcript that doesn't match what the agent saw.
+
 The seed lives **in the case body** rather than in a sibling file, so it travels with the case whatever the source — a JSON on disk, a suite pulled with `--source langtracer`, or a case body handed to a dispatcher. (There used to be a `seedFile` path pointing at a sibling JSON. Only the disk loader could resolve it, so a case delivered any other way lost its seed; the key is gone and a case still carrying it fails at load.)
 
 #### How restore works (all paths)
@@ -871,7 +873,7 @@ evaluations/
 ├── checklist/            # LLM verification with retry
 ├── credentials/          # Test credential seeding
 ├── data/agents/          # authoring dir for intent-resolution cases (the corpus lives in LangTracer suite `agents`)
-├── data/workflows/       # seeded carve-out case JSONs + seeds/ (the corpus lives in LangTracer)
+├── data/workflows/       # seeded carve-out case JSONs (the corpus lives in LangTracer)
 ├── data/subagent/        # workflow-build compatibility fixture JSON files
 ├── data/pairwise/        # Local pairwise fixture (small smoke set)
 ├── harness/              # Runners: buildWorkflow + executeScenario (e2e), in-memory event bus (discovery)

packages/@n8n/instance-ai/evaluations/__tests__/conversation-seed.test.ts

@@ -1,4 +1,5 @@
 import {
+	ConversationSeedSchema,
 	remapSeedWorkflowIds,
 	seedFromProse,
 	transcriptPrefixFromSeed,
@@ -41,6 +42,114 @@ function makeSeed(): ConversationSeed {
 	};
 }
 
+describe('ConversationSeedSchema message envelope', () => {
+	const message = (over: Record<string, unknown> = {}) => ({
+		id: 'm1',
+		role: 'user',
+		type: 'llm',
+		createdAt: '2026-01-01T00:00:00.000Z',
+		content: [{ type: 'text', text: 'build it' }],
+		...over,
+	});
+	const parse = (...messages: Array<Record<string, unknown>>) =>
+		ConversationSeedSchema.safeParse({ messages });
+	const errorOf = (result: ReturnType<typeof parse>) =>
+		result.success ? '' : JSON.stringify(result.error.issues);
+
+	it('accepts a well-formed message', () => {
+		expect(parse(message()).success).toBe(true);
+	});
+
+	for (const field of ['id', 'role', 'type', 'createdAt', 'content'] as const) {
+		it(`rejects a message missing ${field}`, () => {
+			const broken = message();
+			delete broken[field];
+			const result = parse(broken);
+			expect(result.success).toBe(false);
+			expect(errorOf(result)).toContain(field);
+		});
+	}
+
+	it('rejects a role the transcript builder would silently drop', () => {
+		// The typo'd-role case: stored verbatim, then skipped by
+		// transcriptPrefixFromSeed, so the case grades a transcript the agent never saw.
+		const result = parse(message({ role: 'assistent' }));
+		expect(result.success).toBe(false);
+		expect(errorOf(result)).toContain('role');
+	});
+
+	it('rejects a createdAt that is not a real timestamp', () => {
+		// Ordering before the live turn depends on this parsing.
+		const result = parse(message({ createdAt: 'yesterday' }));
+		expect(result.success).toBe(false);
+		expect(errorOf(result)).toContain('createdAt');
+	});
+
+	it('rejects content that is not an array of blocks', () => {
+		expect(parse(message({ content: 'build it' })).success).toBe(false);
+		expect(parse(message({ content: [{ text: 'no type' }] })).success).toBe(false);
+	});
+
+	it('accepts an unknown block type — block shapes are the message store’s contract', () => {
+		expect(parse(message({ content: [{ type: 'some-future-block', payload: {} }] })).success).toBe(
+			true,
+		);
+	});
+
+	it('accepts a custom message with no role and non-array content (stored, never rendered)', () => {
+		const result = parse(
+			{ id: 'c1', type: 'custom', data: { widget: 'card' }, createdAt: '2026-01-01T00:00:00Z' },
+			message(),
+		);
+		expect(result.success).toBe(true);
+	});
+
+	it('preserves unknown keys on messages and blocks rather than stripping them', () => {
+		// The load-bearing one: z.object strips unknown keys by default, which would
+		// silently gut toolCallId/input/output from every tool call before restore.
+		const result = ConversationSeedSchema.safeParse({
+			messages: [
+				message({
+					messageGroupId: 'mg-1',
+					content: [
+						{
+							type: 'tool-call',
+							toolCallId: 'tc-1',
+							toolName: 'build-workflow',
+							state: 'resolved',
+							input: { name: 'Digest' },
+							output: { success: true, workflowId: 'AbCdEf1234567890' },
+						},
+					],
+				}),
+			],
+		});
+		expect(result.success).toBe(true);
+		if (!result.success) return;
+		const [only] = result.data.messages;
+		expect(only.messageGroupId).toBe('mg-1');
+		expect(only.content?.[0]).toMatchObject({
+			toolCallId: 'tc-1',
+			toolName: 'build-workflow',
+			state: 'resolved',
+			input: { name: 'Digest' },
+			output: { success: true, workflowId: 'AbCdEf1234567890' },
+		});
+	});
+
+	it('accepts what seedFromProse produces', () => {
+		const seed = seedFromProse([
+			{ role: 'user', text: 'hi' },
+			{ role: 'assistant', text: 'hello' },
+		]);
+		expect(ConversationSeedSchema.safeParse(seed).success).toBe(true);
+	});
+
+	it('still requires at least one message', () => {
+		expect(ConversationSeedSchema.safeParse({ messages: [] }).success).toBe(false);
+	});
+});
+
 describe('seedFromProse', () => {
 	it('converts turns to llm text messages with ascending past timestamps', () => {
 		const seed = seedFromProse([

packages/@n8n/instance-ai/evaluations/__tests__/data-workflows-schema.test.ts

@@ -101,7 +101,13 @@ describe('EvalTestCaseSchema', () => {
 			...validFixture(),
 			conversationSeed: {
 				messages: [
-					{ id: 'm1', type: 'llm', role: 'user', content: [{ type: 'text', text: 'build it' }] },
+					{
+						id: 'm1',
+						type: 'llm',
+						role: 'user',
+						createdAt: '2026-06-29T09:00:00.000Z',
+						content: [{ type: 'text', text: 'build it' }],
+					},
 				],
 			},
 		});
@@ -122,7 +128,13 @@ describe('EvalTestCaseSchema', () => {
 				...validFixture(),
 				conversationSeed: {
 					messages: [
-						{ id: 'm1', type: 'llm', role: 'user', content: [{ type: 'text', text: 'build it' }] },
+						{
+							id: 'm1',
+							type: 'llm',
+							role: 'user',
+							createdAt: '2026-06-29T09:00:00.000Z',
+							content: [{ type: 'text', text: 'build it' }],
+						},
 					],
 				},
 				priorConversation: [{ role: 'user', text: 'prelude' }],

packages/@n8n/instance-ai/evaluations/__tests__/langsmith-seed.test.ts

@@ -106,7 +106,8 @@ describe('reconstructSeedFromThread', () => {
 		// Only turn 1 is seeded — the pinned turn and everything after it are excluded.
 		const userTexts = result.seed.messages
 			.filter((m) => m.role === 'user')
-			.map((m) => (m.content as Array<{ text: string }>)[0].text);
+			// Block fields are `unknown` by design (the store owns block shapes).
+			.map((m) => (m.content?.[0] as { text: string } | undefined)?.text ?? '');
 		expect(userTexts).toEqual(['Build Otter Digest, daily 9am']);
 	});
 

packages/@n8n/instance-ai/evaluations/__tests__/langtracer-provider.test.ts

@@ -39,6 +39,7 @@ describe('casesFromExportedFiles', () => {
 								id: 'm1',
 								type: 'llm',
 								role: 'user',
+								createdAt: '2026-06-29T09:00:00.000Z',
 								content: [{ type: 'text', text: 'build it' }],
 							},
 						],

packages/@n8n/instance-ai/evaluations/__tests__/langtracer-to-exported.test.ts

@@ -129,6 +129,7 @@ describe('unsupportedPushReason', () => {
 								id: 'm1',
 								type: 'llm',
 								role: 'user',
+								createdAt: '2026-06-29T09:00:00.000Z',
 								content: [{ type: 'text', text: 'build it' }],
 							},
 						],

packages/@n8n/instance-ai/evaluations/__tests__/mcp-builder.test.ts

@@ -140,7 +140,13 @@ describe('unsupportedMcpBuildSetupFields', () => {
 			{
 				conversationSeed: {
 					messages: [
-						{ id: 'm1', type: 'llm', role: 'user', content: [{ type: 'text', text: 'build it' }] },
+						{
+							id: 'm1',
+							type: 'llm',
+							role: 'user',
+							createdAt: '2026-06-29T09:00:00.000Z',
+							content: [{ type: 'text', text: 'build it' }],
+						},
 					],
 					workflows: [],
 					dataTables: [],

packages/@n8n/instance-ai/evaluations/harness/conversation-seed.ts

@@ -43,11 +43,77 @@ const SeedDataTableSchema = z.object({
 	// of a trace and are kept out of the eval instance entirely.
 });
 
+/** A content block. Only `type` is ours to require — the block shapes belong to
+ *  the agent's message store, so an unrecognised one is accepted and simply not
+ *  interpreted. `.passthrough()` is load-bearing: `z.object` strips unknown keys,
+ *  which would silently gut `toolCallId`/`input`/`output` from every tool call. */
+const SeedMessageBlockSchema = z
+	.object({
+		type: z.string().min(1, 'a content block needs a non-empty `type`'),
+	})
+	.passthrough();
+
+/** A message envelope. Validated because a hand-authored seed is now the primary
+ *  path, and a malformed message is stored verbatim AND skipped by
+ *  `transcriptPrefixFromSeed` — so the case grades against a transcript that
+ *  doesn't match what the agent actually saw. Envelope only; block internals are
+ *  the store's contract, not ours. */
+const seedMessageObjectSchema = z
+	.object({
+		id: z.string().min(1),
+		// Restricted to the two roles the transcript builder renders: any other
+		// value is guaranteed to vanish from the judge transcript, which is the
+		// exact silent failure this schema exists to catch. If the message store
+		// gains a role, the builder needs updating too — fail loudly then.
+		// Optional in the shape because a `custom` message carries no role; the
+		// refine below requires it for every message that is actually rendered.
+		role: z.enum(['user', 'assistant']).optional(),
+		/** The store's own discriminator (`llm`, `custom`, …) — not enumerated. */
+		type: z.string().min(1),
+		/** Ordering before the live turn depends on this being a real timestamp. */
+		createdAt: z
+			.string()
+			.min(1)
+			.refine((v) => !Number.isNaN(Date.parse(v)), {
+				message: 'must be a parseable timestamp (e.g. an ISO 8601 string)',
+			}),
+		content: z.array(SeedMessageBlockSchema).optional(),
+	})
+	.passthrough();
+
+/** Inferred from the pre-`superRefine` shape — identical type, but resolving the
+ *  refined `ZodEffects` chain trips "type instantiation excessively deep" under
+ *  CI's type-aware lint (same reason as `EvalTestCaseInput` in schema.ts). */
+export type SeedMessage = z.infer<typeof seedMessageObjectSchema>;
+
+const SeedMessageSchema = seedMessageObjectSchema.superRefine((message, ctx) => {
+	// `custom` messages are stored but never rendered (no role, any content
+	// shape). Everything else is read by the transcript builder, which needs a
+	// role it renders and an array of blocks.
+	if (message.type === 'custom') return;
+	if (message.role === undefined) {
+		ctx.addIssue({
+			code: z.ZodIssueCode.custom,
+			path: ['role'],
+			message:
+				'is required (only `type: custom` messages may omit it — they are stored but never rendered)',
+		});
+	}
+	if (!Array.isArray(message.content)) {
+		ctx.addIssue({
+			code: z.ZodIssueCode.custom,
+			path: ['content'],
+			message:
+				'must be an array of content blocks (only `type: custom` messages may omit it — they are stored but never rendered)',
+		});
+	}
+});
+
 export const ConversationSeedSchema = z.object({
 	/** Provenance (thread id, instance, export time) — informational only. */
 	source: z.record(z.unknown()).optional(),
 	/** Native agent message log (user/assistant turns with resolved tool-call blocks). */
-	messages: z.array(z.record(z.unknown())).min(1),
+	messages: z.array(SeedMessageSchema).min(1),
 	/** Workflows the history references, recreated on restore. */
 	workflows: z.array(SeedWorkflowSchema).default([]),
 	/** Data tables the history references, recreated (and id-rewritten) on restore. */

packages/@n8n/instance-ai/evaluations/harness/langsmith-seed.ts

@@ -8,7 +8,7 @@ import { isRecord } from '@n8n/utils/is-record';
 import { Client } from 'langsmith';
 import type { Run } from 'langsmith/schemas';
 
-import type { ConversationSeed } from './conversation-seed';
+import type { ConversationSeed, SeedMessage } from './conversation-seed';
 import { parseSeedWorkflowCode } from './parse-seed-workflow';
 import { COMPILED_WORKFLOW_TRACE_RUN_NAME, DOMAIN_TOOL_IDS } from '../../src/tools/tool-ids';
 
@@ -255,18 +255,21 @@ function redactDataTableRowPayload(value: unknown): Record<string, unknown> {
 	return out;
 }
 
-interface TextBlock {
+// Type aliases, not interfaces: seed content blocks are open by design
+// (`.passthrough()`), and an interface has no index signature so it isn't
+// assignable to that. Converting these back to interfaces breaks the build.
+type TextBlock = {
 	type: 'text';
 	text: string;
-}
-interface ToolCallBlock {
+};
+type ToolCallBlock = {
 	type: 'tool-call';
 	toolCallId: string;
 	toolName: string;
 	state: 'resolved';
 	input: unknown;
 	output: unknown;
-}
+};
 
 /**
  * Reconstruct a thread's seed + live turn. The workspace holding the thread is
@@ -417,11 +420,7 @@ async function reconstructWithClient(
 }
 
 /** Rebuild the native message log for every run before the seed boundary. */
-function buildSeedMessages(
-	rootRuns: Run[],
-	toolRuns: Run[],
-	boundaryMs: number,
-): Array<Record<string, unknown>> {
+function buildSeedMessages(rootRuns: Run[], toolRuns: Run[], boundaryMs: number): SeedMessage[] {
 	const toolsByRoot = new Map<string, Run[]>();
 	for (const tool of toolRuns) {
 		const rootId = asString(metadata(tool).langsmith_root_run_id) ?? tool.trace_id ?? '';
@@ -431,7 +430,9 @@ function buildSeedMessages(
 	}
 
 	const emittedToolCallIds = new Set<string>();
-	const messages: Array<Record<string, unknown>> = [];
+	// Typed, so the compiler enforces the envelope on the machine-producer side
+	// while ConversationSeedSchema enforces it on hand-authored seeds.
+	const messages: SeedMessage[] = [];
 
 	for (const root of rootRuns) {
 		if (new Date(root.start_time ?? NaN).getTime() >= boundaryMs) break;