diff --git a/packages/@n8n/instance-ai/evaluations/README.md b/packages/@n8n/instance-ai/evaluations/README.md index a89dd573a46..60b18c8a887 100644 --- a/packages/@n8n/instance-ai/evaluations/README.md +++ b/packages/@n8n/instance-ai/evaluations/README.md @@ -818,6 +818,8 @@ For a **synthetic, sanitized** seed you want pinned in git (never a real user's Schema in `harness/conversation-seed.ts` — `messages` plus optional `workflows` and `dataTables`. Two constraints worth knowing: a workflow `id` must be ≥8 characters (`remapSeedWorkflowIds` refuses to rewrite shorter ids safely), and a seeded `build-workflow` tool call's `output.workflowId` must match the seeded workflow's `id`, or the remap separates them and the agent can't find the workflow it's meant to act on. +**Each message must carry the envelope** — `id`, `role` (`user` or `assistant`), `type` (`llm`, `custom`, …), `createdAt` (a parseable timestamp; ordering before the live turn depends on it), and `content` as an array of blocks each with a `type`. Only the envelope is validated: **unknown block types are accepted**, because block shapes belong to the agent's message store rather than to the harness, and unknown keys are preserved rather than stripped. A `type: 'custom'` message is the one exception — it's stored but never rendered, so it may omit `role` and carry any `content` shape. The envelope is checked because a malformed message would otherwise be stored verbatim *and* skipped by `transcriptPrefixFromSeed`, leaving the case graded against a transcript that doesn't match what the agent saw. + The seed lives **in the case body** rather than in a sibling file, so it travels with the case whatever the source — a JSON on disk, a suite pulled with `--source langtracer`, or a case body handed to a dispatcher. (There used to be a `seedFile` path pointing at a sibling JSON. Only the disk loader could resolve it, so a case delivered any other way lost its seed; the key is gone and a case still carrying it fails at load.) #### How restore works (all paths) @@ -871,7 +873,7 @@ evaluations/ ├── checklist/ # LLM verification with retry ├── credentials/ # Test credential seeding ├── data/agents/ # authoring dir for intent-resolution cases (the corpus lives in LangTracer suite `agents`) -├── data/workflows/ # seeded carve-out case JSONs + seeds/ (the corpus lives in LangTracer) +├── data/workflows/ # seeded carve-out case JSONs (the corpus lives in LangTracer) ├── data/subagent/ # workflow-build compatibility fixture JSON files ├── data/pairwise/ # Local pairwise fixture (small smoke set) ├── harness/ # Runners: buildWorkflow + executeScenario (e2e), in-memory event bus (discovery) diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/conversation-seed.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/conversation-seed.test.ts index bb5d92b5d64..ef1766ecf4a 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/conversation-seed.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/conversation-seed.test.ts @@ -1,4 +1,5 @@ import { + ConversationSeedSchema, remapSeedWorkflowIds, seedFromProse, transcriptPrefixFromSeed, @@ -41,6 +42,114 @@ function makeSeed(): ConversationSeed { }; } +describe('ConversationSeedSchema message envelope', () => { + const message = (over: Record = {}) => ({ + id: 'm1', + role: 'user', + type: 'llm', + createdAt: '2026-01-01T00:00:00.000Z', + content: [{ type: 'text', text: 'build it' }], + ...over, + }); + const parse = (...messages: Array>) => + ConversationSeedSchema.safeParse({ messages }); + const errorOf = (result: ReturnType) => + result.success ? '' : JSON.stringify(result.error.issues); + + it('accepts a well-formed message', () => { + expect(parse(message()).success).toBe(true); + }); + + for (const field of ['id', 'role', 'type', 'createdAt', 'content'] as const) { + it(`rejects a message missing ${field}`, () => { + const broken = message(); + delete broken[field]; + const result = parse(broken); + expect(result.success).toBe(false); + expect(errorOf(result)).toContain(field); + }); + } + + it('rejects a role the transcript builder would silently drop', () => { + // The typo'd-role case: stored verbatim, then skipped by + // transcriptPrefixFromSeed, so the case grades a transcript the agent never saw. + const result = parse(message({ role: 'assistent' })); + expect(result.success).toBe(false); + expect(errorOf(result)).toContain('role'); + }); + + it('rejects a createdAt that is not a real timestamp', () => { + // Ordering before the live turn depends on this parsing. + const result = parse(message({ createdAt: 'yesterday' })); + expect(result.success).toBe(false); + expect(errorOf(result)).toContain('createdAt'); + }); + + it('rejects content that is not an array of blocks', () => { + expect(parse(message({ content: 'build it' })).success).toBe(false); + expect(parse(message({ content: [{ text: 'no type' }] })).success).toBe(false); + }); + + it('accepts an unknown block type — block shapes are the message store’s contract', () => { + expect(parse(message({ content: [{ type: 'some-future-block', payload: {} }] })).success).toBe( + true, + ); + }); + + it('accepts a custom message with no role and non-array content (stored, never rendered)', () => { + const result = parse( + { id: 'c1', type: 'custom', data: { widget: 'card' }, createdAt: '2026-01-01T00:00:00Z' }, + message(), + ); + expect(result.success).toBe(true); + }); + + it('preserves unknown keys on messages and blocks rather than stripping them', () => { + // The load-bearing one: z.object strips unknown keys by default, which would + // silently gut toolCallId/input/output from every tool call before restore. + const result = ConversationSeedSchema.safeParse({ + messages: [ + message({ + messageGroupId: 'mg-1', + content: [ + { + type: 'tool-call', + toolCallId: 'tc-1', + toolName: 'build-workflow', + state: 'resolved', + input: { name: 'Digest' }, + output: { success: true, workflowId: 'AbCdEf1234567890' }, + }, + ], + }), + ], + }); + expect(result.success).toBe(true); + if (!result.success) return; + const [only] = result.data.messages; + expect(only.messageGroupId).toBe('mg-1'); + expect(only.content?.[0]).toMatchObject({ + toolCallId: 'tc-1', + toolName: 'build-workflow', + state: 'resolved', + input: { name: 'Digest' }, + output: { success: true, workflowId: 'AbCdEf1234567890' }, + }); + }); + + it('accepts what seedFromProse produces', () => { + const seed = seedFromProse([ + { role: 'user', text: 'hi' }, + { role: 'assistant', text: 'hello' }, + ]); + expect(ConversationSeedSchema.safeParse(seed).success).toBe(true); + }); + + it('still requires at least one message', () => { + expect(ConversationSeedSchema.safeParse({ messages: [] }).success).toBe(false); + }); +}); + describe('seedFromProse', () => { it('converts turns to llm text messages with ascending past timestamps', () => { const seed = seedFromProse([ diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/data-workflows-schema.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/data-workflows-schema.test.ts index 6167bad27ec..0aadeff1969 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/data-workflows-schema.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/data-workflows-schema.test.ts @@ -101,7 +101,13 @@ describe('EvalTestCaseSchema', () => { ...validFixture(), conversationSeed: { messages: [ - { id: 'm1', type: 'llm', role: 'user', content: [{ type: 'text', text: 'build it' }] }, + { + id: 'm1', + type: 'llm', + role: 'user', + createdAt: '2026-06-29T09:00:00.000Z', + content: [{ type: 'text', text: 'build it' }], + }, ], }, }); @@ -122,7 +128,13 @@ describe('EvalTestCaseSchema', () => { ...validFixture(), conversationSeed: { messages: [ - { id: 'm1', type: 'llm', role: 'user', content: [{ type: 'text', text: 'build it' }] }, + { + id: 'm1', + type: 'llm', + role: 'user', + createdAt: '2026-06-29T09:00:00.000Z', + content: [{ type: 'text', text: 'build it' }], + }, ], }, priorConversation: [{ role: 'user', text: 'prelude' }], diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/langsmith-seed.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/langsmith-seed.test.ts index d05dfbf8610..c61628738c9 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/langsmith-seed.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/langsmith-seed.test.ts @@ -106,7 +106,8 @@ describe('reconstructSeedFromThread', () => { // Only turn 1 is seeded — the pinned turn and everything after it are excluded. const userTexts = result.seed.messages .filter((m) => m.role === 'user') - .map((m) => (m.content as Array<{ text: string }>)[0].text); + // Block fields are `unknown` by design (the store owns block shapes). + .map((m) => (m.content?.[0] as { text: string } | undefined)?.text ?? ''); expect(userTexts).toEqual(['Build Otter Digest, daily 9am']); }); diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/langtracer-provider.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/langtracer-provider.test.ts index 01841858490..ad45a51edcd 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/langtracer-provider.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/langtracer-provider.test.ts @@ -39,6 +39,7 @@ describe('casesFromExportedFiles', () => { id: 'm1', type: 'llm', role: 'user', + createdAt: '2026-06-29T09:00:00.000Z', content: [{ type: 'text', text: 'build it' }], }, ], diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/langtracer-to-exported.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/langtracer-to-exported.test.ts index b5af4d83913..46b6346cf42 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/langtracer-to-exported.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/langtracer-to-exported.test.ts @@ -129,6 +129,7 @@ describe('unsupportedPushReason', () => { id: 'm1', type: 'llm', role: 'user', + createdAt: '2026-06-29T09:00:00.000Z', content: [{ type: 'text', text: 'build it' }], }, ], diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/mcp-builder.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/mcp-builder.test.ts index 4b88650da04..9e4b4a19dec 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/mcp-builder.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/mcp-builder.test.ts @@ -140,7 +140,13 @@ describe('unsupportedMcpBuildSetupFields', () => { { conversationSeed: { messages: [ - { id: 'm1', type: 'llm', role: 'user', content: [{ type: 'text', text: 'build it' }] }, + { + id: 'm1', + type: 'llm', + role: 'user', + createdAt: '2026-06-29T09:00:00.000Z', + content: [{ type: 'text', text: 'build it' }], + }, ], workflows: [], dataTables: [], diff --git a/packages/@n8n/instance-ai/evaluations/harness/conversation-seed.ts b/packages/@n8n/instance-ai/evaluations/harness/conversation-seed.ts index a9ba9b76e90..020b43ec51c 100644 --- a/packages/@n8n/instance-ai/evaluations/harness/conversation-seed.ts +++ b/packages/@n8n/instance-ai/evaluations/harness/conversation-seed.ts @@ -43,11 +43,77 @@ const SeedDataTableSchema = z.object({ // of a trace and are kept out of the eval instance entirely. }); +/** A content block. Only `type` is ours to require — the block shapes belong to + * the agent's message store, so an unrecognised one is accepted and simply not + * interpreted. `.passthrough()` is load-bearing: `z.object` strips unknown keys, + * which would silently gut `toolCallId`/`input`/`output` from every tool call. */ +const SeedMessageBlockSchema = z + .object({ + type: z.string().min(1, 'a content block needs a non-empty `type`'), + }) + .passthrough(); + +/** A message envelope. Validated because a hand-authored seed is now the primary + * path, and a malformed message is stored verbatim AND skipped by + * `transcriptPrefixFromSeed` — so the case grades against a transcript that + * doesn't match what the agent actually saw. Envelope only; block internals are + * the store's contract, not ours. */ +const seedMessageObjectSchema = z + .object({ + id: z.string().min(1), + // Restricted to the two roles the transcript builder renders: any other + // value is guaranteed to vanish from the judge transcript, which is the + // exact silent failure this schema exists to catch. If the message store + // gains a role, the builder needs updating too — fail loudly then. + // Optional in the shape because a `custom` message carries no role; the + // refine below requires it for every message that is actually rendered. + role: z.enum(['user', 'assistant']).optional(), + /** The store's own discriminator (`llm`, `custom`, …) — not enumerated. */ + type: z.string().min(1), + /** Ordering before the live turn depends on this being a real timestamp. */ + createdAt: z + .string() + .min(1) + .refine((v) => !Number.isNaN(Date.parse(v)), { + message: 'must be a parseable timestamp (e.g. an ISO 8601 string)', + }), + content: z.array(SeedMessageBlockSchema).optional(), + }) + .passthrough(); + +/** Inferred from the pre-`superRefine` shape — identical type, but resolving the + * refined `ZodEffects` chain trips "type instantiation excessively deep" under + * CI's type-aware lint (same reason as `EvalTestCaseInput` in schema.ts). */ +export type SeedMessage = z.infer; + +const SeedMessageSchema = seedMessageObjectSchema.superRefine((message, ctx) => { + // `custom` messages are stored but never rendered (no role, any content + // shape). Everything else is read by the transcript builder, which needs a + // role it renders and an array of blocks. + if (message.type === 'custom') return; + if (message.role === undefined) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['role'], + message: + 'is required (only `type: custom` messages may omit it — they are stored but never rendered)', + }); + } + if (!Array.isArray(message.content)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: ['content'], + message: + 'must be an array of content blocks (only `type: custom` messages may omit it — they are stored but never rendered)', + }); + } +}); + export const ConversationSeedSchema = z.object({ /** Provenance (thread id, instance, export time) — informational only. */ source: z.record(z.unknown()).optional(), /** Native agent message log (user/assistant turns with resolved tool-call blocks). */ - messages: z.array(z.record(z.unknown())).min(1), + messages: z.array(SeedMessageSchema).min(1), /** Workflows the history references, recreated on restore. */ workflows: z.array(SeedWorkflowSchema).default([]), /** Data tables the history references, recreated (and id-rewritten) on restore. */ diff --git a/packages/@n8n/instance-ai/evaluations/harness/langsmith-seed.ts b/packages/@n8n/instance-ai/evaluations/harness/langsmith-seed.ts index fcec5823499..3a168b6a6aa 100644 --- a/packages/@n8n/instance-ai/evaluations/harness/langsmith-seed.ts +++ b/packages/@n8n/instance-ai/evaluations/harness/langsmith-seed.ts @@ -8,7 +8,7 @@ import { isRecord } from '@n8n/utils/is-record'; import { Client } from 'langsmith'; import type { Run } from 'langsmith/schemas'; -import type { ConversationSeed } from './conversation-seed'; +import type { ConversationSeed, SeedMessage } from './conversation-seed'; import { parseSeedWorkflowCode } from './parse-seed-workflow'; import { COMPILED_WORKFLOW_TRACE_RUN_NAME, DOMAIN_TOOL_IDS } from '../../src/tools/tool-ids'; @@ -255,18 +255,21 @@ function redactDataTableRowPayload(value: unknown): Record { return out; } -interface TextBlock { +// Type aliases, not interfaces: seed content blocks are open by design +// (`.passthrough()`), and an interface has no index signature so it isn't +// assignable to that. Converting these back to interfaces breaks the build. +type TextBlock = { type: 'text'; text: string; -} -interface ToolCallBlock { +}; +type ToolCallBlock = { type: 'tool-call'; toolCallId: string; toolName: string; state: 'resolved'; input: unknown; output: unknown; -} +}; /** * Reconstruct a thread's seed + live turn. The workspace holding the thread is @@ -417,11 +420,7 @@ async function reconstructWithClient( } /** Rebuild the native message log for every run before the seed boundary. */ -function buildSeedMessages( - rootRuns: Run[], - toolRuns: Run[], - boundaryMs: number, -): Array> { +function buildSeedMessages(rootRuns: Run[], toolRuns: Run[], boundaryMs: number): SeedMessage[] { const toolsByRoot = new Map(); for (const tool of toolRuns) { const rootId = asString(metadata(tool).langsmith_root_run_id) ?? tool.trace_id ?? ''; @@ -431,7 +430,9 @@ function buildSeedMessages( } const emittedToolCallIds = new Set(); - const messages: Array> = []; + // Typed, so the compiler enforces the envelope on the machine-producer side + // while ConversationSeedSchema enforces it on hand-authored seeds. + const messages: SeedMessage[] = []; for (const root of rootRuns) { if (new Date(root.start_time ?? NaN).getTime() >= boundaryMs) break;