diff --git a/packages/@n8n/instance-ai/evaluations/__tests__/langtracer-push.test.ts b/packages/@n8n/instance-ai/evaluations/__tests__/langtracer-push.test.ts index 22acf1d5a0a..5f2c1fa71e8 100644 --- a/packages/@n8n/instance-ai/evaluations/__tests__/langtracer-push.test.ts +++ b/packages/@n8n/instance-ai/evaluations/__tests__/langtracer-push.test.ts @@ -110,11 +110,42 @@ describe('planPush', () => { expect(plan.toUpdate).toEqual([]); }); - it('ignores tags and datasets differences (the suite export does not round-trip them)', () => { + it('ignores tags differences (the suite export returns them empty)', () => { const plan = planPush( - [item('c', { tags: ['build', 'ai'], datasets: ['full'] })], - // export comes back with empty tags and null datasets — must not count as a change - { 'c.json': body({ tags: [], datasets: null }) }, + [item('c', { tags: ['build', 'ai'] })], + { 'c.json': body({ tags: [] }) }, + { c: 5 }, + ); + expect(plan.unchanged.map((c) => c.fileSlug)).toEqual(['c']); + expect(plan.toUpdate).toEqual([]); + }); + + it('treats a datasets difference as an update so tier edits re-sync', () => { + const plan = planPush( + [item('c', { datasets: ['mcp', 'pr', 'full'] })], + // the stored case lost its tiers (exported as null) — the push must restore them + { 'c.json': body({ datasets: null }) }, + { c: 5 }, + ); + expect(plan.toUpdate).toHaveLength(1); + expect(plan.toUpdate[0].id).toBe(5); + expect(plan.unchanged).toEqual([]); + }); + + it('treats the default datasets as unchanged whether the export nulls or omits it', () => { + const omitted = body(); + delete omitted.datasets; + for (const exported of [body({ datasets: null }), omitted]) { + const plan = planPush([item('c', { datasets: ['full'] })], { 'c.json': exported }, { c: 5 }); + expect(plan.unchanged.map((c) => c.fileSlug)).toEqual(['c']); + expect(plan.toUpdate).toEqual([]); + } + }); + + it('ignores datasets ordering', () => { + const plan = planPush( + [item('c', { datasets: ['pr', 'full', 'mcp'] })], + { 'c.json': body({ datasets: ['mcp', 'pr', 'full'] }) }, { c: 5 }, ); expect(plan.unchanged.map((c) => c.fileSlug)).toEqual(['c']); diff --git a/packages/@n8n/instance-ai/evaluations/langtracer/push.ts b/packages/@n8n/instance-ai/evaluations/langtracer/push.ts index c147b7c71cb..67ebbafc89e 100644 --- a/packages/@n8n/instance-ai/evaluations/langtracer/push.ts +++ b/packages/@n8n/instance-ai/evaluations/langtracer/push.ts @@ -7,6 +7,7 @@ import type { LangTracerUpdateCaseBody } from './client'; import { normalizeExportedCase } from './normalize'; import { unsupportedPushReason, type LangTracerCreateCaseBody } from './to-exported'; import type { WorkflowTestCaseWithFile } from '../data/workflows'; +import { DEFAULT_DATASETS } from '../harness/schema'; export interface PushPlan { toCreate: WorkflowTestCaseWithFile[]; @@ -15,12 +16,12 @@ export interface PushPlan { skipped: Array<{ fileSlug: string; reason: string }>; } -/** Disk fields compared to decide create-vs-update. Deliberately EXCLUDES two - * fields that would make a re-push never converge (always "update"): - * - `tags` and `datasets`: the lang-tracer suite export does not round-trip these - * (tags come back empty, default `datasets` comes back null/omitted), so a diff - * on them always fires. They're still SENT on create so new cases carry them; - * edits to only tags/tier on an existing case aren't re-synced. */ +/** Disk fields compared to decide create-vs-update. Deliberately EXCLUDES `tags`: + * the lang-tracer suite export returns them empty, so a diff on them would fire + * on every case; they're still SENT on create so new cases carry them. + * `datasets` IS compared — a tier edit must re-sync on push — but the export + * omits (or nulls) the stored default, so `projectComparable` folds the default + * to absent on both sides to keep re-pushes convergent. */ const COMPARED_KEYS = [ 'description', 'conversation', @@ -30,6 +31,7 @@ const COMPARED_KEYS = [ 'outcomeExpectations', 'messageBudget', 'credentials', + 'datasets', // Round-trips faithfully: PATCH /cases/:id reconciles scenario rows by name // (lang-tracer #48) and the export emits them back in disk shape. 'executionScenarios', @@ -104,6 +106,16 @@ function projectComparable(src: unknown): Record { // The export only emits `messageBudget` for multi-turn cases (it's ignored for // single-turn auto-approve builds), so ignore it there to stay convergent. if (key === 'messageBudget' && !isMultiTurn) continue; + // The loader defaults an absent disk `datasets` while the export omits (or + // nulls) the stored default — fold the default to absent on both sides, and + // compare order-insensitively since tiers are a set. + if (key === 'datasets') { + if (!Array.isArray(value)) continue; + const datasets = value.slice().sort(); + if (canonicalize(datasets) === canonicalize([...DEFAULT_DATASETS].sort())) continue; + out[key] = datasets; + continue; + } out[key] = value; } return out;