fix(ai-builder): Detect datasets changes when re-pushing eval cases to LangTracer (no-changelog) (#34658)

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

packages/@n8n/instance-ai/evaluations/__tests__/langtracer-push.test.ts

@@ -110,11 +110,42 @@ describe('planPush', () => {
 		expect(plan.toUpdate).toEqual([]);
 	});
 
-	it('ignores tags and datasets differences (the suite export does not round-trip them)', () => {
+	it('ignores tags differences (the suite export returns them empty)', () => {
 		const plan = planPush(
-			[item('c', { tags: ['build', 'ai'], datasets: ['full'] })],
-			// export comes back with empty tags and null datasets — must not count as a change
-			{ 'c.json': body({ tags: [], datasets: null }) },
+			[item('c', { tags: ['build', 'ai'] })],
+			{ 'c.json': body({ tags: [] }) },
+			{ c: 5 },
+		);
+		expect(plan.unchanged.map((c) => c.fileSlug)).toEqual(['c']);
+		expect(plan.toUpdate).toEqual([]);
+	});
+
+	it('treats a datasets difference as an update so tier edits re-sync', () => {
+		const plan = planPush(
+			[item('c', { datasets: ['mcp', 'pr', 'full'] })],
+			// the stored case lost its tiers (exported as null) — the push must restore them
+			{ 'c.json': body({ datasets: null }) },
+			{ c: 5 },
+		);
+		expect(plan.toUpdate).toHaveLength(1);
+		expect(plan.toUpdate[0].id).toBe(5);
+		expect(plan.unchanged).toEqual([]);
+	});
+
+	it('treats the default datasets as unchanged whether the export nulls or omits it', () => {
+		const omitted = body();
+		delete omitted.datasets;
+		for (const exported of [body({ datasets: null }), omitted]) {
+			const plan = planPush([item('c', { datasets: ['full'] })], { 'c.json': exported }, { c: 5 });
+			expect(plan.unchanged.map((c) => c.fileSlug)).toEqual(['c']);
+			expect(plan.toUpdate).toEqual([]);
+		}
+	});
+
+	it('ignores datasets ordering', () => {
+		const plan = planPush(
+			[item('c', { datasets: ['pr', 'full', 'mcp'] })],
+			{ 'c.json': body({ datasets: ['mcp', 'pr', 'full'] }) },
 			{ c: 5 },
 		);
 		expect(plan.unchanged.map((c) => c.fileSlug)).toEqual(['c']);

packages/@n8n/instance-ai/evaluations/langtracer/push.ts

@@ -7,6 +7,7 @@ import type { LangTracerUpdateCaseBody } from './client';
 import { normalizeExportedCase } from './normalize';
 import { unsupportedPushReason, type LangTracerCreateCaseBody } from './to-exported';
 import type { WorkflowTestCaseWithFile } from '../data/workflows';
+import { DEFAULT_DATASETS } from '../harness/schema';
 
 export interface PushPlan {
 	toCreate: WorkflowTestCaseWithFile[];
@@ -15,12 +16,12 @@ export interface PushPlan {
 	skipped: Array<{ fileSlug: string; reason: string }>;
 }
 
-/** Disk fields compared to decide create-vs-update. Deliberately EXCLUDES two
- *  fields that would make a re-push never converge (always "update"):
- *  - `tags` and `datasets`: the lang-tracer suite export does not round-trip these
- *    (tags come back empty, default `datasets` comes back null/omitted), so a diff
- *    on them always fires. They're still SENT on create so new cases carry them;
- *    edits to only tags/tier on an existing case aren't re-synced. */
+/** Disk fields compared to decide create-vs-update. Deliberately EXCLUDES `tags`:
+ *  the lang-tracer suite export returns them empty, so a diff on them would fire
+ *  on every case; they're still SENT on create so new cases carry them.
+ *  `datasets` IS compared — a tier edit must re-sync on push — but the export
+ *  omits (or nulls) the stored default, so `projectComparable` folds the default
+ *  to absent on both sides to keep re-pushes convergent. */
 const COMPARED_KEYS = [
 	'description',
 	'conversation',
@@ -30,6 +31,7 @@ const COMPARED_KEYS = [
 	'outcomeExpectations',
 	'messageBudget',
 	'credentials',
+	'datasets',
 	// Round-trips faithfully: PATCH /cases/:id reconciles scenario rows by name
 	// (lang-tracer #48) and the export emits them back in disk shape.
 	'executionScenarios',
@@ -104,6 +106,16 @@ function projectComparable(src: unknown): Record<string, unknown> {
 		// The export only emits `messageBudget` for multi-turn cases (it's ignored for
 		// single-turn auto-approve builds), so ignore it there to stay convergent.
 		if (key === 'messageBudget' && !isMultiTurn) continue;
+		// The loader defaults an absent disk `datasets` while the export omits (or
+		// nulls) the stored default — fold the default to absent on both sides, and
+		// compare order-insensitively since tiers are a set.
+		if (key === 'datasets') {
+			if (!Array.isArray(value)) continue;
+			const datasets = value.slice().sort();
+			if (canonicalize(datasets) === canonicalize([...DEFAULT_DATASETS].sort())) continue;
+			out[key] = datasets;
+			continue;
+		}
 		out[key] = value;
 	}
 	return out;