feat(core): Extend evaluations to support node groups (no-changelog) (#35629)

Co-authored-by: Miguel Ángel Moreno <1411774+miguelsaddress@users.noreply.github.com>

.env.eval.example

@@ -15,6 +15,11 @@ N8N_LOG_LEVEL=debug
 N8N_AI_ENABLED=true
 N8N_ENABLED_MODULES=instance-ai
 
+# Uncomment when calibrating MCP grouping cases (--build-via-mcp): canvas-group
+# support in the MCP tools is PostHog-gated and PostHog is unreachable locally
+# (fails closed), so the env var is the only lever. Mirrors test-evals-mcp.yml.
+# N8N_MCP_CANVAS_GROUPS_ENABLED=true
+
 # Model key for the builder and the eval helper (mock generation, verifier,
 # user-proxy, expectation judge). When the instance runs without the proxy, the
 # model client reads ANTHROPIC_API_KEY directly, so export that too if you go

.github/workflows/test-evals-mcp.yml

@@ -192,11 +192,14 @@ jobs:
           # agents registers the agent-builder MCP tools so the suite measures
           # workflow building with the production tool surface, catching
           # tool-choice regressions the extra tools could introduce.
+          # MCP canvas-group support is PostHog-gated and PostHog is unreachable
+          # here (fails closed), so the env var force-enables it;
           for i in "${!PORTS[@]}"; do
             port="${PORTS[$i]}"
             docker run -d --name "n8n-eval-mcp-$((i + 1))" \
               -e E2E_TESTS=true \
               -e N8N_ENABLED_MODULES=instance-ai,agents \
+              -e N8N_MCP_CANVAS_GROUPS_ENABLED=true \
               -e N8N_AI_ENABLED=true \
               -e N8N_INSTANCE_AI_MODEL_API_KEY="$EVALS_ANTHROPIC_KEY" \
               -e N8N_AI_ASSISTANT_BASE_URL="" \

packages/@n8n/instance-ai/evaluations/__tests__/fixtures.ts

@@ -41,7 +41,7 @@ export function assistantMessage(agentTree: InstanceAiAgentNode): InstanceAiMess
 	};
 }
 
-export function workflow(id: string): WorkflowResponse {
+export function workflow(id: string, overrides: Partial<WorkflowResponse> = {}): WorkflowResponse {
 	return {
 		id,
 		name: `Workflow ${id}`,
@@ -49,6 +49,7 @@ export function workflow(id: string): WorkflowResponse {
 		versionId: `version-${id}`,
 		nodes: [],
 		connections: {},
+		...overrides,
 	};
 }
 

packages/@n8n/instance-ai/evaluations/__tests__/workflow-context.test.ts

@@ -0,0 +1,82 @@
+import { workflow } from './fixtures';
+import type { WorkflowNodeResponse } from '../clients/n8n-client';
+import { buildWorkflowContextBlock } from '../harness/workflow-context';
+
+const node = (id: string, name: string): WorkflowNodeResponse => ({
+	id,
+	name,
+	type: 'n8n-nodes-base.set',
+});
+
+/** The rendered groups JSON, exactly as the block should print it. */
+const groupsJson = (groups: Array<Record<string, unknown>>): string =>
+	['**Node groups:**', '```json', JSON.stringify(groups, null, 2), '```'].join('\n');
+
+describe('buildWorkflowContextBlock', () => {
+	it('renders "(no workflow built)" without a workflow', () => {
+		expect(buildWorkflowContextBlock(undefined)).toBe(
+			'## Workflow structure\n\n(no workflow built)',
+		);
+	});
+
+	it('renders group members by node name, not id', () => {
+		const wf = workflow('wf-1', {
+			nodes: [node('id-a', 'Fetch Data'), node('id-b', 'Parse CSV'), node('id-c', 'Send Email')],
+			nodeGroups: [
+				{ id: 'g-1', name: 'Ingestion', nodeIds: ['id-a', 'id-b'] },
+				{ id: 'g-2', name: 'Notify', nodeIds: ['id-c'] },
+			],
+		});
+
+		const block = buildWorkflowContextBlock(wf);
+
+		expect(block).toContain(
+			groupsJson([
+				{ name: 'Ingestion', nodes: ['Fetch Data', 'Parse CSV'] },
+				{ name: 'Notify', nodes: ['Send Email'] },
+			]),
+		);
+		// The judge context is name-keyed throughout — ids must not leak in.
+		expect(block).not.toContain('id-a');
+		expect(block).not.toContain('g-1');
+	});
+
+	it('drops member ids that no longer resolve to a node', () => {
+		const wf = workflow('wf-1', {
+			nodes: [node('id-a', 'Fetch Data')],
+			nodeGroups: [{ id: 'g-1', name: 'Ingestion', nodeIds: ['id-a', 'id-ghost'] }],
+		});
+
+		expect(buildWorkflowContextBlock(wf)).toContain(
+			groupsJson([{ name: 'Ingestion', nodes: ['Fetch Data'] }]),
+		);
+	});
+
+	it('includes the group description only when present', () => {
+		const wf = workflow('wf-1', {
+			nodes: [node('id-a', 'Fetch Data'), node('id-b', 'Send Email')],
+			nodeGroups: [
+				{ id: 'g-1', name: 'Ingestion', nodeIds: ['id-a'], description: 'Pulls the raw CSV' },
+				{ id: 'g-2', name: 'Notify', nodeIds: ['id-b'] },
+			],
+		});
+
+		expect(buildWorkflowContextBlock(wf)).toContain(
+			groupsJson([
+				{ name: 'Ingestion', nodes: ['Fetch Data'], description: 'Pulls the raw CSV' },
+				{ name: 'Notify', nodes: ['Send Email'] },
+			]),
+		);
+	});
+
+	it('states "(none)" when the workflow has no groups', () => {
+		// Absent field (REST omits it) and empty array must both read as "no groups".
+		const withoutField = workflow('wf-1', { nodes: [node('id-a', 'Fetch Data')] });
+		const withEmpty = workflow('wf-2', { nodes: [node('id-a', 'Fetch Data')], nodeGroups: [] });
+
+		for (const wf of [withoutField, withEmpty]) {
+			const block = buildWorkflowContextBlock(wf);
+			expect(block).toContain('**Node groups:**\n\n(none)');
+		}
+	});
+});

packages/@n8n/instance-ai/evaluations/clients/n8n-client.ts

@@ -149,6 +149,14 @@ export interface WorkflowNodeResponse {
 	credentials?: Record<string, unknown>;
 }
 
+/** A canvas node group as returned by the n8n REST API — members are node *ids*. */
+export interface WorkflowNodeGroupResponse {
+	id: string;
+	name: string;
+	nodeIds: string[];
+	description?: string;
+}
+
 /** A workflow as returned by GET /rest/workflows/:id. */
 export interface WorkflowResponse {
 	id: string;
@@ -158,6 +166,7 @@ export interface WorkflowResponse {
 	description?: string;
 	nodes: WorkflowNodeResponse[];
 	connections: Record<string, unknown>;
+	nodeGroups?: WorkflowNodeGroupResponse[];
 	pinData?: Record<string, unknown>;
 }
 

packages/@n8n/instance-ai/evaluations/harness/workflow-context.ts

@@ -1,6 +1,38 @@
 import type { WorkflowResponse } from '../clients/n8n-client';
 
-/** Render the per-build workflow structure: nodes, connections, all configs. */
+/**
+ * Renders node groups for the judge. Groups persist member node *ids*, but the
+ * judge context is name-keyed and never exposes ids — so members are mapped to
+ * node names and stale ids are dropped, mirroring the MCP read path
+ * (`toNodeGroupSummary` in packages/cli/src/modules/mcp/tools/schemas.ts).
+ */
+function renderNodeGroupLines(wf: WorkflowResponse): string[] {
+	const groups = wf.nodeGroups ?? [];
+	if (groups.length === 0) {
+		// Stated absence, not omission — a negative assertion ("the nodes are not
+		// grouped") needs the judge to see that no groups exist.
+		return ['**Node groups:**', '', '(none)'];
+	}
+	const nameById = new Map(
+		wf.nodes.flatMap((node) => (node.id === undefined ? [] : [[node.id, node.name] as const])),
+	);
+	return [
+		'**Node groups:**',
+		'```json',
+		JSON.stringify(
+			groups.map((group) => ({
+				name: group.name,
+				nodes: group.nodeIds.flatMap((nodeId) => nameById.get(nodeId) ?? []),
+				...(group.description !== undefined ? { description: group.description } : {}),
+			})),
+			null,
+			2,
+		),
+		'```',
+	];
+}
+
+/** Render the per-build workflow structure: nodes, connections, all configs, node groups. */
 export function buildWorkflowContextBlock(wf: WorkflowResponse | undefined): string {
 	if (!wf) return '## Workflow structure\n\n(no workflow built)';
 	const lines: string[] = ['## Workflow structure', ''];
@@ -35,6 +67,7 @@ export function buildWorkflowContextBlock(wf: WorkflowResponse | undefined): str
 		'',
 	);
 	lines.push('**Connections:**');
-	lines.push('```json', JSON.stringify(wf.connections, null, 2), '```');
+	lines.push('```json', JSON.stringify(wf.connections, null, 2), '```', '');
+	lines.push(...renderNodeGroupLines(wf));
 	return lines.join('\n');
 }