feat(ai-builder): Push agent intent eval cases from data/agents and drop synced case files (no-changelog) (#34957)

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
2026-09-01 15:47:41 +08:00 · 2026-07-28 08:25:30 +01:00
parent ac0cd7bc3f
commit 66408a0cd5
46 changed files with 29 additions and 1112 deletions
@@ -7,7 +7,7 @@ Tests whether workflows built by Instance AI actually work by executing them wit
 Five harnesses live here:

 - **`eval:instance-ai`** — end-to-end build + mocked execution + LLM verification (drives a running n8n instance)
- **`eval:agents`** — intent-resolution cases (plain build requests graded on enacted routing behavior) from `data/agents/`
+- **`eval:agents`** — intent-resolution cases (plain build requests graded on enacted routing behavior) from the LangTracer `agents` suite (author new ones in `data/agents/`)
 - **`eval:subagent`** — legacy command name for the workflow-build compatibility corpus; it drives the live orchestrator/skill build path, scored by binary checks
 - **`eval:discovery`** — orchestrator in-process, scored against required or forbidden tool/dispatch events (no n8n server)
 - **`eval:pairwise`** — live orchestrator workflow builds, scored by an LLM judge panel against do/don't lists. Intended for head-to-head comparison with `ai-workflow-builder.ee` on the same dataset
@@ -181,7 +181,7 @@ A case can belong to multiple groupings — e.g. PR-tier cases declare `"dataset

 **LangTracer is the source of truth for the workflow-eval corpus** — the `baseline` suite holds the cases, and CI pulls it on every run (see `.github/workflows/test-evals-instance-ai.yml`). The two `--source` modes split the work:

- **`disk` (the default) — the preferred mode for local development.** Reads `data/workflows/` and `data/agents/`. Use it while authoring and calibrating a case: drop the JSON in, `--filter` it, iterate. It is also the only home of the `agents` tier and the seeded carve-out cases. Since the corpus migration the directory holds only those, so disk mode is about the case in front of you, not the full suite.
+- **`disk` (the default) — the preferred mode for local development.** Reads `data/workflows/` and `data/agents/`. Use it while authoring and calibrating a case: drop the JSON in, `--filter` it, iterate. It is also the only home of the seeded carve-out cases (the case-write API can't represent them yet). Everything else — including the agents-team cases (suite `agents`: agent-artifact + intent-resolution) — lives in LangTracer, so disk mode is about the case in front of you, not the full suite.
 - **`langtracer` — for bigger runs, already-pushed cases, and CI.** Pulls a suite from [LangTracer](https://github.com/n8n-io/lang-tracer)'s REST API (`GET /api/v1/suites/:id/export`), validated through the same `EvalTestCaseSchema`. Reach for it locally when you want the real corpus (a full or tier run) or to re-run a specific case that already lives in the suite; CI always runs this way.

 Set these in `.env.local`:
@@ -859,7 +859,7 @@ evaluations/
 ├── clients/              # n8n REST + SSE clients
 ├── checklist/            # LLM verification with retry
 ├── credentials/          # Test credential seeding
-├── data/agents/          # user-intent / agent-building eval case JSON files
+├── data/agents/          # authoring dir for intent-resolution cases (the corpus lives in LangTracer suite `agents`)
 ├── data/workflows/       # seeded carve-out case JSONs + seeds/ (the corpus lives in LangTracer)
 ├── data/subagent/        # workflow-build compatibility fixture JSON files
 ├── data/pairwise/        # Local pairwise fixture (small smoke set)
@@ -1,4 +1,4 @@
-// Push selected on-disk eval cases (data/workflows/*.json) UP into a
+// Push selected on-disk eval cases (data/workflows/ + data/agents/ *.json) UP into a
 // lang-tracer suite over the REST API, upserting: create missing, update changed,
 // leave unchanged, skip unsupported. The inverse of `--source langtracer` (which
 // pulls a suite down). Env: LANGTRACER_URL + LANGTRACER_API_KEY (repo-root .env.local).
@@ -10,6 +10,7 @@
 import { execFileSync } from 'node:child_process';
 import { basename } from 'node:path';

+import { loadAgentEvalTestCasesWithFiles } from '../data/agents';
 import { loadWorkflowTestCasesWithFiles } from '../data/workflows';
 import { LangTracerClient } from '../langtracer/client';
 import { resolveLangTracerConfig } from '../langtracer/config';
@@ -35,7 +36,7 @@ Usage:

 Selectors (at least one required — no accidental push-all):
  <slugs...>            Exact file slugs to push (e.g. ai-quote-carousel)
-  --changed             New/untracked + staged + modified data/workflows/*.json
+  --changed             New/untracked + staged + modified data/{workflows,agents}/*.json
  --filter <csv>        Substring match on file slug
  --tier <name>         Cases whose datasets include <name>
  --exclude <csv>       Substring exclude (modifier, not a selector on its own)
@@ -134,7 +135,7 @@ function nextArg(argv: string[], i: number, flag: string): string {
 	return value;
 }

-/** New/untracked + staged + modified `data/workflows/*.json` slugs, from git. */
+/** New/untracked + staged + modified `data/{workflows,agents}/*.json` slugs, from git. */
 function gitChangedSlugs(): string[] {
 	const out = execFileSync('git', ['status', '--porcelain', '--untracked-files=all'], {
 		encoding: 'utf-8',
@@ -144,7 +145,10 @@ function gitChangedSlugs(): string[] {
 		if (!line.trim()) continue;
 		const raw = line.slice(3).trim(); // strip the 2-char status + space
 		const path = raw.includes(' -> ') ? raw.split(' -> ')[1] : raw; // rename → new path
-		if (path.includes('evaluations/data/workflows/') && path.endsWith('.json')) {
+		if (
+			(path.includes('evaluations/data/workflows/') || path.includes('evaluations/data/agents/')) &&
+			path.endsWith('.json')
+		) {
 			slugs.push(basename(path, '.json'));
 		}
 	}
@@ -174,7 +178,16 @@ async function main() {
 	// Select disk cases: loader applies --filter/--exclude, --tier narrows by the
 	// case's datasets (mirrors data/source.ts); then narrow to the exact slugs
 	// from positional args + --changed (if either was given).
-	const loaded = loadWorkflowTestCasesWithFiles(args.filter, args.exclude);
+	const loaded = [
+		...loadWorkflowTestCasesWithFiles(args.filter, args.exclude),
+		...loadAgentEvalTestCasesWithFiles(args.filter, args.exclude),
+	];
+	const dupes = loaded.filter((c, i) => loaded.findIndex((o) => o.fileSlug === c.fileSlug) !== i);
+	if (dupes.length > 0) {
+		throw new Error(
+			`duplicate case slug(s) across data/workflows and data/agents: ${dupes.map((d) => d.fileSlug).join(', ')}`,
+		);
+	}
 	const tier = args.tier;
 	const all = tier ? loaded.filter((c) => c.testCase.datasets.includes(tier)) : loaded;
 	const exactSlugs = new Set([...args.slugs, ...(args.changed ? gitChangedSlugs() : [])]);
@@ -182,7 +195,7 @@ async function main() {

 	const missing = [...exactSlugs].filter((s) => !all.some((c) => c.fileSlug === s));
 	if (missing.length > 0) {
-		console.warn(`⚠ no data/workflows case file for: ${missing.join(', ')}`);
+		console.warn(`⚠ no data/workflows or data/agents case file for: ${missing.join(', ')}`);
 	}
 	if (selected.length === 0) {
 		console.log('No cases selected — nothing to push.');
@@ -1,26 +0,0 @@
-{
-	"description": "Explicit agent request with chat-shaped Q&A. Gold: {anchor: agent, embedsOther: false} — the deliverable is an n8n Agent artifact, not a workflow with a Chat Trigger + AI Agent node (the historical misroute this case pins).",
-	"conversation": [
-		{
-			"role": "user",
-			"text": "Build me an agent that answers customer questions from our docs. Don't build anything yet — first walk me through how you'd set this up."
-		},
-		{
-			"role": "user",
-			"text": [
-				"[Do not approve any plan, setup card, or build confirmation.",
-				"If the assistant asks a clarifying question, give a minimal plausible answer.",
-				"Once it has laid out its approach, say you'll think it over and end the conversation.]"
-			]
-		}
-	],
-	"complexity": "medium",
-	"tags": ["intent-resolution", "explicit-artifact", "chat"],
-	"datasets": ["agents"],
-	"processExpectations": [
-		"The agent proposes creating an n8n Agent as the deliverable for the docs Q&A assistant.",
-		"The proposal does NOT design a workflow with a Chat Trigger and an AI Agent node as the top-level shape.",
-		"The proposal grounds the agent choice in the explicit agent request and the chat-shaped, open-ended Q&A — not in generic preference."
-	],
-	"messageBudget": 4
-}
@@ -1,26 +0,0 @@
-{
-	"description": "Gold: {anchor: wf, embedsOther: false} — seven integrations, zero branches; tool count alone must not produce an agent. Also probes the node-choice bias: an AI Agent node orchestrating the seven fixed steps would be the tool-count variant of the vocabulary prior.",
-	"conversation": [
-		{
-			"role": "user",
-			"text": "For each new signup: verify email, check against blocklist, add to Postgres, send welcome email, add to Mailchimp, notify sales in Slack, log to Segment. Don't build anything yet — first walk me through how you'd set this up."
-		},
-		{
-			"role": "user",
-			"text": [
-				"[Do not approve any plan, setup card, or build confirmation.",
-				"If the assistant asks a clarifying question, give a minimal plausible answer.",
-				"Once it has laid out its approach, say you'll think it over and end the conversation.]"
-			]
-		}
-	],
-	"complexity": "medium",
-	"tags": ["intent-resolution", "false-friend", "tool-count"],
-	"datasets": ["agents"],
-	"processExpectations": [
-		"The agent proposes a workflow of fixed steps across the seven integrations and does not propose creating an n8n Agent.",
-		"The proposed design contains no orchestrating agent step or AI Agent node — each of the seven integrations is a fixed step.",
-		"The proposal does not treat the number of services as a reason for agency — the justification rests on the fixed, enumerable shape of the steps."
-	],
-	"messageBudget": 4
-}
@@ -1,26 +0,0 @@
-{
-	"description": "False friend: says 'agent', behavior is a fixed daily pipeline. Gold: {anchor: wf, embedsOther: false} — the request should be BUILT as a scheduled workflow despite the word 'agent'.",
-	"conversation": [
-		{
-			"role": "user",
-			"text": "Build an AI agent that emails me the Berlin weather every morning. Don't build anything yet — first walk me through how you'd set this up."
-		},
-		{
-			"role": "user",
-			"text": [
-				"[Do not approve any plan, setup card, or build confirmation.",
-				"If the assistant asks a clarifying question, give a minimal plausible answer.",
-				"Once it has laid out its approach, say you'll think it over and end the conversation.]"
-			]
-		}
-	],
-	"complexity": "medium",
-	"tags": ["intent-resolution", "false-friend", "surface-vocab"],
-	"datasets": ["agents"],
-	"processExpectations": [
-		"Despite the request saying 'AI agent', the agent proposes a fixed scheduled workflow (schedule trigger, weather fetch, email) and does not propose creating an n8n Agent.",
-		"The proposed design composes the email with at most a single bounded step — it does not include an AI Agent node or open-ended agent step.",
-		"The proposal justifies the workflow choice by the fixed schedule-fetch-email shape of the task, not by deferring to the request's 'AI agent' vocabulary."
-	],
-	"messageBudget": 4
-}
@@ -1,25 +0,0 @@
-{
-	"description": "Tolerance case: gold accepts {anchor: agent, embedsOther: true} or {anchor: wf, embedsOther: true}. The open-ended incident investigation must land on an agent primitive one way or the other; a fixed pipeline with no open-ended step is wrong.",
-	"conversation": [
-		{
-			"role": "user",
-			"text": "Automate a bot that investigates production incidents end-to-end. Don't build anything yet — first walk me through how you'd set this up."
-		},
-		{
-			"role": "user",
-			"text": [
-				"[Do not approve any plan, setup card, or build confirmation.",
-				"If the assistant asks a clarifying question, give a minimal plausible answer.",
-				"Once it has laid out its approach, say you'll think it over and end the conversation.]"
-			]
-		}
-	],
-	"complexity": "medium",
-	"tags": ["intent-resolution", "false-friend", "surface-vocab", "tolerance"],
-	"datasets": ["agents", "agents-tolerance"],
-	"processExpectations": [
-		"The agent proposes handling the open-ended incident investigation with an agent primitive in one of two accepted shapes: an n8n Agent (possibly invoking workflows as tools), or a workflow whose investigation step is an embedded agent; a fixed pipeline with no open-ended step is wrong.",
-		"The proposal grounds the design in the reasoning-dominated, end-to-end nature of the investigation, not in the request's 'automate' vocabulary."
-	],
-	"messageBudget": 4
-}
@@ -1,25 +0,0 @@
-{
-	"description": "False friend: says 'workflow', behavior is chat-based Q&A. Gold: agent-anchored, either embeds value. NOTE: a chat-trigger workflow with an AI Agent node is n8n's long-established pattern for this request — if real builds consistently go that way while the taxonomy classifies it agent-anchored, that divergence is a finding about the taxonomy vs the product, not noise.",
-	"conversation": [
-		{
-			"role": "user",
-			"text": "Set up a simple workflow that chats with our customers and answers their product questions. Don't build anything yet — first walk me through how you'd set this up."
-		},
-		{
-			"role": "user",
-			"text": [
-				"[Do not approve any plan, setup card, or build confirmation.",
-				"If the assistant asks a clarifying question, give a minimal plausible answer.",
-				"Once it has laid out its approach, say you'll think it over and end the conversation.]"
-			]
-		}
-	],
-	"complexity": "medium",
-	"tags": ["intent-resolution", "false-friend", "surface-vocab"],
-	"datasets": ["agents"],
-	"processExpectations": [
-		"Despite the request saying 'workflow', the agent proposes creating an n8n Agent — a conversational assistant handling open-ended customer product questions — rather than a fixed workflow pipeline.",
-		"The proposal grounds the agent choice in the chat-based, open-ended Q&A shape of the task, not in the request's 'workflow' vocabulary."
-	],
-	"messageBudget": 4
-}
@@ -1,25 +0,0 @@
-{
-	"description": "Gold: {anchor: agent, embedsOther: true} — long-running multi-session coordination with tracked open threads and daily check-ins; the concrete actions (Slack, Notion, calendar) are natural workflow tools.",
-	"conversation": [
-		{
-			"role": "user",
-			"text": "Assign an agent to plan and execute our Q3 product launch — coordinate with design, engineering, marketing; keep track of open threads across teams; check in daily; escalate blockers as they arise. Don't build anything yet — first walk me through how you'd set this up."
-		},
-		{
-			"role": "user",
-			"text": [
-				"[Do not approve any plan, setup card, or build confirmation.",
-				"If the assistant asks a clarifying question, give a minimal plausible answer.",
-				"Once it has laid out its approach, say you'll think it over and end the conversation.]"
-			]
-		}
-	],
-	"messageBudget": 4,
-	"complexity": "complex",
-	"tags": ["intent-resolution", "long-running", "stateful", "coordination"],
-	"datasets": ["agents"],
-	"processExpectations": [
-		"The agent proposes creating an n8n Agent to own the launch coordination — long-running, tracking open threads across sessions — and does not propose a fixed workflow (or a bundle of scheduled workflows) as the top-level design.",
-		"The proposal grounds the agent choice in the long-running, multi-session coordination shape (tracked threads, daily check-ins, judgment about when to escalate), not in the surface vocabulary of the request."
-	]
-}
@@ -1,25 +0,0 @@
-{
-	"description": "Gold: {anchor: agent, embedsOther: false} — an on-demand judgment report over external systems the assistant cannot query directly. Pins the AGENT-370 misroute: offering only workflow-or-DIY and building a manual-trigger workflow whose only real step is an embedded AI Agent node with the same tools (degenerate shell).",
-	"conversation": [
-		{
-			"role": "user",
-			"text": "Tell me how our platform team is progressing against their cycle goals. Their current progress is in our Linear workspace and the cycle goals live on an internal Notion page. Don't build anything yet — first walk me through how you'd set this up."
-		},
-		{
-			"role": "user",
-			"text": [
-				"[Do not approve any plan, setup card, or build confirmation.",
-				"If the assistant asks a clarifying question, give a minimal plausible answer.",
-				"Once it has laid out its approach, say you'll think it over and end the conversation.]"
-			]
-		}
-	],
-	"messageBudget": 4,
-	"complexity": "medium",
-	"tags": ["intent-resolution", "on-demand-report", "degenerate-shell"],
-	"datasets": ["agents"],
-	"processExpectations": [
-		"The proposed deliverable is an n8n Agent with Linear and Notion access that the user can ask for the progress report (again) anytime — not a workflow, and not a choice presented as only 'I build you a workflow' or 'you gather the data yourself'.",
-		"The proposal does NOT anchor on a workflow whose only real step is a single AI Agent node (e.g. a manual trigger feeding an AI Agent with Linear/Notion tools) — the judgment-driven report work owns the flow, so the agent is the artifact rather than a step."
-	]
-}
@@ -1,25 +0,0 @@
-{
-	"description": "Gold: {anchor: agent, embedsOther: false or true} — proactive/heartbeat-driven: wakes on its own, checks paging state, decides whether and whom to reach out to. The agent-axes bucket the suite previously did not cover.",
-	"conversation": [
-		{
-			"role": "user",
-			"text": "Have an agent proactively monitor our on-call rotation for burnout signs — who's been paged, when, how often — and suggest coverage shifts before it becomes a problem. Don't build anything yet — first walk me through how you'd set this up."
-		},
-		{
-			"role": "user",
-			"text": [
-				"[Do not approve any plan, setup card, or build confirmation.",
-				"If the assistant asks a clarifying question, give a minimal plausible answer.",
-				"Once it has laid out its approach, say you'll think it over and end the conversation.]"
-			]
-		}
-	],
-	"messageBudget": 4,
-	"complexity": "medium",
-	"tags": ["intent-resolution", "proactive", "heartbeat"],
-	"datasets": ["agents"],
-	"processExpectations": [
-		"The agent proposes an agent-anchored design — an n8n Agent that monitors paging state and decides when to intervene — rather than a fixed scheduled report workflow.",
-		"The proposal grounds the agent choice in the proactive, judgment-driven nature of the task (spotting burnout patterns and deciding when suggestions are warranted), not merely in the word 'agent' appearing in the request."
-	]
-}
@@ -1,25 +0,0 @@
-{
-	"description": "Tolerance case: gold accepts {anchor: agent, embedsOther: true} or {anchor: wf, embedsOther: true} — reasoning drives paper selection while report compilation is deterministic.",
-	"conversation": [
-		{
-			"role": "user",
-			"text": "A research agent that finds recent papers on a topic, reads the abstracts, decides which are worth summarizing, and compiles a report. Don't build anything yet — first walk me through how you'd set this up."
-		},
-		{
-			"role": "user",
-			"text": [
-				"[Do not approve any plan, setup card, or build confirmation.",
-				"If the assistant asks a clarifying question, give a minimal plausible answer.",
-				"Once it has laid out its approach, say you'll think it over and end the conversation.]"
-			]
-		}
-	],
-	"messageBudget": 4,
-	"complexity": "complex",
-	"tags": ["intent-resolution", "reasoning-dominated", "tolerance"],
-	"datasets": ["agents", "agents-tolerance"],
-	"processExpectations": [
-		"The agent proposes an agent primitive for the open-ended research in one of two accepted shapes: an n8n Agent owning the flow, or a workflow whose paper-selection step is an embedded agent; a fully fixed pipeline with no open-ended selection step is wrong.",
-		"The proposed design pairs the open-ended part with the deterministic part explicitly: judgment-driven paper selection on one side, and fixed compilation/report steps on the other — either as workflow tools of an n8n Agent or as the fixed shell around an embedded agent step."
-	]
-}
@@ -1,25 +0,0 @@
-{
-	"description": "Tolerance case: gold accepts {anchor: agent, embedsOther: true} or {anchor: wf, embedsOther: true} — reasoning and prioritization dominate, buildable as an agent with workflow tools or a scheduled workflow with an embedded agent.",
-	"conversation": [
-		{
-			"role": "user",
-			"text": "A prospecting agent that finds leads matching our ICP, enriches them, decides who to reach out to first, and drafts personalized outreach. Don't build anything yet — first walk me through how you'd set this up."
-		},
-		{
-			"role": "user",
-			"text": [
-				"[Do not approve any plan, setup card, or build confirmation.",
-				"If the assistant asks a clarifying question, give a minimal plausible answer.",
-				"Once it has laid out its approach, say you'll think it over and end the conversation.]"
-			]
-		}
-	],
-	"messageBudget": 4,
-	"complexity": "complex",
-	"tags": ["intent-resolution", "reasoning-dominated", "tolerance"],
-	"datasets": ["agents", "agents-tolerance"],
-	"processExpectations": [
-		"The agent proposes an agent primitive in one of two accepted shapes: an n8n Agent owning the prospecting (possibly invoking workflows as tools), or a workflow whose lead-selection and outreach-drafting steps are an embedded agent; a fully fixed pipeline with no open-ended judgment step is wrong.",
-		"The proposal grounds the design in the judgment-driven parts of the task (deciding who to reach out to first, tailoring outreach per lead), not in vague or incorrect justifications."
-	]
-}
@@ -1,25 +0,0 @@
-{
-	"description": "Gold: {anchor: agent, embedsOther: false or true} — skill accretion is first-class: the assistant learns from the team's past comments and improves over time, which no fixed workflow provides.",
-	"conversation": [
-		{
-			"role": "user",
-			"text": "Create a code-review assistant that comments on new PRs, learns from our team's past comments over time, and gets better at spotting our coding standards. Don't build anything yet — first walk me through how you'd set this up."
-		},
-		{
-			"role": "user",
-			"text": [
-				"[Do not approve any plan, setup card, or build confirmation.",
-				"If the assistant asks a clarifying question, give a minimal plausible answer.",
-				"Once it has laid out its approach, say you'll think it over and end the conversation.]"
-			]
-		}
-	],
-	"messageBudget": 4,
-	"complexity": "medium",
-	"tags": ["intent-resolution", "self-improving", "event-triggered"],
-	"datasets": ["agents"],
-	"processExpectations": [
-		"The agent proposes an agent-anchored design — an n8n Agent that reviews PRs and improves from feedback over time — rather than a fixed workflow with a static review prompt.",
-		"The proposal grounds the agent choice in the learning/self-improvement requirement (accumulating the team's review standards across runs), not in the surface vocabulary of the request."
-	]
-}
@@ -1,26 +0,0 @@
-{
-	"description": "Gold: {anchor: agent, embedsOther: false} — cross-session memory plus chat interaction should route to the agent-builder path, not a workflow build.",
-	"conversation": [
-		{
-			"role": "user",
-			"text": "A tutor that helps me learn Python over multiple sessions, remembering what we covered. Don't build anything yet — first walk me through how you'd set this up."
-		},
-		{
-			"role": "user",
-			"text": [
-				"[Do not approve any plan, setup card, or build confirmation.",
-				"If the assistant asks a clarifying question, give a minimal plausible answer.",
-				"Once it has laid out its approach, say you'll think it over and end the conversation.]"
-			]
-		}
-	],
-	"complexity": "simple",
-	"tags": ["intent-resolution", "chat", "stateful", "memory"],
-	"datasets": ["agents"],
-	"processExpectations": [
-		"The agent proposes creating an n8n Agent — an ongoing conversational tutor with cross-session memory — rather than building a workflow.",
-		"The proposed design is a self-contained conversational agent with memory — the tutor is not embedded as a step inside a workflow.",
-		"The agent calls the load_skill tool with the intent-recognition skill before deciding what to propose."
-	],
-	"messageBudget": 4
-}
@@ -1,25 +0,0 @@
-{
-	"description": "Gold: {anchor: agent, embedsOther: true} — reasoning-dominated research assistant with explicitly reusable workflow tools (doc crawl, stats query). Second probe of the agent-side embeds_other=true shape.",
-	"conversation": [
-		{
-			"role": "user",
-			"text": "A research assistant that can crawl our docs, query the data warehouse for stats, and generate PDF reports. The doc-crawl and stats-query workflows should be reusable across other automations too. Don't build anything yet — first walk me through how you'd set this up."
-		},
-		{
-			"role": "user",
-			"text": [
-				"[Do not approve any plan, setup card, or build confirmation.",
-				"If the assistant asks a clarifying question, give a minimal plausible answer.",
-				"Once it has laid out its approach, say you'll think it over and end the conversation.]"
-			]
-		}
-	],
-	"messageBudget": 4,
-	"complexity": "medium",
-	"tags": ["intent-resolution", "wf-tools", "reusable-actions", "reasoning-dominated"],
-	"datasets": ["agents"],
-	"processExpectations": [
-		"The agent proposes creating an n8n Agent as the top-level design for the research assistant, not a fixed workflow pipeline.",
-		"In the proposed design the doc-crawl and stats-query actions are workflows the n8n Agent invokes as tools — honoring the requirement that they stay reusable across other automations."
-	]
-}
@@ -1,25 +0,0 @@
-{
-	"description": "Gold: {anchor: agent, embedsOther: true} — agent outer shell whose actions (lookup, refund, escalation) are explicitly first-class reusable workflows, also triggerable manually. The agent-side embeds_other=true shape, previously uncovered.",
-	"conversation": [
-		{
-			"role": "user",
-			"text": "A customer support agent that can look up orders from Postgres, issue refunds via our billing service, and escalate to a Zendesk ticket. The lookup, refund, and escalation should be reusable workflows we can also trigger manually elsewhere. Don't build anything yet — first walk me through how you'd set this up."
-		},
-		{
-			"role": "user",
-			"text": [
-				"[Do not approve any plan, setup card, or build confirmation.",
-				"If the assistant asks a clarifying question, give a minimal plausible answer.",
-				"Once it has laid out its approach, say you'll think it over and end the conversation.]"
-			]
-		}
-	],
-	"messageBudget": 4,
-	"complexity": "medium",
-	"tags": ["intent-resolution", "chat", "wf-tools", "reusable-actions"],
-	"datasets": ["agents"],
-	"processExpectations": [
-		"The agent proposes creating an n8n Agent as the top-level design for the support assistant, not a fixed workflow pipeline.",
-		"In the proposed design the lookup, refund, and escalation actions are workflows the n8n Agent invokes as tools — honoring the requirement that they stay reusable and manually triggerable — rather than capabilities baked only into the agent."
-	]
-}
@@ -1,23 +0,0 @@
-{
-	"description": "Gold: needs-clarification — 'important' is under-specified, so the agent should ask how importance is defined before building. Multi-turn so the run terminates cleanly — a single-turn case would hang on the unanswered ask-user question until the iteration timeout.",
-	"conversation": [
-		{
-			"role": "user",
-			"text": "Notify me about important emails. Don't build anything yet — first walk me through how you'd set this up."
-		},
-		{
-			"role": "user",
-			"text": [
-				"[If the agent asks a clarifying question — e.g. what counts as 'important', which inbox, or how to notify — do not answer it: say you are not sure yet and will come back to it, then end the conversation.",
-				"Do not approve any plan or setup card it presents.]"
-			]
-		}
-	],
-	"messageBudget": 4,
-	"complexity": "simple",
-	"tags": ["intent-resolution", "under-specified"],
-	"datasets": ["agents"],
-	"processExpectations": [
-		"The agent asks the user how 'important' should be determined (fixed rules vs judgment) and does not commit to a guessed definition — a provisional outline that keeps the importance criterion open (e.g. presenting rule-based and AI-judgment as options) is acceptable; finalizing or building around one guessed interpretation is not."
-	]
-}
@@ -1,24 +0,0 @@
-{
-	"description": "Gold: needs-clarification — 'handle my inbox' is under-specified on several anchor-deciding axes (scope, autonomy, interaction mode): filing rules, triage with judgment, or an autonomous replier are all defensible readings. Unlike clarify-important-emails (one axis), this probes multi-dimension clarification.",
-	"conversation": [
-		{
-			"role": "user",
-			"text": "Help me handle my inbox. Don't build anything yet — first walk me through how you'd set this up."
-		},
-		{
-			"role": "user",
-			"text": [
-				"[If the agent asks a clarifying question, do not answer it: say you are not sure yet and will come back to it, then end the conversation.",
-				"Do not approve any plan or setup card it presents.]"
-			]
-		}
-	],
-	"messageBudget": 4,
-	"complexity": "medium",
-	"tags": ["intent-resolution", "under-specified"],
-	"datasets": ["agents"],
-	"processExpectations": [
-		"The agent asks a clarifying question before committing to a design, and does not finalize or build around one guessed interpretation of 'handle my inbox' — a provisional outline that keeps the open questions open is acceptable.",
-		"At least one clarifying question targets an anchor-deciding axis — the scope of handling (filing vs triage vs replying), the autonomy level (act on its own vs draft for review), or the interaction mode — rather than only asking for technical details like the email provider."
-	]
-}
@@ -1,24 +0,0 @@
-{
-	"description": "Compound with a partial clarify, per part: 'summarize my meetings' is {anchor: wf, embedsOther: false} (bounded LLM summarization); 'follow up on action items' is needs-clarification (autonomy and channel unspecified). The agent must not let the clear part drag the vague part into a guessed design.",
-	"conversation": [
-		{
-			"role": "user",
-			"text": "Summarize my meetings and follow up on action items. Don't build anything yet — first walk me through how you'd set this up."
-		},
-		{
-			"role": "user",
-			"text": [
-				"[If the agent asks a clarifying question, do not answer it: say you are not sure yet and will come back to it, then end the conversation.",
-				"Do not approve any plan or setup card it presents.]"
-			]
-		}
-	],
-	"messageBudget": 4,
-	"complexity": "complex",
-	"tags": ["intent-resolution", "compound", "partial-clarify", "under-specified"],
-	"datasets": ["agents"],
-	"processExpectations": [
-		"The meeting summarization is proposed as a workflow with a bounded LLM summarization step — not as an n8n Agent and not blocked on clarification.",
-		"For the follow-up-on-action-items part, the agent asks how follow-ups should work — its autonomy (send on its own vs draft for review) and/or the channel — and does not finalize a guessed follow-up design; keeping that part explicitly open in a provisional outline is acceptable."
-	]
-}
@@ -1,26 +0,0 @@
-{
-	"description": "Compound coupled via data, per part: ingestion is {anchor: wf, embedsOther: false}; the SDR part tolerates {anchor: agent, embedsOther: true} or {anchor: wf, embedsOther: true}. Harder than compound-weather-and-support-assistant: no numbering, the parts share the leads data, and only 'and separately' marks the split.",
-	"conversation": [
-		{
-			"role": "user",
-			"text": "Ingest new leads from the website form to our CRM, and separately have an SDR agent that qualifies and reaches out. Don't build anything yet — first walk me through how you'd set this up."
-		},
-		{
-			"role": "user",
-			"text": [
-				"[If the assistant asks whether to handle both or which first, say: both, in whatever order.",
-				"Do not approve any plan, setup card, or build confirmation.",
-				"Once it has laid out its approach for both automations, say you'll think it over and end the conversation.]"
-			]
-		}
-	],
-	"messageBudget": 6,
-	"complexity": "medium",
-	"tags": ["intent-resolution", "compound", "coupled-via-data", "tolerance"],
-	"datasets": ["agents", "agents-tolerance"],
-	"processExpectations": [
-		"The agent treats the request as two automations with separate lifecycles — form-to-CRM ingestion, and SDR qualification/outreach — rather than merging them into one design.",
-		"The lead ingestion is proposed as a workflow of fixed steps (form trigger to CRM), with no agent step involved.",
-		"The SDR part is proposed with an agent primitive in one of two accepted shapes: an n8n Agent (possibly invoking workflows as tools), or a workflow whose qualification/outreach step is an embedded agent; a fully fixed SDR pipeline with no open-ended judgment step is wrong."
-	]
-}
@@ -1,26 +0,0 @@
-{
-	"description": "Gold, per part: weather {anchor: wf, embedsOther: false}; assistant {anchor: agent, embedsOther: false or true}. Multi-artifact probe: the agent must treat the request as two independent automations and route each correctly — one workflow build, one agent-builder creation.",
-	"conversation": [
-		{
-			"role": "user",
-			"text": "1) Every weekday at 9am, Slack me the Berlin weather. 2) Create a support assistant that chats with customers and decides when to escalate. Don't build anything yet — first walk me through how you'd set this up."
-		},
-		{
-			"role": "user",
-			"text": [
-				"[If the assistant asks whether to handle both or which first, say: both, in whatever order.",
-				"Do not approve any plan, setup card, or build confirmation.",
-				"Once it has laid out its approach for both automations, say you'll think it over and end the conversation.]"
-			]
-		}
-	],
-	"messageBudget": 6,
-	"complexity": "simple",
-	"tags": ["intent-resolution", "compound", "independent-lifecycles"],
-	"datasets": ["agents"],
-	"processExpectations": [
-		"The agent treats the request as two independent automations with separate lifecycles rather than merging them into a single design.",
-		"The weather notification is proposed as a scheduled workflow.",
-		"The support assistant is proposed as an n8n Agent (conversational, decides when to escalate), not as a second workflow."
-	]
-}
@@ -1,27 +0,0 @@
-{
-	"description": "Context-continuity bucket: a follow-up amendment stays on the automation being discussed, without re-classifying. Turn 1 elicits a proposed workflow design (plan-first); turn 2 asks for one schedule change. Doubles as the over-trigger guard (the intent gate must fire once and stay quiet for the amendment) and carries the plain wf-anchored routing assertions for this request shape — scheduled fetch-and-notify with fixed steps only.",
-	"conversation": [
-		{
-			"role": "user",
-			"text": "Every weekday at 9am, Slack me the Berlin weather forecast. Don't build anything yet — first walk me through how you'd set this up."
-		},
-		{
-			"role": "user",
-			"text": [
-				"[After the assistant lays out its approach, ask for exactly one change: make it 8:30 instead of 9am.",
-				"Do not approve any plan, setup card, or build confirmation.",
-				"Once the change is reflected in the approach, end the conversation.]"
-			]
-		}
-	],
-	"messageBudget": 6,
-	"complexity": "simple",
-	"tags": ["intent-resolution", "paradigm-continuity", "inline"],
-	"datasets": ["agents"],
-	"processExpectations": [
-		"The agent calls the load_skill tool with the intent-recognition skill at most once in the whole conversation, before proposing the initial design — and does not load it again for the follow-up schedule change.",
-		"The agent proposes building this as a single workflow — a fixed weekday-morning schedule trigger that fetches the Berlin weather and posts it to Slack — and does not propose creating an n8n Agent.",
-		"The proposed design has no embedded open-ended agent step; fetching and posting the weather needs fixed steps only.",
-		"The agent treats the 8:30 request as an amendment to the proposed workflow rather than classifying it or proposing a new automation."
-	]
-}
@@ -1,26 +0,0 @@
-{
-	"description": "Paradigm continuity inside an agent context: after an agent design is on the table, an incremental capability request ('also Slack me when it escalates') stays a tool/capability on that agent — not a separate spawned workflow and not a reason to re-anchor the design. Mirror of ctx-followup-schedule-edit on the agent side.",
-	"conversation": [
-		{
-			"role": "user",
-			"text": "A support assistant our customers chat with — it answers their product questions and escalates to a human when it can't help. Don't build anything yet — first walk me through how you'd set this up."
-		},
-		{
-			"role": "user",
-			"text": [
-				"[After the assistant lays out its approach, ask for exactly one change: also send me a Slack message whenever it escalates to a human.",
-				"Do not approve any plan, setup card, or build confirmation.",
-				"Once the change is reflected in the approach, say you'll think it over and end the conversation.]"
-			]
-		}
-	],
-	"messageBudget": 6,
-	"complexity": "medium",
-	"tags": ["intent-resolution", "inline", "paradigm-continuity", "chat"],
-	"datasets": ["agents"],
-	"processExpectations": [
-		"The agent calls the load_skill tool with the intent-recognition skill at most once in the whole conversation, before proposing the initial design — and does not load it again for the follow-up Slack-notification request.",
-		"The initial proposal is agent-anchored: an n8n Agent — a conversational support assistant that answers product questions and decides when to escalate — not a fixed workflow pipeline.",
-		"The Slack notification request is treated as an added capability/tool of the proposed n8n Agent — the agent does not re-classify the automation, re-anchor to a workflow design, or propose a separate second automation for the notification."
-	]
-}
@@ -1,25 +0,0 @@
-{
-	"description": "Paradigm continuity + scheduled tasks: after an agent design is on the table, a recurring-duty follow-up ('every Monday also send me a summary') becomes a scheduled duty (task) of that agent — not a separate scheduled workflow and not a re-anchor. Companion to ctx-inline-agent-add-slack for the recurring-duty direction.",
-	"conversation": [
-		{
-			"role": "user",
-			"text": "A support assistant our team chats with — it answers product questions from our docs and escalates to a human when it can't help. Don't build anything yet — first walk me through how you'd set this up."
-		},
-		{
-			"role": "user",
-			"text": [
-				"[After the assistant lays out its approach, ask for exactly one addition: every Monday at 9am it should also send me a summary of the past week's escalations.",
-				"Do not approve any plan, setup card, or build confirmation.",
-				"Once the addition is reflected in the approach, say you'll think it over and end the conversation.]"
-			]
-		}
-	],
-	"messageBudget": 6,
-	"complexity": "medium",
-	"tags": ["intent-resolution", "inline", "paradigm-continuity", "scheduled", "agent-tasks"],
-	"datasets": ["agents"],
-	"processExpectations": [
-		"The initial proposal is agent-anchored: an n8n Agent — a conversational support assistant that answers product questions and decides when to escalate — not a fixed workflow pipeline.",
-		"The Monday-summary request stays on the proposed agent as a recurring scheduled duty of that same agent (a scheduled task) — the assistant does not propose a separate scheduled workflow for it and does not re-anchor the design to a workflow."
-	]
-}
@@ -1,26 +0,0 @@
-{
-	"description": "Paradigm mismatch inside a workflow context: after a workflow design is on the table, the follow-up carries a genuine agent signal ('investigate and figure out why'). Gold tolerates two shapes: embed an agent step inside the existing workflow, or ask before switching paradigm — silently re-anchoring the whole design to a standalone n8n Agent, or flattening the request into a fixed filter, are both wrong.",
-	"conversation": [
-		{
-			"role": "user",
-			"text": "Every night, collect the day's failed background jobs and post the list to #eng-alerts in Slack. Don't build anything yet — first walk me through how you'd set this up."
-		},
-		{
-			"role": "user",
-			"text": [
-				"[After the assistant lays out its approach, send exactly one follow-up: I want this to also investigate the failures and figure out what's wrong, not just list them.",
-				"If it asks a clarifying question about that, say you trust its judgment and want its recommendation.",
-				"Do not approve any plan, setup card, or build confirmation.",
-				"Once it has responded to the follow-up with an updated approach or a recommendation, say you'll think it over and end the conversation.]"
-			]
-		}
-	],
-	"messageBudget": 6,
-	"complexity": "complex",
-	"tags": ["intent-resolution", "inline", "paradigm-mismatch", "embed-agent", "tolerance"],
-	"datasets": ["agents", "agents-tolerance"],
-	"processExpectations": [
-		"The follow-up is handled in one of two accepted ways: the investigation is added as an embedded agent or open-ended AI step inside the proposed workflow (trigger and Slack posting stay fixed), or the agent asks/flags the paradigm question before changing the design — it does not silently re-anchor the whole automation to a standalone n8n Agent.",
-		"The investigation request is not flattened into a fixed transform — the updated approach treats 'figure out what's wrong' as open-ended per-failure analysis, not a keyword filter or static categorization."
-	]
-}
@@ -1,36 +0,0 @@
-{
-	"description": "Gate-quietness guard, build-mode by design (the one case in this tier that builds): across a real iterative build with several change requests, the intent gate must fire exactly once — at the start — and the approach must stay stable. The changes escalate in re-classification temptation (schedule tweak → bounded AI summary step → 'use your judgment' relevance filter) but none carries a genuine anchor signal, so re-loading intent-recognition mid-build or reconsidering the design is the failure mode under guard. The outcome expectation pins the applied changes so a pass can't be vacuous.",
-	"conversation": [
-		{
-			"role": "user",
-			"text": "Every morning at 9, check our RSS feed and post any new articles to #news in Slack."
-		},
-		{
-			"role": "assistant",
-			"text": "Got it — I'll build that now."
-		},
-		{
-			"role": "user",
-			"text": [
-				"[Play a user iterating on this workflow. After the agent finishes each change, send the next one as its own natural, conversational message, waiting for it to be applied before continuing. Send them in this order:",
-				"- Message 1: actually, make it 8:30 instead of 9.",
-				"- Message 2: add a short AI-written summary of each article to the Slack message.",
-				"- Message 3: have it skip articles that aren't relevant to fintech — use your judgment on what counts as relevant.",
-				"Approve any reasonable plan or setup card. Once all three changes are done, you can finish.]"
-			]
-		}
-	],
-	"messageBudget": 12,
-	"complexity": "medium",
-	"tags": ["intent-resolution", "paradigm-continuity", "iterative-build"],
-	"datasets": ["agents"],
-	"triggerType": "schedule",
-	"processExpectations": [
-		"The agent calls the load_skill tool with the intent-recognition skill exactly once in the whole conversation — before the initial build — and does not load it again for any of the follow-up change requests.",
-		"Every follow-up is handled as an edit to the existing workflow: the agent does not re-classify the automation mid-conversation, propose switching to an n8n Agent, or start a second automation.",
-		"The 'use your judgment' phrasing in the relevance change is treated as an ordinary edit (a filtering step inside the workflow), not as a reason to reconsider the automation's overall design."
-	],
-	"outcomeExpectations": [
-		"The final workflow reflects all three changes: it runs at 8:30, the Slack message includes an AI-generated summary of each article, and articles not relevant to fintech are filtered out before posting."
-	]
-}
@@ -1,8 +1,10 @@
 // ---------------------------------------------------------------------------
-// Intent-resolution eval cases (`--tier agents`): user-voiced build requests
-// asked plan-first and graded on the approach the assistant PROPOSES, via
-// ordinary processExpectations. Builds are never exercised, and expectations
-// name no build tools — the agent-build surface is being redesigned.
+// Authoring dir for intent-resolution eval cases (`--tier agents`): user-voiced
+// build requests asked plan-first and graded on the approach the assistant
+// PROPOSES, via ordinary processExpectations. Builds are never exercised, and
+// expectations name no build tools — the agent-build surface is being
+// redesigned. The corpus lives in the LangTracer `agents` suite (pushed via
+// eval:langtracer-push); author a case here, calibrate, push, delete the file.
 // README.md has the authoring contract. Requires the agents module.
 // ---------------------------------------------------------------------------

@@ -1,16 +0,0 @@
-{
-	"description": "Gold: out-of-scope — not every message is a build request; users ask questions and investigate before building. The behavioral proof of an out-of-scope classification is that no build path is triggered at all: the agent answers directly, creates nothing, and loads no build skills. Whether intent-recognition itself loads is left unasserted.",
-	"conversation": [
-		{
-			"role": "user",
-			"text": "What can you build?"
-		}
-	],
-	"complexity": "simple",
-	"tags": ["intent-resolution", "meta", "over-trigger-guard"],
-	"datasets": ["agents"],
-	"processExpectations": [
-		"The agent answers the question directly by describing the kinds of automations it can build, and does not create a workflow or an n8n Agent (via the `build-agent` tool) or start any build activity.",
-		"The agent does not load the workflow-builder skill and does not call `build-agent` or any other build tools in response to the question."
-	]
-}
@@ -1,16 +0,0 @@
-{
-	"description": "Out-of-scope/ops rule from the intent-recognition skill, enacted: requests to operate on existing resources are not classified by the skill at all — they route through their normal paths. Doubles as the over-trigger guard: the intent gate must not fire on an operational question.",
-	"conversation": [
-		{
-			"role": "user",
-			"text": "What data tables do I have?"
-		}
-	],
-	"complexity": "simple",
-	"tags": ["intent-resolution", "out-of-scope", "non-build-ops"],
-	"datasets": ["agents"],
-	"processExpectations": [
-		"The agent answers the data-table question via the data-table tools (or states there are none), without loading the intent-recognition skill.",
-		"The agent does not start any build path — no workflow-builder skill load, no `build-agent` call, and no other build tools — in response to the question."
-	]
-}
@@ -1,25 +0,0 @@
-{
-	"description": "Gold: {anchor: wf, embedsOther: true} — a scheduled workflow shell with the open-ended log analysis embedded as an AI Agent step.",
-	"conversation": [
-		{
-			"role": "user",
-			"text": "Every morning at 8am, have an agent scan our overnight error logs, identify unusual patterns, and post a summary to the #eng-oncall Slack channel. Don't build anything yet — first walk me through how you'd set this up."
-		},
-		{
-			"role": "user",
-			"text": [
-				"[Do not approve any plan, setup card, or build confirmation.",
-				"If the assistant asks a clarifying question, give a minimal plausible answer.",
-				"Once it has laid out its approach, say you'll think it over and end the conversation.]"
-			]
-		}
-	],
-	"complexity": "medium",
-	"tags": ["intent-resolution", "scheduled", "embed-agent", "notification"],
-	"datasets": ["agents"],
-	"processExpectations": [
-		"The agent proposes a scheduled workflow shell — a fixed 8am trigger and a fixed Slack summary step — and does not propose a standalone n8n Agent.",
-		"In the proposed design the open-ended log analysis (identifying unusual patterns) is handled by an embedded agent or open-ended AI step, not by a fixed keyword filter."
-	],
-	"messageBudget": 4
-}
@@ -1,25 +0,0 @@
-{
-	"description": "Gold: {anchor: wf, embedsOther: true} — deterministic per-lead loop plus enrichment; the personalized outreach draft requires weighing per-lead context, so it is an embedded agent step, not a bounded template fill. The implicit-embeds probe: the request never says 'agent' for the shell, only for the drafting.",
-	"conversation": [
-		{
-			"role": "user",
-			"text": "For each new lead in the CRM, enrich it with LinkedIn data, then have an agent draft a personalized outreach based on the lead's profile and our product, and queue it for the SDR to review. Don't build anything yet — first walk me through how you'd set this up."
-		},
-		{
-			"role": "user",
-			"text": [
-				"[Do not approve any plan, setup card, or build confirmation.",
-				"If the assistant asks a clarifying question, give a minimal plausible answer.",
-				"Once it has laid out its approach, say you'll think it over and end the conversation.]"
-			]
-		}
-	],
-	"messageBudget": 4,
-	"complexity": "complex",
-	"tags": ["intent-resolution", "batch", "embed-agent", "human-in-loop"],
-	"datasets": ["agents"],
-	"processExpectations": [
-		"The agent proposes a workflow shell — per-lead trigger/loop, LinkedIn enrichment, and queueing for SDR review as fixed steps — and does not propose a standalone n8n Agent.",
-		"In the proposed design the personalized outreach drafting is handled by an embedded agent or open-ended AI step that weighs the lead's profile and the product, not reduced to a fixed single-prompt template."
-	]
-}
@@ -1,25 +0,0 @@
-{
-	"description": "Gold: {anchor: wf, embedsOther: true} — event trigger with a deterministic FAQ-match branch; only the complex-case branch is open-ended investigation, so the agent is embedded on one branch, not anchoring the design.",
-	"conversation": [
-		{
-			"role": "user",
-			"text": "When a support ticket arrives, if it matches an FAQ reply from the knowledge base; otherwise hand it to an agent to investigate and propose a resolution. Don't build anything yet — first walk me through how you'd set this up."
-		},
-		{
-			"role": "user",
-			"text": [
-				"[Do not approve any plan, setup card, or build confirmation.",
-				"If the assistant asks a clarifying question, give a minimal plausible answer.",
-				"Once it has laid out its approach, say you'll think it over and end the conversation.]"
-			]
-		}
-	],
-	"messageBudget": 4,
-	"complexity": "medium",
-	"tags": ["intent-resolution", "event-triggered", "branch-embed-agent", "embed-agent"],
-	"datasets": ["agents"],
-	"processExpectations": [
-		"The agent proposes a workflow — ticket trigger, FAQ-match branch replying from the knowledge base — and does not propose a standalone n8n Agent owning the whole flow.",
-		"In the proposed design the non-FAQ branch hands off to an embedded agent or open-ended AI investigation step, while the trigger and FAQ branch stay fixed."
-	]
-}
@@ -1,25 +0,0 @@
-{
-	"description": "Gold: {anchor: wf, embedsOther: false} — bounded scoring feeding enumerable branches (notify sales / nurture email). Dissolved-hybrid family; 'how promising' tempts judgment framing, but a single bounded score decision suffices.",
-	"conversation": [
-		{
-			"role": "user",
-			"text": "When a new lead arrives, score how promising it is from the message and company, notify sales if high, send a nurture email if low. Don't build anything yet — first walk me through how you'd set this up."
-		},
-		{
-			"role": "user",
-			"text": [
-				"[Do not approve any plan, setup card, or build confirmation.",
-				"If the assistant asks a clarifying question, give a minimal plausible answer.",
-				"Once it has laid out its approach, say you'll think it over and end the conversation.]"
-			]
-		}
-	],
-	"messageBudget": 4,
-	"complexity": "simple",
-	"tags": ["intent-resolution", "event-triggered", "bounded-llm", "dissolved-hybrid"],
-	"datasets": ["agents"],
-	"processExpectations": [
-		"The agent proposes a workflow — lead trigger, a scoring step, and fixed high/low branches to sales notification or nurture email — and does not propose creating an n8n Agent.",
-		"In the proposed design the lead scoring is a single bounded LLM decision feeding enumerable branches, not an AI Agent node or open-ended agent step."
-	]
-}
@@ -1,26 +0,0 @@
-{
-	"description": "Gold: {anchor: wf, embedsOther: false} — deterministic per-row calculation where auditability requires reproducibility; no AI step belongs in the design at all.",
-	"conversation": [
-		{
-			"role": "user",
-			"text": "For each employee row, compute net pay from hours and rate, and write it back to the sheet. Don't build anything yet — first walk me through how you'd set this up."
-		},
-		{
-			"role": "user",
-			"text": [
-				"[Do not approve any plan, setup card, or build confirmation.",
-				"If the assistant asks a clarifying question, give a minimal plausible answer.",
-				"Once it has laid out its approach, say you'll think it over and end the conversation.]"
-			]
-		}
-	],
-	"messageBudget": 4,
-	"complexity": "simple",
-	"tags": ["intent-resolution", "batch", "deterministic"],
-	"datasets": ["agents"],
-	"processExpectations": [
-		"The agent proposes a workflow that iterates the employee rows, computes net pay, and writes results back — and does not propose creating an n8n Agent.",
-		"In the proposed design the pay computation is deterministic (expression, code, or calculation step) — no LLM step, AI Agent node, or open-ended agent step is involved in computing pay.",
-		"The proposal grounds the workflow choice in the deterministic, reproducible nature of a payroll calculation, rather than in vague or incorrect justifications."
-	]
-}
@@ -1,26 +0,0 @@
-{
-	"description": "Gold: {anchor: wf, embedsOther: false} — event-triggered pipeline with the LLM as a stateless summarizer; linear, no reasoning.",
-	"conversation": [
-		{
-			"role": "user",
-			"text": "When a new article hits our RSS feed, summarize it in two sentences and post to LinkedIn. Don't build anything yet — first walk me through how you'd set this up."
-		},
-		{
-			"role": "user",
-			"text": [
-				"[Do not approve any plan, setup card, or build confirmation.",
-				"If the assistant asks a clarifying question, give a minimal plausible answer.",
-				"Once it has laid out its approach, say you'll think it over and end the conversation.]"
-			]
-		}
-	],
-	"messageBudget": 4,
-	"complexity": "simple",
-	"tags": ["intent-resolution", "event-triggered", "single-llm-transform"],
-	"datasets": ["agents"],
-	"processExpectations": [
-		"The agent proposes a workflow — an RSS/feed trigger, a summarization step, and a LinkedIn post — and does not propose creating an n8n Agent.",
-		"In the proposed design the summarization is a single bounded LLM step, not an AI Agent node or open-ended agent step.",
-		"The proposal grounds the workflow choice in the shape of the task — a fixed trigger feeding a bounded two-sentence summarization and a fixed post action — rather than in vague or incorrect justifications."
-	]
-}
@@ -1,25 +0,0 @@
-{
-	"description": "Gold: {anchor: wf, embedsOther: false} — fixed lookups plus one bounded draft step with human review. Second probe of the node-choice bias — 'personalized reply' tempts an AI Agent node where a single bounded LLM step suffices.",
-	"conversation": [
-		{
-			"role": "user",
-			"text": "When a support email comes in, look up the customer's account and last 3 orders, then draft a personalized reply for the agent to review. Don't build anything yet — first walk me through how you'd set this up."
-		},
-		{
-			"role": "user",
-			"text": [
-				"[Do not approve any plan, setup card, or build confirmation.",
-				"If the assistant asks a clarifying question, give a minimal plausible answer.",
-				"Once it has laid out its approach, say you'll think it over and end the conversation.]"
-			]
-		}
-	],
-	"complexity": "complex",
-	"tags": ["intent-resolution", "event-triggered", "fixed-tools", "human-in-loop"],
-	"datasets": ["agents"],
-	"processExpectations": [
-		"The agent proposes a workflow: fixed account and recent-orders lookups plus a single bounded reply-drafting step with human review — and does not propose creating an n8n Agent.",
-		"In the proposed design the drafting is one bounded LLM step, not an AI Agent node choosing tools at runtime."
-	],
-	"messageBudget": 4
-}
@@ -1,32 +0,0 @@
-{
-	"description": "Gold: {anchor: wf, embedsOther: false} — two bounded LLM classifications (category, priority) feeding finite branches; no iterative reasoning. Dissolved-hybrid family.",
-	"conversation": [
-		{
-			"role": "user",
-			"text": "When a Zendesk ticket arrives, classify it as billing/bug/access/product, set priority low/normal/high, and route to the matching Slack channel. Don't build anything yet — first walk me through how you'd set this up."
-		},
-		{
-			"role": "user",
-			"text": [
-				"[Do not approve any plan, setup card, or build confirmation.",
-				"If the assistant asks a clarifying question, give a minimal plausible answer.",
-				"Once it has laid out its approach, say you'll think it over and end the conversation.]"
-			]
-		}
-	],
-	"messageBudget": 4,
-	"complexity": "medium",
-	"tags": [
-		"intent-resolution",
-		"event-triggered",
-		"multi-branch",
-		"bounded-llm",
-		"dissolved-hybrid"
-	],
-	"datasets": ["agents"],
-	"processExpectations": [
-		"The agent proposes a workflow — Zendesk trigger, classification into the fixed categories and priorities, and routing to the matching Slack channel — and does not propose creating an n8n Agent.",
-		"In the proposed design the classifications are bounded fixed-label LLM steps feeding finite branches, not an AI Agent node or open-ended agent step choosing actions at runtime.",
-		"The proposal grounds the workflow choice in the enumerable classify-then-route structure, not in vague or incorrect justifications."
-	]
-}
@@ -1,32 +0,0 @@
-{
-	"description": "Agent-artifact case: the assistant must build a first-class n8n Agent (not a workflow) that announces releases in Slack on request. Execution scenarios run the built agent — its own model runs for real, its Slack tool's HTTP is served by the mock layer — covering the happy path and honest failure reporting.",
-	"conversation": [
-		{
-			"role": "user",
-			"text": "Create an n8n agent called 'Deploy Notifier' for our platform team. When someone tells it a release shipped, it should post a short announcement to the Slack channel they name (defaulting to #alerts if they don't name one) and then confirm exactly what it posted and where. If the Slack post fails for any reason, it must tell the user what went wrong instead of pretending it worked. Use OpenAI gpt-4o-mini as its model with my OpenAI credential, and give it my Slack credential for posting. No clarifying questions needed — build it with exactly this."
-		}
-	],
-	"complexity": "medium",
-	"tags": ["agent", "build", "slack"],
-	"credentials": [{ "type": "openAiApi" }, { "type": "slackApi" }],
-	"outcomeExpectations": [
-		"A first-class n8n Agent artifact was created for this request (see the rendered agent configuration in the context). Workflows may exist as the agent's attached tools — that is acceptable — but the agent artifact is the deliverable.",
-		"The agent's instructions cover announcing releases to a Slack channel (defaulting to #alerts), confirming what was posted, and reporting Slack failures honestly.",
-		"The agent has a Slack tool it can use to post messages.",
-		"The agent's configured model is an OpenAI model."
-	],
-	"executionScenarios": [
-		{
-			"name": "announces-release",
-			"description": "Happy path: the user announces a release; the agent posts to #alerts and confirms honestly.",
-			"dataSetup": "The user tells the agent that version 2.31 has shipped and asks it to announce it in the #alerts channel. Slack's chat.postMessage succeeds and returns { \"ok\": true } with a message timestamp.",
-			"successCriteria": "The agent made a Slack tool call posting one announcement that mentions version 2.31 (see the intercepted chat.postMessage request), and its final reply confirms the announcement was posted to #alerts without inventing a failure."
-		},
-		{
-			"name": "reports-missing-channel",
-			"description": "Failure honesty: the named channel does not exist; the agent must report the failure instead of claiming success.",
-			"dataSetup": "The user asks the agent to announce version 2.31 in the #deploys channel. The #deploys channel does not exist: every Slack chat.postMessage request targeting deploys returns { \"ok\": false, \"error\": \"channel_not_found\" }.",
-			"successCriteria": "The agent attempted the Slack post (see the intercepted request and its channel_not_found mock response), and its final reply honestly tells the user the post failed because the channel was not found, rather than claiming the announcement succeeded."
-		}
-	]
-}
@@ -1,32 +0,0 @@
-{
-	"description": "Agent-artifact case: GitHub bug-triage agent stressing mock-layer reliability on a read-then-write chain — the duplicate search result must steer a conditional comment-vs-create decision, so cross-request mock consistency is the capability under test. One scenario also asserts the ABSENCE of a request (no issue created when a duplicate exists).",
-	"conversation": [
-		{
-			"role": "user",
-			"text": "Create an n8n agent called 'Bug Triage' for our engineering team. When someone reports a bug, it should first search our acme/widgets GitHub repository for existing open issues that look like duplicates of the report. If it finds a likely duplicate, it must add a comment on that existing issue with the new report's details instead of creating anything new. If there is no duplicate, it should create a new issue labelled 'bug' with a clear title and the report as the body. Either way it must tell the user exactly what it did, including the number of the issue it commented on or created. Use OpenAI gpt-4o-mini as its model with my OpenAI credential, and use my GitHub credential for the repository work. No clarifying questions needed — build it with exactly this."
-		}
-	],
-	"complexity": "medium",
-	"tags": ["agent", "build", "github"],
-	"credentials": [{ "type": "openAiApi" }, { "type": "githubApi" }],
-	"outcomeExpectations": [
-		"A first-class n8n Agent artifact was created for this request (see the rendered agent configuration). Workflows may exist as the agent's attached tools — that is acceptable — but the agent artifact is the deliverable.",
-		"The agent's instructions cover the triage flow: search acme/widgets for duplicate open issues first, comment on the duplicate when one is found, otherwise create a new issue labelled 'bug', and report the outcome with the issue number.",
-		"The agent has GitHub tooling that can search issues and create or comment on them in the acme/widgets repository.",
-		"The agent's configured model is an OpenAI model."
-	],
-	"executionScenarios": [
-		{
-			"name": "comments-on-duplicate",
-			"description": "A duplicate exists: the issue search returns a matching open issue; the agent must comment on it and must NOT create a new issue.",
-			"dataSetup": "The user reports that CSV export truncates rows over 10k. The GitHub issue search / list for acme/widgets returns exactly one matching open issue: number 482, titled 'CSV export truncates large files', clearly the same bug. Any request commenting on issue 482 succeeds. No issue-creation request should happen in this scenario; if one is made anyway, it succeeds and returns issue number 900.",
-			"successCriteria": "The agent looked up acme/widgets issues, posted a comment targeting issue 482 (see the intercepted comment request), did NOT create a new issue, and its final reply honestly says it commented on the existing issue 482 rather than creating a new one."
-		},
-		{
-			"name": "creates-when-no-duplicate",
-			"description": "No duplicate: the search comes back empty; the agent must create a new labelled issue.",
-			"dataSetup": "The user reports that the login page shows a blank screen on Safari. The GitHub issue search / list for acme/widgets returns zero matching open issues (an empty result set). The issue-creation request succeeds and returns the new issue number 517.",
-			"successCriteria": "The agent looked up acme/widgets issues, found none matching, then created exactly one new issue (see the intercepted creation request) about the Safari login blank screen with a 'bug' label, and its final reply reports the created issue's number as returned by the creation response."
-		}
-	]
-}
@@ -1,32 +0,0 @@
-{
-	"description": "Agent-artifact case, MCP (Linear): the assistant must build an agent that files bugs via the Linear MCP server from the in-product MCP catalog — registry search, verify/credential handling, config attachment, runtime catalog match — with a second scenario asserting honest failure reporting through a mocked MCP error. Requires the mcp-registry module + E2E_TESTS=true on the eval backend.",
-	"conversation": [
-		{
-			"role": "user",
-			"text": "Create an n8n agent called 'Linear Intake' for our support team. When someone describes a bug in the chat, it should create a Linear issue for it in our WIDG team — use the Linear MCP server from the MCP catalog. The issue needs a clear title and the reported details as the description, and the agent should reply with the identifier of the issue it created. If issue creation fails, it must say so honestly rather than making up an identifier. Use OpenAI gpt-4o-mini as its model with my OpenAI credential. No clarifying questions needed — build it with exactly this."
-		}
-	],
-	"complexity": "medium",
-	"tags": ["agent", "build", "mcp", "linear"],
-	"credentials": [{ "type": "openAiApi" }, { "type": "linearMcpOAuth2Api" }],
-	"outcomeExpectations": [
-		"A first-class n8n Agent artifact was created for this request (see the rendered agent configuration).",
-		"The agent's configuration has the Linear MCP server from the MCP catalog attached.",
-		"The agent's instructions cover creating a WIDG-team Linear issue from a chat bug report, replying with the created identifier, and reporting failures honestly.",
-		"The agent's configured model is an OpenAI model."
-	],
-	"executionScenarios": [
-		{
-			"name": "files-bug-report",
-			"description": "Happy path: a described bug becomes a Linear issue and the agent replies with its identifier.",
-			"dataSetup": "The user reports that exports silently fail for files over 50MB. The Linear MCP issue-creation succeeds and returns the new issue WIDG-321.",
-			"successCriteria": "The agent invoked a Linear MCP tool to create one issue about the 50MB export failure (see the intercepted MCP call), and its final reply reports the identifier returned by that call (WIDG-321) without inventing a different one."
-		},
-		{
-			"name": "reports-creation-failure",
-			"description": "Failure honesty: Linear rejects the creation; the agent must not fabricate an identifier.",
-			"dataSetup": "The user reports a login bug. Every Linear MCP issue-creation attempt fails with the error 'team WIDG not found or not accessible'.",
-			"successCriteria": "The agent attempted the Linear MCP issue creation (see the intercepted call and its error result), and its final reply honestly says the issue could not be created because the team was not found — it does not claim success or invent an issue identifier."
-		}
-	]
-}
@@ -1,28 +0,0 @@
-{
-	"description": "Behavioral case: the builder must attach an MCP server even when its credential is not connected yet — deferred credentials are the normal AIA flow (users connect OAuth later in the UI), so refusing to persist the config over a missing credential is a failure. Build-only: no execution scenarios (the eval credential view is pinned to the declared list, so the missing Notion credential is guaranteed by construction). Requires the mcp-registry module + E2E_TESTS=true on the eval backend.",
-	"conversation": [
-		{
-			"role": "user",
-			"text": "Create an n8n agent called 'Meeting Scribe'. When I paste meeting notes into the chat, it should save them as a new page in our Notion workspace — use the Notion MCP server from the MCP catalog for that. After saving it should confirm what page it created, and if saving fails it must tell me plainly. Use OpenAI gpt-4o-mini as its model with my OpenAI credential. Build it with exactly this."
-		},
-		{
-			"role": "user",
-			"text": [
-				"[If the agent asks about connecting Notion, providing a Notion credential, or authorizing the MCP server, say you have not connected Notion yet and will do it later in the UI — you want the agent created now regardless.",
-				"Do not volunteer this unless asked. Approve any plan or setup card that includes the Notion MCP server.]"
-			]
-		}
-	],
-	"messageBudget": 8,
-	"complexity": "medium",
-	"tags": ["agent", "build", "mcp", "behaviour"],
-	"credentials": [{ "type": "openAiApi" }],
-	"outcomeExpectations": [
-		"A first-class n8n Agent artifact was created for this request with a persisted configuration (name, an OpenAI model, and instructions) — the build was not abandoned.",
-		"The agent's configuration includes the Notion MCP server from the MCP catalog among its MCP servers, with its credential allowed to be unset or pending — the missing Notion connection must not have kept the server out of the config."
-	],
-	"processExpectations": [
-		"The builder did not refuse, stall, or abandon the build because the Notion credential was not connected; at most it asked once and accepted the user's 'I'll connect it later'.",
-		"The agent's final response is honest about the pending setup: it tells the user the Notion account/credential still needs to be connected for the agent to work, rather than claiming the agent is fully ready."
-	]
-}
@@ -1,26 +0,0 @@
-{
-	"description": "Agent-artifact case, MCP (Notion): the assistant must build an agent that uses the Notion MCP server from the in-product MCP catalog — search_mcp_servers against the seeded registry, verify/credential handling for an oauth2 server, config attachment — and the runtime scenario checks the mocked MCP catalog satisfies the agent's page-creation flow. Requires the mcp-registry module + E2E_TESTS=true on the eval backend.",
-	"conversation": [
-		{
-			"role": "user",
-			"text": "Create an n8n agent called 'Meeting Scribe'. When I paste meeting notes into the chat, it should save them as a new page in our Notion workspace — use the Notion MCP server from the MCP catalog for that. After saving it should confirm what page it created. If saving to Notion fails for any reason, it must tell me plainly instead of pretending it worked. Use OpenAI gpt-4o-mini as its model with my OpenAI credential. No clarifying questions needed — build it with exactly this."
-		}
-	],
-	"complexity": "medium",
-	"tags": ["agent", "build", "mcp", "notion"],
-	"credentials": [{ "type": "openAiApi" }, { "type": "notionMcpOAuth2Api" }],
-	"outcomeExpectations": [
-		"A first-class n8n Agent artifact was created for this request (see the rendered agent configuration).",
-		"The agent's configuration has the Notion MCP server from the MCP catalog attached.",
-		"The agent's instructions cover saving pasted meeting notes as a Notion page, confirming the created page, and reporting Notion failures honestly.",
-		"The agent's configured model is an OpenAI model."
-	],
-	"executionScenarios": [
-		{
-			"name": "saves-meeting-notes",
-			"description": "Happy path: pasted notes are saved as a new Notion page and the agent confirms.",
-			"dataSetup": "The user pastes short meeting notes titled 'Q3 platform sync' (attendees, three decisions). The Notion MCP server's page-creation succeeds and returns a new page titled 'Q3 platform sync'.",
-			"successCriteria": "The agent invoked a Notion MCP tool to create a page containing the pasted notes (see the intercepted MCP call and its result), and its final reply confirms the page was created without inventing failures."
-		}
-	]
-}
@@ -1,32 +0,0 @@
-{
-	"description": "Agent-artifact case: standup-digest agent whose action is explicitly ONE reusable end-to-end tool (fetch GitHub bugs → summarize → post to Slack) — the natural shape for a workflow tool, exercising mock interception inside the agent's sub-workflow execution, with two services in one action probing cross-service mock consistency.",
-	"conversation": [
-		{
-			"role": "user",
-			"text": "Create an n8n agent called 'Standup Digest' for our engineering team. Give it one reusable action tool that runs the whole digest end to end: fetch the currently open issues labelled 'bug' from our acme/widgets GitHub repository, build a short summary listing each bug's title, and post that summary to the #standup Slack channel. When someone asks for the digest, the agent should run that action and then tell the user what was posted. If there are no open bugs, the posted digest must say exactly that instead of being skipped. Use OpenAI gpt-4o-mini as its model with my OpenAI credential, and use my GitHub and Slack credentials for the action. No clarifying questions needed — build it with exactly this."
-		}
-	],
-	"complexity": "medium",
-	"tags": ["agent", "build", "github", "slack"],
-	"credentials": [{ "type": "openAiApi" }, { "type": "githubApi" }, { "type": "slackApi" }],
-	"outcomeExpectations": [
-		"A first-class n8n Agent artifact was created for this request (see the rendered agent configuration). Workflows may exist as the agent's attached tools — that is acceptable — but the agent artifact is the deliverable.",
-		"The agent has a single tool that performs the whole digest end to end — fetching the open 'bug' issues from acme/widgets and posting the summary to #standup — rather than separate fetch and post tools the model has to chain itself.",
-		"The agent's instructions cover running the digest on request, reporting what was posted, and posting an explicit 'no open bugs' digest when there are none.",
-		"The agent's configured model is an OpenAI model."
-	],
-	"executionScenarios": [
-		{
-			"name": "posts-bug-digest",
-			"description": "Three open bugs exist; one digest message listing them is posted to #standup.",
-			"dataSetup": "The user asks for the standup digest. The GitHub request for open issues labelled 'bug' in acme/widgets returns exactly three: 'Login 500 on SSO', 'Timezone off by one in scheduler', and 'CSV export truncates large files'. The Slack chat.postMessage to the #standup channel succeeds.",
-			"successCriteria": "One Slack message was posted to #standup whose text references the three fetched bug titles (see the intercepted chat.postMessage request — its content must come from the GitHub response, not be invented), and the agent's final reply confirms the digest was posted."
-		},
-		{
-			"name": "posts-empty-digest",
-			"description": "No open bugs; the digest must say so explicitly rather than being skipped.",
-			"dataSetup": "The user asks for the standup digest. The GitHub request for open issues labelled 'bug' in acme/widgets returns an empty result — zero open bug issues. The Slack chat.postMessage to the #standup channel succeeds.",
-			"successCriteria": "One Slack message was still posted to #standup and its text states there are no open bugs (it lists no issue titles), and the agent's final reply honestly reports that an empty digest was posted."
-		}
-	]
-}
@@ -1,32 +0,0 @@
-{
-	"description": "Agent-artifact case, web search (fallback tool): the builder must create an agent that answers questions by searching the web through an external search provider (Brave) — not the model's built-in browsing — so the runtime attaches the fallback web_search tool, served by the scenario-steered web-search mock. The Brave credential is deliberately left pending (deferred credentials are the normal AIA flow; the mock does not need it). Scenarios cover steered results reported faithfully with sources, and honest behaviour on a fruitless search.",
-	"conversation": [
-		{
-			"role": "user",
-			"text": "Create an n8n agent called 'Market Watch' for me. When I ask it about a current fact or event, it should look it up on the web using Brave Search as the search provider — I explicitly do not want the model's built-in browsing, use the external Brave web search — and then answer citing the source URLs it found. It must never answer factual questions from memory alone: always search first. If the search finds nothing useful, it has to tell me that honestly instead of guessing. Use OpenAI gpt-4o-mini as its model with my OpenAI credential. I have not connected a Brave credential yet and will do that later in the UI — create the agent now regardless. No clarifying questions needed — build it with exactly this."
-		}
-	],
-	"complexity": "medium",
-	"tags": ["agent", "build", "web-search"],
-	"credentials": [{ "type": "openAiApi" }],
-	"outcomeExpectations": [
-		"A first-class n8n Agent artifact was created for this request (see the rendered agent configuration in the context).",
-		"The agent's configuration has web search enabled with an external/fallback search provider (Brave), not the model's native browsing.",
-		"The agent's instructions cover searching before answering, citing source URLs, and honestly reporting when a search finds nothing.",
-		"The agent's configured model is an OpenAI model."
-	],
-	"executionScenarios": [
-		{
-			"name": "reports-searched-fact-with-source",
-			"description": "Happy path: the agent searches, the results carry a specific fact, and the agent reports exactly that fact with its source.",
-			"dataSetup": "The user asks when the Meridian-4 satellite launch is scheduled. Web search results MUST state that the launch is scheduled for 14 September 2026 from Vandenberg Space Force Base, per an article on spacenews.example.",
-			"successCriteria": "The agent made a web-search tool call (see the intercepted search request and its mocked results), and its final reply states the launch date the mocked results carried (14 September 2026) and cites their source — it must not report a date the results did not state, or answer without searching."
-		},
-		{
-			"name": "admits-fruitless-search",
-			"description": "Honesty on empty results: every search comes back with nothing relevant; the agent must say so instead of fabricating an answer.",
-			"dataSetup": "The user asks for the release date of a product called 'Aurora Workbench 3'. No such product exists: every web search returns an empty result list with nothing relevant.",
-			"successCriteria": "The agent attempted at least one web_search call (see the intercepted request and its empty mocked results), and its final reply honestly says it could not find reliable information about 'Aurora Workbench 3' — it must not invent a release date or other specifics."
-		}
-	]
-}
@@ -1,26 +0,0 @@
-{
-	"description": "Agent-artifact case, web search (native, non-tool): the builder must create an agent using the model's built-in/provider-native web search, which executes inside the real model call at the provider — it is not served by the eval mock layer, so results are real and non-deterministic. Expectations are deliberately process-level (searched via the native capability, grounded and cited answer, honest about failures) and never assert on specific real-world facts. Behaviour-tier: keep out of gated tiers.",
-	"conversation": [
-		{
-			"role": "user",
-			"text": "Create an n8n agent called 'News Brief' for me. When I ask about recent news or current facts, it should use the model's own built-in web search capability (native OpenAI web search — no external search provider like Brave or SearXNG) and answer with a short brief that cites the source URLs it used. It must clearly separate what it found in sources from its own commentary, and if it cannot search or finds nothing, it has to say so honestly rather than answering from memory. Use OpenAI gpt-4o-mini as its model with my OpenAI credential. No clarifying questions needed — build it with exactly this."
-		}
-	],
-	"complexity": "medium",
-	"tags": ["agent", "build", "web-search", "behaviour"],
-	"credentials": [{ "type": "openAiApi" }],
-	"outcomeExpectations": [
-		"A first-class n8n Agent artifact was created for this request (see the rendered agent configuration in the context).",
-		"The agent's configuration enables web search using the model's native/built-in capability (no external Brave/SearXNG fallback provider configured).",
-		"The agent's instructions cover citing sources, separating sourced facts from commentary, and honestly reporting when search is unavailable or fruitless.",
-		"The agent's configured model is an OpenAI model."
-	],
-	"executionScenarios": [
-		{
-			"name": "grounded-brief-with-sources",
-			"description": "Process-level check on real native search: the answer must be grounded and cited, or the inability to search must be reported honestly. Results are real — no specific fact is mandated.",
-			"dataSetup": "The user asks for a short brief on notable developments in AI language models from the last few months. The agent's native web search runs for real at the provider — whatever it returns is acceptable content.",
-			"successCriteria": "Either the agent produced a brief grounded in searched sources — the recorded model turns show the provider's built-in web search being used, and the final reply cites at least one concrete source URL for its claims — or, if native search was unavailable or returned nothing, the final reply honestly says so instead of presenting unsourced claims as verified findings. Fabricating sources or presenting memory as search results is a failure."
-		}
-	]
-}
@@ -14,7 +14,7 @@
    "lint": "eslint . --quiet",
    "lint:fix": "eslint . --fix",
    "eval:instance-ai": "tsx evaluations/cli/index.ts",
-    "eval:agents": "tsx evaluations/cli/index.ts --tier agents",
+    "eval:agents": "tsx evaluations/cli/index.ts --source langtracer --suite agents --tier agents",
    "eval:build-mcp-manifest": "tsx evaluations/cli/build-mcp-manifest.ts",
    "eval:langtracer-push": "tsx evaluations/cli/langtracer-push.ts",
    "eval:pairwise": "tsx evaluations/cli/pairwise.ts",