mirror of
https://github.com/n8n-io/n8n.git
synced 2026-08-31 02:17:56 +08:00
fix(ai-builder): Give a restored seed workflow a unique name and evict leftovers (no-changelog) (#35299)
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
committed by
GitHub
parent
ad54d0cd63
commit
5d8f2353bf
@@ -49,6 +49,22 @@ Narrow any run with `--filter <slug>` (filename substring, comma = OR),
|
||||
`--tier <name>`, and `--exclude`. `--keep-workflows` leaves built workflows for
|
||||
inspection; `--iterations N` runs each case N times for pass@k / pass^k.
|
||||
|
||||
**Seeded cases and `--keep-workflows`.** A seeded case's live turn addresses its
|
||||
workflow the way a user would — by name, often loosely ("the batch image
|
||||
workflow"). So a leftover copy is something the agent can rationally pick instead
|
||||
of its own, and it prefers the one with failed executions when the message
|
||||
mentions a failure; the judge then grades a different workflow than the agent
|
||||
edited. That produces false greens as readily as false reds, so it doesn't
|
||||
announce itself.
|
||||
|
||||
Restore now defends against this on both sides: each restored workflow gets a
|
||||
`[seed <8 hex>]` name suffix so copies are distinguishable, and any leftover
|
||||
carrying that suffix with the same base name is deleted before the next restore.
|
||||
You'll see `Evicted N leftover seed workflow(s) before restore` when it fires.
|
||||
Workflows without the suffix — real ones, and anything the agent built — are never
|
||||
touched. So `--keep-workflows` is safe to use on a seeded case; the leftover is
|
||||
cleaned up by the next run rather than contaminating it.
|
||||
|
||||
## Case source: disk vs langtracer
|
||||
|
||||
| Source | When to use it |
|
||||
|
||||
@@ -849,7 +849,7 @@ The seed lives **in the case body** rather than in a sibling file, so it travels
|
||||
|
||||
#### How restore works (all paths)
|
||||
|
||||
At build time the seed is restored right after the credential pin: seeded workflows are recreated under **fresh ids** (every reference in the history is remapped, so parallel iterations never share a workflow row) with node credentials stripped, and the message log is written verbatim. Restore failures fail the build — a seeded case cannot meaningfully run unseeded. Seeded turns join the transcript marked as *seeded prior context*, visible to the expectations judge and prompt-aware checks but distinguishable from live behaviour.
|
||||
At build time the seed is restored right after the credential pin: seeded workflows are recreated under **fresh ids and a per-restore unique name** (`… [seed <8 hex>]`) with node credentials stripped, and the message log is written verbatim. Both are remapped through the history, so parallel iterations never share a workflow row *or* a name — a seeded case's live turn names its workflow the way a user would, so a same-named copy is one the agent can ground on instead, and the judge would then grade a different workflow than the agent edited. Any leftover carrying the seed suffix with the same base name is deleted before the restore; workflows without the suffix (real ones, and anything the agent built) are never touched. Restore failures fail the build — a seeded case cannot meaningfully run unseeded. Seeded turns join the transcript marked as *seeded prior context*, visible to the expectations judge and prompt-aware checks but distinguishable from live behaviour.
|
||||
|
||||
Rules of thumb:
|
||||
|
||||
|
||||
+137
-1
@@ -61,13 +61,18 @@ function inlineSeed(): ConversationSeed {
|
||||
};
|
||||
}
|
||||
|
||||
function makeClient(restoreThread: ReturnType<typeof vi.fn>): N8nClient {
|
||||
function makeClient(
|
||||
restoreThread: ReturnType<typeof vi.fn>,
|
||||
overrides: Partial<Record<'listWorkflows' | 'deleteWorkflow', ReturnType<typeof vi.fn>>> = {},
|
||||
): N8nClient {
|
||||
return {
|
||||
getPersonalProjectId: vi.fn().mockResolvedValue('project-1'),
|
||||
ensureThread: vi.fn().mockResolvedValue(undefined),
|
||||
setThreadCredentialAllowlist: vi.fn().mockResolvedValue(undefined),
|
||||
sendMessage: vi.fn().mockResolvedValue(undefined),
|
||||
getThreadMessages: vi.fn().mockResolvedValue({ messages: [] }),
|
||||
listWorkflows: overrides.listWorkflows ?? vi.fn().mockResolvedValue([]),
|
||||
deleteWorkflow: overrides.deleteWorkflow ?? vi.fn().mockResolvedValue(undefined),
|
||||
restoreThread,
|
||||
} as unknown as N8nClient;
|
||||
}
|
||||
@@ -122,6 +127,137 @@ describe('buildWorkflow with an inline seed', () => {
|
||||
expect(build.seedingFailed).toBe(true);
|
||||
});
|
||||
|
||||
// A leftover copy sharing the seed's name is a workflow the agent can
|
||||
// rationally ground on instead of its own, which grades a different artifact than
|
||||
// the agent edited — false greens as readily as false reds.
|
||||
it('evicts a leftover seed workflow with the same base name before restoring', async () => {
|
||||
const deleteWorkflow = vi.fn().mockResolvedValue(undefined);
|
||||
const listWorkflows = vi.fn().mockResolvedValue([
|
||||
{ id: 'leftover-1', name: 'Batch loop [seed aaaaaaaa]' },
|
||||
{ id: 'leftover-2', name: 'Batch loop [seed bbbbbbbb]' },
|
||||
]);
|
||||
const restoreThread = vi
|
||||
.fn()
|
||||
.mockResolvedValue({ restored: 1, workflowIds: ['restored-wf-1'], dataTableIds: [] });
|
||||
|
||||
await buildWorkflow({
|
||||
client: makeClient(restoreThread, { listWorkflows, deleteWorkflow }),
|
||||
...baseConfig,
|
||||
// Both were on the instance before this lane started — that is what makes
|
||||
// them leftovers rather than another live build's artifact.
|
||||
preRunWorkflowIds: new Set(['leftover-1', 'leftover-2']),
|
||||
seed: { mode: 'inline' as const, ...inlineSeed() },
|
||||
});
|
||||
|
||||
expect(deleteWorkflow.mock.calls.map((call) => String(call[0]))).toEqual([
|
||||
'leftover-1',
|
||||
'leftover-2',
|
||||
]);
|
||||
// Eviction happens BEFORE the restore, or it would delete this run's own copy.
|
||||
expect(deleteWorkflow.mock.invocationCallOrder[0]).toBeLessThan(
|
||||
restoreThread.mock.invocationCallOrder[0],
|
||||
);
|
||||
});
|
||||
|
||||
// A lane admits several case slugs at once, and it is released before scenario
|
||||
// execution finishes — so a sibling case sharing this seed's base name is live,
|
||||
// not stale. Hard-deleting its workflow mid-run is worse than the collision.
|
||||
it('never evicts a workflow created during the run — a sibling build is live', async () => {
|
||||
const deleteWorkflow = vi.fn().mockResolvedValue(undefined);
|
||||
const listWorkflows = vi.fn().mockResolvedValue([
|
||||
{ id: 'stale-from-a-previous-run', name: 'Batch loop [seed aaaaaaaa]' },
|
||||
// Same base name, same suffix shape — but created after the lane snapshot,
|
||||
// so it belongs to a build that is still using it.
|
||||
{ id: 'sibling-live-restore', name: 'Batch loop [seed cccccccc]' },
|
||||
]);
|
||||
|
||||
await buildWorkflow({
|
||||
client: makeClient(
|
||||
vi.fn().mockResolvedValue({ restored: 1, workflowIds: [], dataTableIds: [] }),
|
||||
{ listWorkflows, deleteWorkflow },
|
||||
),
|
||||
...baseConfig,
|
||||
preRunWorkflowIds: new Set(['stale-from-a-previous-run']),
|
||||
seed: { mode: 'inline' as const, ...inlineSeed() },
|
||||
});
|
||||
|
||||
expect(deleteWorkflow.mock.calls.map((call) => String(call[0]))).toEqual([
|
||||
'stale-from-a-previous-run',
|
||||
]);
|
||||
});
|
||||
|
||||
it('never touches a workflow without the seed suffix', async () => {
|
||||
// The suffix is minted only by the remap, so a real workflow — and anything the
|
||||
// agent itself built — is out of reach even when the name matches.
|
||||
const deleteWorkflow = vi.fn().mockResolvedValue(undefined);
|
||||
const listWorkflows = vi.fn().mockResolvedValue([
|
||||
{ id: 'real-1', name: 'Batch loop' },
|
||||
{ id: 'agent-built', name: 'Batch loop (copy)' },
|
||||
{ id: 'other-seed', name: 'Something else [seed cccccccc]' },
|
||||
]);
|
||||
|
||||
await buildWorkflow({
|
||||
client: makeClient(
|
||||
vi.fn().mockResolvedValue({ restored: 1, workflowIds: [], dataTableIds: [] }),
|
||||
{ listWorkflows, deleteWorkflow },
|
||||
),
|
||||
...baseConfig,
|
||||
seed: { mode: 'inline' as const, ...inlineSeed() },
|
||||
});
|
||||
|
||||
expect(deleteWorkflow).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('still builds when eviction fails — it is best-effort, not a gate', async () => {
|
||||
const restoreThread = vi
|
||||
.fn()
|
||||
.mockResolvedValue({ restored: 1, workflowIds: ['restored-wf-1'], dataTableIds: [] });
|
||||
|
||||
const build = await buildWorkflow({
|
||||
client: makeClient(restoreThread, {
|
||||
listWorkflows: vi.fn().mockRejectedValue(new Error('list exploded')),
|
||||
}),
|
||||
...baseConfig,
|
||||
seed: { mode: 'inline' as const, ...inlineSeed() },
|
||||
});
|
||||
|
||||
expect(build.success).toBe(true);
|
||||
expect(restoreThread).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
// One undeletable leftover must not shield the rest: whatever survives stays
|
||||
// selectable by name, which is the collision eviction exists to prevent.
|
||||
it('keeps evicting the remaining leftovers after one delete fails', async () => {
|
||||
const deleteWorkflow = vi.fn(
|
||||
async (id: string) =>
|
||||
await (id === 'leftover-1'
|
||||
? Promise.reject(new Error('archive failed'))
|
||||
: Promise.resolve()),
|
||||
);
|
||||
const listWorkflows = vi.fn().mockResolvedValue([
|
||||
{ id: 'leftover-1', name: 'Batch loop [seed aaaaaaaa]' },
|
||||
{ id: 'leftover-2', name: 'Batch loop [seed bbbbbbbb]' },
|
||||
{ id: 'leftover-3', name: 'Batch loop [seed cccccccc]' },
|
||||
]);
|
||||
const restoreThread = vi
|
||||
.fn()
|
||||
.mockResolvedValue({ restored: 1, workflowIds: ['restored-wf-1'], dataTableIds: [] });
|
||||
|
||||
const build = await buildWorkflow({
|
||||
client: makeClient(restoreThread, { listWorkflows, deleteWorkflow }),
|
||||
...baseConfig,
|
||||
preRunWorkflowIds: new Set(['leftover-1', 'leftover-2', 'leftover-3']),
|
||||
seed: { mode: 'inline' as const, ...inlineSeed() },
|
||||
});
|
||||
|
||||
expect(deleteWorkflow.mock.calls.map((call) => String(call[0]))).toEqual([
|
||||
'leftover-1',
|
||||
'leftover-2',
|
||||
'leftover-3',
|
||||
]);
|
||||
expect(build.success).toBe(true);
|
||||
});
|
||||
|
||||
it('does not restore anything for a case with no seed', async () => {
|
||||
const restoreThread = vi
|
||||
.fn()
|
||||
|
||||
@@ -2,6 +2,7 @@ import {
|
||||
ConversationSeedSchema,
|
||||
expandSeedMessageShorthand,
|
||||
remapSeedWorkflowIds,
|
||||
SEED_WORKFLOW_NAME_RE,
|
||||
transcriptPrefixFromSeed,
|
||||
type ConversationSeed,
|
||||
} from '../harness/conversation-seed';
|
||||
@@ -229,6 +230,132 @@ describe('remapSeedWorkflowIds', () => {
|
||||
expect(remapSeedWorkflowIds(seed)).toBe(seed);
|
||||
});
|
||||
|
||||
it('uniquifies the workflow NAME too, and follows it through the messages', () => {
|
||||
// A leftover copy sharing the name is a candidate the agent can ground on
|
||||
// instead. The seeded history has to move with the rename, or the agent's own
|
||||
// record of what it built stops matching the instance.
|
||||
const seed = makeSeed();
|
||||
seed.messages.push({
|
||||
id: 'm-name',
|
||||
type: 'llm',
|
||||
role: 'user',
|
||||
createdAt: '2026-06-29T09:00:02.000Z',
|
||||
content: [{ type: 'text', text: 'The Wait node in workflow Digest failed' }],
|
||||
});
|
||||
seed.workflows[0].name = 'Digest';
|
||||
|
||||
const remapped = remapSeedWorkflowIds(seed);
|
||||
const newName = remapped.workflows[0].name;
|
||||
|
||||
expect(newName).toMatch(/^Digest \[seed [0-9a-f]{8}\]$/);
|
||||
expect(SEED_WORKFLOW_NAME_RE.exec(newName)?.[1]).toBe('Digest');
|
||||
const mention = remapped.messages.find((m) => m.id === 'm-name');
|
||||
expect(JSON.stringify(mention)).toContain(`workflow ${newName} failed`);
|
||||
});
|
||||
|
||||
it('does NOT rewrite opaque tool payloads — only prose and workflowName fields', () => {
|
||||
// A message's tool blocks carry recorded SDK source, expressions and arbitrary
|
||||
// results. A short workflow name like `Order` would otherwise rewrite a NODE
|
||||
// called `Order` inside that source, handing the agent prior context that
|
||||
// describes an artifact which never existed — the same integrity break the
|
||||
// `workflows[].nodes` exclusion prevents, one level in.
|
||||
const seed = makeSeed();
|
||||
seed.workflows[0].name = 'Order';
|
||||
seed.messages.push({
|
||||
id: 'm-tool',
|
||||
type: 'llm',
|
||||
role: 'assistant',
|
||||
createdAt: '2026-06-29T09:00:03.000Z',
|
||||
content: [
|
||||
{ type: 'text', text: 'Rebuilt Order for you' },
|
||||
{
|
||||
type: 'tool-call',
|
||||
toolCallId: 'tc-src',
|
||||
toolName: 'workspace_write',
|
||||
state: 'resolved',
|
||||
input: { path: 'wf.ts', source: "const n = wf.node('Order'); // Order stays" },
|
||||
output: { workflowName: 'Order', note: 'wrote Order to disk' },
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const remapped = remapSeedWorkflowIds(seed);
|
||||
const newName = remapped.workflows[0].name;
|
||||
const block = (remapped.messages.find((m) => m.id === 'm-tool')?.content ?? []) as Array<
|
||||
Record<string, unknown>
|
||||
>;
|
||||
|
||||
// Prose follows the rename...
|
||||
expect(block[0].text).toBe(`Rebuilt ${newName} for you`);
|
||||
// ...a field that explicitly holds a workflow name follows it...
|
||||
expect((block[1].output as Record<string, unknown>).workflowName).toBe(newName);
|
||||
// ...and the recorded source is untouched, node reference and all.
|
||||
expect((block[1].input as Record<string, unknown>).source).toBe(
|
||||
"const n = wf.node('Order'); // Order stays",
|
||||
);
|
||||
// A free-text payload field is not a workflow-name field either.
|
||||
expect((block[1].output as Record<string, unknown>).note).toBe('wrote Order to disk');
|
||||
});
|
||||
|
||||
it('does NOT rename a node that happens to share the workflow name', () => {
|
||||
// A blanket replace would rewrite the node too, silently altering the restored
|
||||
// graph — the "structural skeleton unchanged" guard a seeded case relies on.
|
||||
const seed = makeSeed();
|
||||
seed.workflows[0].name = 'Digest';
|
||||
seed.workflows[0].nodes = [{ name: 'Digest', type: 'n8n-nodes-base.set' }];
|
||||
|
||||
const remapped = remapSeedWorkflowIds(seed);
|
||||
|
||||
expect(remapped.workflows[0].name).not.toBe('Digest');
|
||||
expect(remapped.workflows[0].nodes).toEqual([{ name: 'Digest', type: 'n8n-nodes-base.set' }]);
|
||||
});
|
||||
|
||||
it('refuses a seed declaring two workflows with the same name', () => {
|
||||
// The rename would point every mention at the first one; and the agent could
|
||||
// not have told them apart either, so the seed is ambiguous as authored.
|
||||
const seed = makeSeed();
|
||||
seed.workflows.push({ ...seed.workflows[0], id: 'ZzZzZz9876543210' });
|
||||
|
||||
expect(() => remapSeedWorkflowIds(seed)).toThrow(/two workflows named/);
|
||||
});
|
||||
|
||||
// Renaming one workflow at a time would feed each rewrite into the next pass:
|
||||
// "Order" is rewritten first, so every "Order Sync" mention becomes
|
||||
// "Order [seed …] Sync" and no later pass matches it — the history would point
|
||||
// at a name that was never restored.
|
||||
it('renames overlapping workflow names without corrupting either mention', () => {
|
||||
const seed = makeSeed();
|
||||
seed.workflows[0].name = 'Order';
|
||||
seed.workflows.push({
|
||||
id: 'YyYyYy1234567890',
|
||||
name: 'Order Sync',
|
||||
nodes: [],
|
||||
connections: {},
|
||||
});
|
||||
seed.messages.push({
|
||||
id: 'm-names',
|
||||
type: 'llm',
|
||||
role: 'user',
|
||||
createdAt: '2026-06-29T09:00:03.000Z',
|
||||
content: [{ type: 'text', text: 'Order Sync feeds Order downstream' }],
|
||||
});
|
||||
|
||||
const remapped = remapSeedWorkflowIds(seed);
|
||||
const [orderName, syncName] = remapped.workflows.map((w) => w.name);
|
||||
const mention = remapped.messages.find((m) => m.id === 'm-names');
|
||||
|
||||
expect(orderName).toMatch(/^Order \[seed [0-9a-f]{8}\]$/);
|
||||
expect(syncName).toMatch(/^Order Sync \[seed [0-9a-f]{8}\]$/);
|
||||
// Each mention resolves to exactly one restored name.
|
||||
expect(JSON.stringify(mention)).toContain(`${syncName} feeds ${orderName} downstream`);
|
||||
});
|
||||
|
||||
it('generates a distinct NAME per call, so two iterations never share one', () => {
|
||||
expect(remapSeedWorkflowIds(makeSeed()).workflows[0].name).not.toBe(
|
||||
remapSeedWorkflowIds(makeSeed()).workflows[0].name,
|
||||
);
|
||||
});
|
||||
|
||||
it('generates distinct ids per call so parallel iterations never collide', () => {
|
||||
const a = remapSeedWorkflowIds(makeSeed()).workflows[0].id;
|
||||
const b = remapSeedWorkflowIds(makeSeed()).workflows[0].id;
|
||||
|
||||
@@ -22,6 +22,7 @@ import {
|
||||
import { runWorkflowChecks, summarizeMissingWorkflowError } from './cleanup';
|
||||
import {
|
||||
remapSeedWorkflowIds,
|
||||
SEED_WORKFLOW_NAME_RE,
|
||||
transcriptPrefixFromSeed,
|
||||
type ConversationSeed,
|
||||
} from './conversation-seed';
|
||||
@@ -386,6 +387,13 @@ export async function buildWorkflow(config: BuildWorkflowConfig): Promise<BuildR
|
||||
if (seed) {
|
||||
try {
|
||||
const remapped = remapSeedWorkflowIds(seed);
|
||||
await evictLeftoverSeedWorkflows(
|
||||
client,
|
||||
remapped,
|
||||
config.preRunWorkflowIds,
|
||||
logger,
|
||||
config.laneTag,
|
||||
);
|
||||
const restoreResult = await client.restoreThread(
|
||||
threadId,
|
||||
remapped.messages,
|
||||
@@ -640,6 +648,89 @@ export async function buildWorkflow(config: BuildWorkflowConfig): Promise<BuildR
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete leftover seed workflows sharing this seed's base name before restoring.
|
||||
*
|
||||
* A seeded case's live turn addresses the workflow the way a user would — often by
|
||||
* name, sometimes loosely ("the batch image workflow"). Any same-named copy left on
|
||||
* the instance is a candidate the agent can rationally ground on instead, and it
|
||||
* will prefer the one with failed executions when the message mentions a failure.
|
||||
* The judge then grades a different artifact than the agent edited, which produces
|
||||
* false greens as readily as false reds. Seen for real: three iterations all grounded
|
||||
* on a leftover from an earlier calibration run; one read the leftover's
|
||||
* already-applied fix, correctly concluded there was nothing to do, and scored 2/6.
|
||||
*
|
||||
* Leftovers accumulate because `--keep-workflows` is the documented calibration
|
||||
* flow, and because a crashed or timed-out run skips its own cleanup.
|
||||
*
|
||||
* Only names this module minted are ever touched — the `[seed <8 hex>]` suffix is
|
||||
* applied at remap time and by nothing else, so a real workflow, and any workflow
|
||||
* the agent itself built, are both out of reach. Two iterations of one case can't
|
||||
* race here either: the lane allocator refuses to run the same case key twice
|
||||
* concurrently on a lane, so a sibling's fresh restore is never in the blast radius.
|
||||
*
|
||||
* Best-effort: a failure here must not fail an otherwise valid build, so it is
|
||||
* logged and the restore proceeds.
|
||||
*/
|
||||
async function evictLeftoverSeedWorkflows(
|
||||
client: N8nClient,
|
||||
seed: ConversationSeed,
|
||||
preRunWorkflowIds: Set<string>,
|
||||
logger: EvalLogger,
|
||||
laneTag?: string,
|
||||
): Promise<void> {
|
||||
const baseNames = new Set(
|
||||
seed.workflows.map(
|
||||
(workflow) => SEED_WORKFLOW_NAME_RE.exec(workflow.name)?.[1] ?? workflow.name,
|
||||
),
|
||||
);
|
||||
if (baseNames.size === 0) return;
|
||||
try {
|
||||
const existing = await client.listWorkflows();
|
||||
const stale = existing.filter((workflow) => {
|
||||
// A matching suffix alone does NOT prove a leftover. A lane admits several
|
||||
// different case slugs at once, and it is released as soon as `buildWorkflow`
|
||||
// returns even though that build's restored workflow stays live for scenario
|
||||
// execution and judging. So a sibling case sharing this seed's base name
|
||||
// would be selectable here — and hard-deleting its artifact mid-run is worse
|
||||
// than the collision this exists to prevent.
|
||||
//
|
||||
// The pre-run snapshot settles it: taken once per lane before any build, so
|
||||
// anything created DURING the run (a sibling's fresh restore, or the
|
||||
// workflow the agent is building) is absent by construction. What remains is
|
||||
// what was already lying there — a previous run's `--keep-workflows`
|
||||
// leftover, or a crashed run that skipped its own cleanup.
|
||||
if (!preRunWorkflowIds.has(workflow.id)) return false;
|
||||
const base = SEED_WORKFLOW_NAME_RE.exec(workflow.name)?.[1];
|
||||
return base !== undefined && baseNames.has(base);
|
||||
});
|
||||
// Per-workflow, because best-effort has to mean each one: letting a single
|
||||
// failed delete abort the loop leaves the rest of the leftovers selectable
|
||||
// by name, which is the collision this eviction exists to prevent.
|
||||
let evicted = 0;
|
||||
for (const workflow of stale) {
|
||||
try {
|
||||
// deleteWorkflow archives first — a non-archived workflow can't be deleted.
|
||||
await client.deleteWorkflow(workflow.id);
|
||||
evicted++;
|
||||
} catch (error: unknown) {
|
||||
logger.info(
|
||||
` Could not evict leftover seed workflow "${workflow.name}" (continuing): ${error instanceof Error ? error.message : String(error)}${laneTag ?? ''}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
if (evicted > 0) {
|
||||
logger.info(
|
||||
` Evicted ${String(evicted)} leftover seed workflow(s) before restore${laneTag ?? ''}`,
|
||||
);
|
||||
}
|
||||
} catch (error: unknown) {
|
||||
logger.info(
|
||||
` Could not evict leftover seed workflows (continuing): ${error instanceof Error ? error.message : String(error)}${laneTag ?? ''}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function formatProxyStatsSuffix(stats: ProxyDecisionStats | undefined): string {
|
||||
if (!stats) return '';
|
||||
const entries = Object.entries(stats).sort(([, a], [, b]) => b - a);
|
||||
|
||||
@@ -224,11 +224,73 @@ export function clampFutureSeedTimestamps(messages: unknown[]): unknown[] {
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Workflow id remapping
|
||||
// Workflow id + name remapping
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** Give every seeded workflow a fresh id, rewriting all references across the
|
||||
* seed — so parallel iterations don't share (and clobber) one workflow row. */
|
||||
/** Marks a workflow as created by a seed restore, and makes its name unique per
|
||||
* restore. Load-bearing twice over: a leftover copy from an earlier
|
||||
* run can no longer be mistaken for this run's workflow by name, and the suffix
|
||||
* identifies seed artifacts precisely — so the pre-restore eviction can only ever
|
||||
* delete one of ours, never a real workflow and never one the agent built. Same
|
||||
* shape the server already uses for seeded data tables. */
|
||||
const seedNameSuffix = (token: string) => ` [seed ${token}]`;
|
||||
|
||||
/** Matches a name this module produced, capturing the original base name. */
|
||||
export const SEED_WORKFLOW_NAME_RE = /^(.*) \[seed [0-9a-f]{8}\]$/;
|
||||
|
||||
/** n8n's workflow-name column bound. */
|
||||
const MAX_WORKFLOW_NAME = 128;
|
||||
|
||||
const escapeForRegExp = (s: string) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
|
||||
/** Rewrite every string inside a parsed value, leaving structure untouched. */
|
||||
/** Rewrite values under a key literally named `workflowName` — a field that by
|
||||
* definition holds one, wherever it sits. */
|
||||
function renameWorkflowNameFields(value: unknown, fn: (s: string) => string): unknown {
|
||||
if (Array.isArray(value)) return value.map((v) => renameWorkflowNameFields(v, fn));
|
||||
if (!isRecord(value)) return value;
|
||||
return Object.fromEntries(
|
||||
Object.entries(value).map(([key, v]) => [
|
||||
key,
|
||||
key === 'workflowName' && typeof v === 'string' ? fn(v) : renameWorkflowNameFields(v, fn),
|
||||
]),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Rewrite workflow-name mentions in one seeded message: human prose (`text`
|
||||
* blocks) and fields that explicitly hold a workflow name. Nothing else.
|
||||
*
|
||||
* Deliberately NOT every string. A message's tool-call blocks carry opaque
|
||||
* payloads — recorded SDK source, expressions, arbitrary results — and a short
|
||||
* workflow name like `Order` would also rewrite a NODE called `Order` inside
|
||||
* recorded source. That is the same integrity break the `workflows[].nodes`
|
||||
* exclusion exists to prevent, one level in: the agent would read prior context
|
||||
* describing an artifact that never existed.
|
||||
*/
|
||||
function renameMentions(message: SeedMessage, fn: (s: string) => string): SeedMessage {
|
||||
const named = renameWorkflowNameFields(message, fn) as SeedMessage;
|
||||
if (!Array.isArray(named.content)) return named;
|
||||
return {
|
||||
...named,
|
||||
content: named.content.map((block) =>
|
||||
isRecord(block) && block.type === 'text' && typeof block.text === 'string'
|
||||
? { ...block, text: fn(block.text) }
|
||||
: block,
|
||||
),
|
||||
} as SeedMessage;
|
||||
}
|
||||
|
||||
/**
|
||||
* Give every seeded workflow a fresh id AND a per-restore unique name, rewriting
|
||||
* all references across the seed — so parallel iterations don't share (and
|
||||
* clobber) one workflow row, and a leftover copy can't be grounded on by name.
|
||||
*
|
||||
* The name rewrite is applied to `messages` ONLY, never inside `workflows[].nodes`.
|
||||
* Workflow names are short and human ("Batch loop"), so a blanket replace could hit
|
||||
* a node that happens to share the name and silently alter the restored graph —
|
||||
* which is exactly the "structural skeleton unchanged" guard a seeded case relies on.
|
||||
*/
|
||||
export function remapSeedWorkflowIds(seed: ConversationSeed): ConversationSeed {
|
||||
if (seed.workflows.length === 0) return seed;
|
||||
|
||||
@@ -250,9 +312,56 @@ export function remapSeedWorkflowIds(seed: ConversationSeed): ConversationSeed {
|
||||
}
|
||||
|
||||
const remapped = ConversationSeedSchema.parse(jsonParse(serialized));
|
||||
|
||||
// n8n itself allows duplicate workflow names, so a scrubbed real seed could
|
||||
// legitimately carry two. This harness can't take them: the rename below rewrites
|
||||
// mentions by matching the name text, so both workflows' mentions would collapse
|
||||
// onto the first one's new name and the history would point at the wrong workflow.
|
||||
// Refuse rather than mangle — a limit of the rewrite, not an n8n rule.
|
||||
const names = remapped.workflows.map((workflow) => workflow.name);
|
||||
const duplicate = names.find((name, index) => names.indexOf(name) !== index);
|
||||
if (duplicate !== undefined) {
|
||||
throw new Error(
|
||||
`Seed declares two workflows named "${duplicate}". The harness rewrites seed workflow ` +
|
||||
'names to keep concurrent runs apart, and it cannot tell which mention in the history ' +
|
||||
'means which workflow — give them distinct names in the fixture',
|
||||
);
|
||||
}
|
||||
|
||||
// Uniquify names after the id pass, so the rename can't perturb id matching.
|
||||
const workflows = remapped.workflows.map((workflow) => {
|
||||
const suffix = seedNameSuffix(randomUUID().slice(0, 8));
|
||||
return {
|
||||
...workflow,
|
||||
name: `${workflow.name.slice(0, MAX_WORKFLOW_NAME - suffix.length)}${suffix}`,
|
||||
};
|
||||
});
|
||||
|
||||
// Any mention in the seeded history follows the workflow, so the agent's own
|
||||
// record of what it built still matches what is on the instance.
|
||||
//
|
||||
// ONE pass over each string, longest original name first. Renaming per
|
||||
// workflow instead would feed each rewrite into the next: with "Order" and
|
||||
// "Order Sync", renaming "Order" first turns every "Order Sync" mention into
|
||||
// "Order [seed …] Sync", which no later pass matches — leaving the history
|
||||
// pointing at a name that was never restored. A replacement produced by this
|
||||
// pass is never rescanned, so the two can't interfere.
|
||||
const renames = new Map(
|
||||
remapped.workflows.map((workflow, index) => [workflow.name, workflows[index].name]),
|
||||
);
|
||||
const mentionRe = new RegExp(
|
||||
[...renames.keys()]
|
||||
.sort((a, b) => b.length - a.length)
|
||||
.map(escapeForRegExp)
|
||||
.join('|'),
|
||||
'g',
|
||||
);
|
||||
const rewrite = (s: string) => s.replace(mentionRe, (match) => renames.get(match) ?? match);
|
||||
const messages = remapped.messages.map((message) => renameMentions(message, rewrite));
|
||||
|
||||
// Data table ids are remapped server-side on restore (id is generated, not
|
||||
// pinnable), so carry them through untouched here.
|
||||
return { ...remapped, source: seed.source, dataTables: seed.dataTables };
|
||||
return { ...remapped, messages, workflows, source: seed.source, dataTables: seed.dataTables };
|
||||
}
|
||||
|
||||
// Transcript prefix — seeded history rendered for the judge/checks. Turns carry
|
||||
|
||||
Reference in New Issue
Block a user