feat(core): Add HITL to agent builder test runs (#35731)

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Robin Braumann
2026-08-06 16:15:54 +02:00
committed by GitHub
parent 07adeaea87
commit 0df0041019
27 changed files with 1057 additions and 185 deletions
+8 -1
View File
@@ -117,7 +117,14 @@ export {
raceWithAbort,
throwIfAborted,
} from './sdk/abort';
export { Tool, wrapToolForApproval, sanitizeToolName } from './sdk/tool';
export {
APPROVAL_RESUME_SCHEMA,
APPROVAL_SUSPEND_SCHEMA,
Tool,
wrapToolForApproval,
sanitizeToolName,
} from './sdk/tool';
export type { ApprovalResumePayload, ApprovalSuspendPayload } from './sdk/tool';
export { Memory } from './sdk/memory';
export { VectorStore } from './sdk/vector-store';
export {
+6 -2
View File
@@ -8,17 +8,21 @@ import type { ToolDescriptor } from '../types/sdk/tool-descriptor';
import type { JSONObject } from '../types/utils/json';
import { isZodSchema, zodToJsonSchema } from '../utils/zod';
const APPROVAL_SUSPEND_SCHEMA = z.object({
export const APPROVAL_SUSPEND_SCHEMA = z.object({
type: z.literal('approval'),
toolName: z.string(),
displayName: z.string().optional(),
args: z.unknown(),
});
const APPROVAL_RESUME_SCHEMA = z.object({
export type ApprovalSuspendPayload = z.infer<typeof APPROVAL_SUSPEND_SCHEMA>;
export const APPROVAL_RESUME_SCHEMA = z.object({
approved: z.boolean(),
});
export type ApprovalResumePayload = z.infer<typeof APPROVAL_RESUME_SCHEMA>;
const APPROVAL_GATE_CONTINUATION_SCHEMA = z
.object({
__n8nApprovalGate: z.literal(true),
+2
View File
@@ -311,6 +311,7 @@ export {
toolErrorPayloadSchema,
confirmationRequestPayloadSchema,
confirmationInputTypeSchema,
instanceAiTargetApprovalSchema,
channelConfigSchema,
credentialRequestSchema,
workflowSetupNodeSchema,
@@ -395,6 +396,7 @@ export type {
InstanceAiChannelConfig,
InstanceAiConfirmationRequestPayload,
InstanceAiConfirmationSeverity,
InstanceAiTargetApproval,
InstanceAiCredentialRequest,
InstanceAiAgentStatus,
InstanceAiAgentKind,
@@ -135,6 +135,11 @@ function makeConfirmationRequest(
args: {},
severity: 'warning',
message: 'Are you sure?',
targetApproval: {
toolName: 'delete_record',
displayName: 'Delete record',
args: { id: 'record-1' },
},
},
};
}
@@ -715,6 +720,11 @@ describe('agent-run-reducer', () => {
requestId: 'req-1',
severity: 'warning',
message: 'Are you sure?',
targetApproval: {
toolName: 'delete_record',
displayName: 'Delete record',
args: { id: 'record-1' },
},
});
});
@@ -475,6 +475,7 @@ export function reduceEvent(state: AgentRunState, event: InstanceAiEvent): Agent
inputThreadId: event.payload.inputThreadId,
severity: event.payload.severity,
message: event.payload.message,
targetApproval: event.payload.targetApproval,
credentialRequests: event.payload.credentialRequests,
projectId: event.payload.projectId,
inputType: event.payload.inputType,
@@ -464,6 +464,13 @@ export const confirmationInputTypeSchema = z.enum([
]);
export type InstanceAiConfirmationInputType = z.infer<typeof confirmationInputTypeSchema>;
export const instanceAiTargetApprovalSchema = z.object({
toolName: z.string(),
displayName: z.string().optional(),
args: z.unknown(),
});
export type InstanceAiTargetApproval = z.infer<typeof instanceAiTargetApprovalSchema>;
export const confirmationRequestPayloadSchema = z.object({
requestId: z.string(),
inputThreadId: z
@@ -475,6 +482,9 @@ export const confirmationRequestPayloadSchema = z.object({
args: z.record(z.unknown()),
severity: instanceAiConfirmationSeveritySchema,
message: z.string().describe('Human-readable description of the action'),
targetApproval: instanceAiTargetApprovalSchema
.optional()
.describe('Target-agent tool approval details rendered instead of the outer tool call'),
credentialRequests: z.array(credentialRequestSchema).optional(),
projectId: z
.string()
@@ -1045,6 +1055,7 @@ export interface InstanceAiConfirmation {
inputThreadId?: string;
severity: InstanceAiConfirmationSeverity;
message: string;
targetApproval?: InstanceAiTargetApproval;
credentialRequests?: InstanceAiCredentialRequest[];
projectId?: string;
inputType?: 'approval' | 'text' | 'questions' | 'plan-review' | 'resource-decision' | 'continue';
+5 -3
View File
@@ -685,7 +685,8 @@ Delegates agent building to the agents-module builder chat
turn per call. Registered in `createOrchestrationTools` only when the host
provides `builderDelegate` (agents module active). The builder's own prompt
and tools drive the build, including its interactive tools (`ask_questions`,
`ask_credential`, `ask_embedding_credential`, `configure_channel`) and
`ask_credential`, `ask_embedding_credential`, `configure_channel`, and
`call_agent` target-tool approvals) and
lifecycle tools (`publish_agent`, `unpublish_agent`) on the bound target agent —
the sub-agent session no longer excludes them. Forward publish/unpublish/
activate/make-live intents to `build-agent`; never tell the user to open the
@@ -712,8 +713,9 @@ starts (agents module not configured, missing `name`/`agentId`, no project
context to bind `agentId`, or a resume whose suspend payload has no
checkpoint ref to carry).
**Interactive questions:** when the builder suspends on one of its interactive
tools (batched questions, a credential picker, or channel setup), this tool
**Interactive requests:** when the builder suspends on one of its interactive
tools (batched questions, a credential picker, channel setup, or a standard SDK
approval requested by a target-agent test run), this tool
cascades the suspension through its own suspend/resume so it renders as a
chat card directly in the assistant conversation — no manual relaying, and the
suspension survives a process restart. On resume, the tool takes the target
@@ -432,6 +432,68 @@ describe('mapAgentChunkToEvent', () => {
});
});
it('maps target approval details without replacing the outer routing fields', () => {
expect(
map({
type: 'tool-call-suspended',
toolCallId: 'build-agent-call',
toolName: 'build-agent',
input: { agentRef: 'support-agent' },
suspendPayload: {
type: 'approval',
requestId: 'approval-1',
toolName: 'delete_record',
displayName: 'Delete record',
args: { id: 'record-1' },
builderCheckpoint: { runId: 'builder-run', toolCallId: 'call-agent' },
},
}),
).toEqual({
type: 'confirmation-request',
runId,
agentId,
payload: {
requestId: 'approval-1',
toolCallId: 'build-agent-call',
toolName: 'build-agent',
args: { agentRef: 'support-agent' },
severity: 'warning',
message: 'Confirmation required',
targetApproval: {
toolName: 'delete_record',
displayName: 'Delete record',
args: { id: 'record-1' },
},
},
});
});
it('keeps direct SDK approvals as ordinary Instance AI confirmations', () => {
const result = map({
type: 'tool-call-suspended',
toolCallId: 'direct-tool-call',
toolName: 'delete_record',
input: { id: 'record-1' },
suspendPayload: {
type: 'approval',
requestId: 'approval-1',
toolName: 'delete_record',
args: { id: 'record-1' },
},
});
expect(result).toMatchObject({
type: 'confirmation-request',
payload: {
toolCallId: 'direct-tool-call',
toolName: 'delete_record',
args: { id: 'record-1' },
},
});
if (result?.type !== 'confirmation-request') throw new Error('Expected confirmation request');
expect(result.payload).not.toHaveProperty('targetApproval');
});
it('maps pause-for-user continue confirmations and web search metadata', () => {
expect(
map({
@@ -219,6 +219,11 @@ describe('OutputRedactor', () => {
args: {},
severity: 'warning',
message: 'What to do with jane@example.com?',
targetApproval: {
toolName: 'send_jane@example.com',
displayName: 'Email jane@example.com',
args: { recipient: 'jane@example.com' },
},
introMessage: 'I noticed jane@example.com in your request.',
questions: [
{
@@ -246,7 +251,49 @@ describe('OutputRedactor', () => {
expect(serialized).not.toContain('jane@example.com');
expect(serialized).toContain('[REDACTED]');
// Control/identifier fields are preserved so suspend/resume keeps working.
expect(out).toMatchObject({ payload: { requestId: 'req-1', toolCallId: 'tc-1' } });
expect(out).toMatchObject({
payload: {
requestId: 'req-1',
toolCallId: 'tc-1',
targetApproval: {
toolName: '[REDACTED]',
displayName: 'Email [REDACTED]',
args: { recipient: '[REDACTED]' },
},
},
});
expect(event.payload.targetApproval?.args).toEqual({ recipient: 'jane@example.com' });
});
it('withholds target approval args nested beyond the redaction depth limit', () => {
const redactor = createRedactor();
const secret = 'Bearer abcdef1234567890';
let nested: unknown = { secret };
for (let depth = 0; depth < 9; depth++) nested = { nested };
const event: InstanceAiEvent = {
type: 'confirmation-request',
runId: 'run-1',
agentId: 'agent-1',
payload: {
requestId: 'req-1',
toolCallId: 'tc-1',
toolName: 'call_agent',
args: {},
severity: 'warning',
message: 'Confirm action',
targetApproval: {
toolName: 'deep_action',
args: { visible: 'ordinary value', nested },
},
},
};
const [out] = redactor.processEvent(event);
if (out.type !== 'confirmation-request') throw new Error('Expected confirmation request');
const serializedArgs = JSON.stringify(out.payload.targetApproval?.args);
expect(serializedArgs).not.toContain(secret);
expect(serializedArgs).toContain('[REDACTED]');
expect(out.payload.targetApproval?.args).toMatchObject({ visible: 'ordinary value' });
});
it('logs a filtering summary with category counts and no values', () => {
@@ -1,4 +1,4 @@
import type { StreamChunk } from '@n8n/agents';
import { APPROVAL_SUSPEND_SCHEMA, type StreamChunk } from '@n8n/agents';
import {
credentialRequestSchema,
workflowSetupNodeSchema,
@@ -344,6 +344,18 @@ function mapSuspendedChunk(
gatewayConfirmationRequiredPayloadSchema,
);
const channelConfig = parseSchemaRecord(suspendPayload.channelConfig, channelConfigSchema);
const targetApprovalResult = isRecord(suspendPayload.builderCheckpoint)
? APPROVAL_SUSPEND_SCHEMA.safeParse(suspendPayload)
: undefined;
const targetApproval = targetApprovalResult?.success
? {
toolName: targetApprovalResult.data.toolName,
...(targetApprovalResult.data.displayName
? { displayName: targetApprovalResult.data.displayName }
: {}),
args: targetApprovalResult.data.args,
}
: undefined;
return {
type: 'confirmation-request',
@@ -358,6 +370,7 @@ function mapSuspendedChunk(
typeof suspendPayload.message === 'string'
? suspendPayload.message
: 'Confirmation required',
...(targetApproval ? { targetApproval } : {}),
...(credentialRequests ? { credentialRequests } : {}),
...(projectId ? { projectId } : {}),
...(inputType ? { inputType } : {}),
@@ -35,6 +35,23 @@ interface OutputRedactorContext {
options?: RedactionOptions | false;
}
const MAX_TARGET_APPROVAL_ARG_DEPTH = 8;
const WITHHELD_TARGET_APPROVAL_ARG = '[REDACTED]';
function withholdDeepTargetApprovalArgs(value: unknown, depth = 0): unknown {
if (value === null || typeof value !== 'object') return value;
if (depth >= MAX_TARGET_APPROVAL_ARG_DEPTH) return WITHHELD_TARGET_APPROVAL_ARG;
if (Array.isArray(value)) {
return value.map((item) => withholdDeepTargetApprovalArgs(item, depth + 1));
}
return Object.fromEntries(
Object.entries(value).map(([key, item]) => [
key,
withholdDeepTargetApprovalArgs(item, depth + 1),
]),
);
}
type DeltaType = 'text-delta' | 'reasoning-delta';
interface Channel {
@@ -175,8 +192,8 @@ export class OutputRedactor {
}
/**
* Redact the human-readable text of a HITL confirmation card (message, intro,
* question/option labels, and task/plan-item descriptions). Control and
* Redact the human-readable content of a HITL confirmation card (message, intro,
* question/option labels, task/plan-item descriptions, and target-tool labels/args). Control and
* identifier fields — `requestId`, `toolCallId`, `inputType`,
* `credentialRequests`, task `id`/`status`, plan `kind`/`deps`, etc. — are
* left untouched so suspend/resume routing keeps working.
@@ -207,6 +224,20 @@ export class OutputRedactor {
title: this.redactString(item.title),
spec: this.redactString(item.spec),
}));
let targetApproval = payload.targetApproval;
if (targetApproval) {
const boundedArgs = withholdDeepTargetApprovalArgs(targetApproval.args);
const { value, matches } = redactDeep(boundedArgs, this.options);
this.recordMatches(matches);
targetApproval = {
...targetApproval,
toolName: this.redactString(targetApproval.toolName),
...(targetApproval.displayName
? { displayName: this.redactString(targetApproval.displayName) }
: {}),
args: value,
};
}
return {
...event,
@@ -217,6 +248,7 @@ export class OutputRedactor {
...(questions ? { questions } : {}),
...(tasks ? { tasks } : {}),
...(planItems ? { planItems } : {}),
...(targetApproval ? { targetApproval } : {}),
},
};
}
@@ -19,20 +19,20 @@
* 1. `ctx.suspendPayload` restoration on resume (the cli's `builderCheckpoint`
* ref, read back from the checkpoint store, not from in-memory state).
* 2. Two-level SDK resume validation: the orchestrator's permissive
* passthrough schema, then the builder's own `questionsResumeSchema` —
* the user's answer must survive both without being stripped or replaced.
* 3. A full "suspend → drop every in-memory object → resume from checkpoint
* storage only" restart, per AGENT-354's testing requirement: phase 2
* below constructs fresh Agent instances, a fresh delegate, and a fresh
* domain context, and resumes purely from the shared checkpoint store and
* thread-metadata record (the only state carried across phases).
* passthrough schema, then the builder tool's own resume schema.
* 3. Repeated "suspend → drop every in-memory object → resume from checkpoint
* storage only" restarts for a question followed by a target-tool approval.
*/
import { Agent, Tool } from '@n8n/agents';
import type {
CheckpointStore,
SerializableAgentState,
StreamChunk,
StreamResult,
import {
Agent,
APPROVAL_RESUME_SCHEMA,
APPROVAL_SUSPEND_SCHEMA,
Tool,
type ApprovalResumePayload,
type CheckpointStore,
type SerializableAgentState,
type StreamChunk,
type StreamResult,
} from '@n8n/agents';
import {
questionsResumeSchema,
@@ -180,16 +180,13 @@ function toTurnStream(result: StreamResult): BuilderTurnStream {
/**
* Build the agent-builder sub-agent: `write_config` (a mutation tool whose
* name drives `configUpdated` via `CONFIG_MUTATION_TOOL_NAMES`) and
* `ask_questions` (an interruptible tool using the real shared contract
* from `@n8n/api-types`, mirroring the cli's own `ask_questions` tool). On
* resume, `onResume` records the exact `ctx.resumeData` the SDK handed back
* after validating it against `questionsResumeSchema` — the central
* assertion of this test is that nothing was stripped along the way.
* `ask_questions` and `call_agent` interruptible tools. On resume, `onResume`
* records the exact `ctx.resumeData` the SDK handed back after validation.
*/
function createBuilderAgent(
checkpointStore: CheckpointStore,
model: MockLanguageModelV3,
onResume: (data: QuestionsResumeData) => void,
onResume: (toolName: 'ask_questions' | 'call_agent', data: unknown) => void,
): Agent {
const writeConfigTool = new Tool('write_config')
.description('Persist the agent configuration')
@@ -214,15 +211,34 @@ function createBuilderAgent(
],
});
}
onResume(ctx.resumeData);
onResume('ask_questions', ctx.resumeData);
return { answered: true };
});
const callAgentTool = new Tool('call_agent')
.description('Test the target agent')
.input(z.object({}))
.suspend(APPROVAL_SUSPEND_SCHEMA)
.resume(APPROVAL_RESUME_SCHEMA)
.handler(async (_input, ctx) => {
if (ctx.resumeData === undefined) {
return await ctx.suspend({
type: 'approval',
toolName: 'delete_record',
displayName: 'Delete record',
args: { id: 'record-1' },
});
}
onResume('call_agent', ctx.resumeData);
return { status: 'completed' };
});
return new Agent('agent-builder')
.model(model)
.instructions('You are the agent builder sub-agent. Use your tools to build the agent.')
.tool(writeConfigTool)
.tool(askQuestionsTool)
.tool(callAgentTool)
.checkpoint(checkpointStore);
}
@@ -234,7 +250,7 @@ function createBuilderAgent(
function createBuilderDelegate(
store: InMemoryCheckpointStore,
model: MockLanguageModelV3,
onResume: (data: QuestionsResumeData) => void,
onResume: (toolName: 'ask_questions' | 'call_agent', data: unknown) => void,
): InstanceAiBuilderDelegate {
return {
createAgent: async (_name: string) =>
@@ -342,7 +358,7 @@ function createOrchestrationContext(params: {
}
describe('build-agent cascade restart (real SDK)', () => {
it('suspends on a builder question, drops all in-memory state, and resumes purely from checkpoint storage with the answer intact', async () => {
it('survives restarts across a builder question and a chained target approval', async () => {
const store = new InMemoryCheckpointStore();
const threadRecords = new Map<string, ThreadRecord>();
threadRecords.set('thread-1', {
@@ -446,9 +462,11 @@ describe('build-agent cascade restart (real SDK)', () => {
let observedBuilderResumeData: QuestionsResumeData | undefined;
const phase2Delegate = createBuilderDelegate(
store,
createScriptedModel([makeTextTurn('Configured Slack as the channel.')]),
(data) => {
observedBuilderResumeData = data;
createScriptedModel([makeToolCallTurn('b-tc-3', 'call_agent', {})]),
(toolName, data) => {
if (toolName === 'ask_questions') {
observedBuilderResumeData = questionsResumeSchema.parse(data);
}
},
);
const phase2Context = createOrchestrationContext({
@@ -456,7 +474,7 @@ describe('build-agent cascade restart (real SDK)', () => {
delegate: phase2Delegate,
agentBuilderTarget: undefined,
});
const orchModel2 = createScriptedModel([makeTextTurn('Your agent is ready.')]);
const orchModel2 = createScriptedModel([makeTextTurn('This turn should remain suspended.')]);
const orchestrator2 = new Agent('ia-orchestrator')
.model(orchModel2)
.instructions('Test orchestrator')
@@ -478,7 +496,54 @@ describe('build-agent cascade restart (real SDK)', () => {
// and the builder's own questionsResumeSchema validation.
expect(observedBuilderResumeData).toEqual(resumeData);
const toolResultChunks = chunksOfType(phase2Chunks, 'tool-result').filter(
const approvalSuspensions = chunksOfType(phase2Chunks, 'tool-call-suspended');
expect(approvalSuspensions).toHaveLength(1);
const approvalSuspension = approvalSuspensions[0];
expect(approvalSuspension.suspendPayload).toMatchObject({
type: 'approval',
toolName: 'delete_record',
displayName: 'Delete record',
args: { id: 'record-1' },
builderCheckpoint: {
configUpdated: true,
},
});
expect(store.listStates().filter(([, state]) => state.status === 'suspended')).toHaveLength(2);
// ── Phase 3: restart again and route only the approval decision back ─
let observedApprovalResumeData: ApprovalResumePayload | undefined;
const phase3Delegate = createBuilderDelegate(
store,
createScriptedModel([makeTextTurn('The target action was declined.')]),
(toolName, data) => {
if (toolName === 'call_agent') {
observedApprovalResumeData = APPROVAL_RESUME_SCHEMA.parse(data);
}
},
);
const phase3Context = createOrchestrationContext({
threadRecords,
delegate: phase3Delegate,
agentBuilderTarget: undefined,
});
const orchestrator3 = new Agent('ia-orchestrator')
.model(createScriptedModel([makeTextTurn('Your agent test was handled.')]))
.instructions('Test orchestrator')
.tool(createBuildAgentTool(phase3Context))
.checkpoint(store);
const phase3Run = await orchestrator3.resume(
'stream',
{ approved: false, scope: 'session' },
{
runId: approvalSuspension.runId,
toolCallId: approvalSuspension.toolCallId,
},
);
const phase3Chunks = await collectStreamChunks(phase3Run.stream);
expect(observedApprovalResumeData).toEqual({ approved: false });
const toolResultChunks = chunksOfType(phase3Chunks, 'tool-result').filter(
(c) => c.toolName === ORCHESTRATION_TOOL_IDS.BUILD_AGENT,
);
expect(toolResultChunks).toHaveLength(1);
@@ -487,9 +552,9 @@ describe('build-agent cascade restart (real SDK)', () => {
configUpdated: true,
});
const output = toolResultChunks[0].output as { builderReply?: string };
expect(output.builderReply).toContain('Configured Slack');
expect(output.builderReply).toContain('target action was declined');
const finishChunks = chunksOfType(phase2Chunks, 'finish');
const finishChunks = chunksOfType(phase3Chunks, 'finish');
expect(finishChunks.length).toBeGreaterThan(0);
expect(finishChunks.at(-1)?.finishReason).toBe('stop');
@@ -179,6 +179,15 @@ function configureChannelSuspendPayload() {
};
}
function targetApprovalSuspendPayload() {
return {
type: 'approval' as const,
toolName: 'delete_record',
displayName: 'Delete record',
args: { id: 'record-1' },
};
}
/** Stub for `context.tracing`: a sentinel telemetry object plus mocked child-run lifecycle. */
function makeTracingStub() {
const sentinelTelemetry = { functionId: 'sentinel' } as unknown as ReturnType<
@@ -1358,6 +1367,7 @@ describe('build-agent tool', () => {
['ask_questions', askQuestionsSuspendPayload],
['ask_credential', askCredentialSuspendPayload],
['configure_channel', configureChannelSuspendPayload],
['call_agent', targetApprovalSuspendPayload],
] as const)(
'cascades a %s suspension into ctx.suspend, passing the shared-contract payload through with a re-minted requestId and builderCheckpoint ref',
async (toolName, buildPayload) => {
@@ -1375,8 +1385,14 @@ describe('build-agent tool', () => {
expect(suspend).toHaveBeenCalledTimes(1);
const payload = suspend.mock.calls[0][0] as Record<string, unknown>;
const { requestId: originalRequestId, ...basePayload } = buildPayload();
expect(payload).toMatchObject(basePayload);
const original = buildPayload();
if ('requestId' in original) {
const { requestId: originalRequestId, ...basePayload } = original;
expect(payload).toMatchObject(basePayload);
expect(payload.requestId).not.toBe(originalRequestId);
} else {
expect(payload).toMatchObject(original);
}
expect(payload).toMatchObject({
builderCheckpoint: {
runId: 'builder-run-1',
@@ -1386,7 +1402,6 @@ describe('build-agent tool', () => {
},
});
expect(typeof payload.requestId).toBe('string');
expect(payload.requestId).not.toBe(originalRequestId);
},
);
@@ -4,12 +4,11 @@
* per invocation.
*
* This is the interactive contract: the delegate session includes the
* builder's full standard toolset, so it may suspend on `ask_questions`,
* `ask_credential`, `ask_embedding_credential`, or `configure_channel`. When
* it does, this tool cascades the suspension through its own `ctx.suspend()`
* — using payloads derived from the shared interaction contract in
* `@n8n/api-types` — so the question renders as a card in the calling
* assistant's chat and the orchestrator's own checkpoint survives a process
* builder's full standard toolset, so it may suspend for builder interactions
* or for a target-agent tool approval. This tool cascades the suspension
* through its own `ctx.suspend()` using the interaction contracts in
* `@n8n/api-types` and the SDK approval contract in `@n8n/agents`, so it renders
* as a card in the calling assistant's chat and the orchestrator checkpoint survives a process
* restart. On resume, the target agent and the builder's open suspension are
* both re-derived from persistence (no in-memory state carried across the
* suspend boundary) and checked for identity against the `builderCheckpoint`
@@ -23,7 +22,7 @@
* builder UI — it is a private sub-agent conversation.
*/
import type { InterruptibleToolContext } from '@n8n/agents';
import { createAbortError, Tool } from '@n8n/agents';
import { APPROVAL_SUSPEND_SCHEMA, createAbortError, Tool } from '@n8n/agents';
import {
BUILDER_CHECKPOINT_UNAVAILABLE_CODE,
BUILDER_NOT_CONFIGURED_CODE,
@@ -247,12 +246,17 @@ const builderSuspendPayloadSchema = z.union([
questionsSuspendPayloadSchema,
credentialSuspendPayloadSchema,
channelSuspendPayloadSchema,
APPROVAL_SUSPEND_SCHEMA,
]);
const buildAgentSuspendSchema = z.union([
questionsSuspendPayloadSchema.extend({ builderCheckpoint: builderCheckpointRefSchema }),
credentialSuspendPayloadSchema.extend({ builderCheckpoint: builderCheckpointRefSchema }),
channelSuspendPayloadSchema.extend({ builderCheckpoint: builderCheckpointRefSchema }),
APPROVAL_SUSPEND_SCHEMA.extend({
requestId: z.string(),
builderCheckpoint: builderCheckpointRefSchema,
}),
]);
/**
@@ -948,7 +952,7 @@ export function createBuildAgentTool(context: OrchestrationContext) {
'agent when the user asks to publish, activate, make it live/usable, or unpublish — ' +
'forward that intent in `message`; never tell the user to open the agent editor and ' +
'click Publish. When the builder needs user input (a choice, a ' +
'credential, or a chat channel), it surfaces automatically as an interactive card in ' +
'credential, a chat channel, or approval for a target-agent tool), it surfaces automatically as an interactive card in ' +
'this chat — do not relay those questions yourself; this tool call resumes with the ' +
'users answer and returns the builders reply. Returns the builders reply, the ' +
'target `agentRef`/`agentId`, and whether it updated the agent config. Prefer the ' +
@@ -798,6 +798,34 @@ describe('AgentExecutionOrchestratorService', () => {
expect(checkpointStorage.cancelSuspended).not.toHaveBeenCalled();
});
it('does not resume a checkpoint outside the expected draft memory scope', async () => {
const { service, checkpointStorage, runtimeCacheService } = makeService();
checkpointStorage.getStatus.mockResolvedValue({
status: 'active',
checkpoint: makeCheckpoint(),
});
for (const expectedMemory of [
{ threadId: 'another-thread', resourceId: 'draft-chat:user-1' },
{ threadId: 'thread-1', resourceId: 'draft-chat:another-user' },
]) {
await expect(
collect(
service.resumeForChat({
agentId,
projectId,
runId: 'run-1',
toolCallId: 'tool-call-1',
resumeData: { approved: true },
expectedMemory,
}),
),
).rejects.toThrow('Checkpoint run-1 does not belong to this chat');
}
expect(runtimeCacheService.getRuntime).not.toHaveBeenCalled();
});
it('does not directly cancel or resume a delegated child checkpoint', async () => {
const { service, checkpointStorage, runtimeCacheService } = makeService();
const checkpoint = makeCheckpoint(
@@ -993,6 +1021,14 @@ describe('AgentExecutionOrchestratorService', () => {
expect(agentRunTracingService.build).toHaveBeenCalledWith(
expect.objectContaining({ source: 'telegram' }),
);
expect(executionService.startExecutionRecording).toHaveBeenCalledWith(
expect.objectContaining({ source: 'telegram' }),
expect.any(Date),
);
expect(executionService.finalizeExecution).toHaveBeenCalledWith(
'execution-1',
expect.objectContaining({ source: 'telegram' }),
);
});
it('falls back to source "unknown" when no suspended execution is found on resume', async () => {
@@ -125,6 +125,69 @@ describe('AgentTestRunService', () => {
});
});
it('resumes the same draft session and returns the next suspended segment', async () => {
const { service, agentExecutionOrchestratorService } = makeService();
agentExecutionOrchestratorService.resumeForChat.mockImplementation(async function* (config) {
config.onExecutionRecorded?.('execution-2');
yield { type: 'text-delta', id: 'text-1', delta: ' Next step.' };
yield {
type: 'tool-call-suspended',
runId: 'run-2',
toolCallId: 'tool-call-2',
toolName: 'notify_owner',
suspendPayload: {
type: 'approval',
toolName: 'notify_owner',
args: { ownerId: 'owner-1' },
},
};
});
const result = await service.resumeDraftRun({
agentId,
projectId,
sessionId: 'session-1',
runId: 'run-1',
toolCallId: 'tool-call-1',
resumeData: { approved: true },
user,
source: 'instance-ai',
response: 'First step.',
});
expect(result).toEqual({
status: 'suspended',
response: 'First step. Next step.',
sessionId: 'session-1',
executionId: 'execution-2',
suspensions: [
{
runId: 'run-2',
toolCallId: 'tool-call-2',
toolName: 'notify_owner',
suspendPayload: {
type: 'approval',
toolName: 'notify_owner',
args: { ownerId: 'owner-1' },
},
},
],
});
expect(agentExecutionOrchestratorService.resumeForChat).toHaveBeenCalledWith(
expect.objectContaining({
runId: 'run-1',
toolCallId: 'tool-call-1',
resumeData: { approved: true },
source: 'instance-ai',
usePublishedVersion: false,
expectedMemory: {
threadId: 'session-1',
resourceId: 'draft-chat:user-1',
},
}),
);
});
it('rejects a session owned by another agent without starting a run', async () => {
const {
service,
@@ -148,8 +211,21 @@ describe('AgentTestRunService', () => {
credentialProvider,
}),
).resolves.toEqual({ status: 'session_not_found' });
await expect(
service.resumeDraftRun({
agentId,
projectId,
sessionId: 'session-1',
runId: 'run-1',
toolCallId: 'tool-call-1',
resumeData: { approved: false },
user,
response: '',
}),
).resolves.toEqual({ status: 'session_not_found' });
expect(agentValidationService.validateAgentIsRunnable).not.toHaveBeenCalled();
expect(agentExecutionOrchestratorService.executeForChat).not.toHaveBeenCalled();
expect(agentExecutionOrchestratorService.resumeForChat).not.toHaveBeenCalled();
});
it('returns missing configuration without starting a run', async () => {
@@ -1,6 +1,14 @@
import type { Mocked } from 'vitest';
import { TELEMETRY_EVENT } from '@n8n/telemetry';
import type { CredentialProvider } from '@n8n/agents';
import {
Agent as SdkAgent,
type CheckpointStore,
type CredentialProvider,
type SerializableAgentState,
type StreamChunk,
zodToJsonSchema,
} from '@n8n/agents';
import { APPROVAL_RESUME_SCHEMA } from '@n8n/agents/tool';
import {
AGENT_SKILL_INSTRUCTIONS_MAX_LENGTH,
type AgentJsonConfig,
@@ -14,6 +22,7 @@ import type {
} from '@n8n/backend-network';
import type { SsrfProtectionConfig } from '@n8n/config';
import type { User } from '@n8n/db';
import { convertArrayToReadableStream, MockLanguageModelV3 } from 'ai/test';
import { NodeConnectionTypes } from 'n8n-workflow';
import { mock } from 'vitest-mock-extended';
@@ -50,6 +59,8 @@ const ctx = {
parentTelemetry: undefined,
};
const standardApprovalResumeSchema = zodToJsonSchema(APPROVAL_RESUME_SCHEMA)!;
type BuilderPurposeServices = Pick<AgentsService, 'findById' | 'findByProjectId'> &
Pick<AgentConfigService, 'updateConfig'> &
Pick<AgentCustomToolsService, 'buildCustomTool'> &
@@ -1770,40 +1781,248 @@ describe('AgentsBuilderToolsService', () => {
expect(agentTestRunService.executeDraftRun).not.toHaveBeenCalled();
});
it('cancels a suspended run and directs the user to Preview without approval data', async () => {
it('suspends for a target approval and resumes the same test run with the human decision', async () => {
vi.spyOn(checkAccess, 'userHasScopes').mockResolvedValue(true);
type MockStreamResult = Awaited<ReturnType<MockLanguageModelV3['doStream']>>;
type MockStreamPart = MockStreamResult['stream'] extends ReadableStream<infer Part>
? Part
: never;
const usage: Extract<MockStreamPart, { type: 'finish' }>['usage'] = {
inputTokens: { total: 10, noCache: 10, cacheRead: 0, cacheWrite: 0 },
outputTokens: { total: 5, text: 5, reasoning: 0 },
};
const toolCallTurn = (
toolCallId: string,
toolName: string,
input: Record<string, unknown>,
): MockStreamResult => ({
stream: convertArrayToReadableStream<MockStreamPart>([
{ type: 'stream-start', warnings: [] },
{ type: 'tool-call', toolCallId, toolName, input: JSON.stringify(input) },
{
type: 'finish',
finishReason: { unified: 'tool-calls', raw: 'tool_calls' },
usage,
},
]),
});
const textTurn = (text: string): MockStreamResult => ({
stream: convertArrayToReadableStream<MockStreamPart>([
{ type: 'stream-start', warnings: [] },
{ type: 'text-start', id: 'text-1' },
{ type: 'text-delta', id: 'text-1', delta: text },
{ type: 'text-end', id: 'text-1' },
{ type: 'finish', finishReason: { unified: 'stop', raw: 'stop' }, usage },
]),
});
const scriptedModel = (turns: MockStreamResult[]) => {
let nextTurn = 0;
return new MockLanguageModelV3({
provider: 'mock',
modelId: 'scripted',
doStream: async () => await Promise.resolve(turns[nextTurn++]),
});
};
const collectChunks = async (stream: ReadableStream<StreamChunk>) => {
const chunks: StreamChunk[] = [];
const reader = stream.getReader();
while (true) {
const { done, value } = await reader.read();
if (done) return chunks;
chunks.push(value);
}
};
const suspendedChunks = (chunks: StreamChunk[]) =>
chunks.filter(
(chunk): chunk is Extract<StreamChunk, { type: 'tool-call-suspended' }> =>
chunk.type === 'tool-call-suspended',
);
class InMemoryCheckpointStore implements CheckpointStore {
private states = new Map<string, SerializableAgentState>();
async save(key: string, state: SerializableAgentState) {
await Promise.resolve(this.states.set(key, structuredClone(state)));
}
async load(key: string) {
const state = this.states.get(key);
return await Promise.resolve(state ? structuredClone(state) : undefined);
}
async delete(key: string) {
await Promise.resolve(this.states.delete(key));
}
}
const checkpointStore = new InMemoryCheckpointStore();
const firstApproval = {
type: 'approval' as const,
toolName: 'delete_record',
displayName: 'Delete record',
args: { id: 'record-1' },
};
const firstContinuation = {
runId: 'target-run-1',
toolCallId: 'target-tool-call-1',
sessionId: 'session-1',
response: 'I need approval before deleting the record.',
};
let outerRunId = '';
let outerToolCallId = '';
{
const { service, agentTestRunService } = makeService();
agentTestRunService.executeDraftRun.mockResolvedValue({
status: 'suspended',
response: firstContinuation.response,
sessionId: firstContinuation.sessionId,
executionId: 'execution-1',
suspensions: [
{
runId: firstContinuation.runId,
toolCallId: firstContinuation.toolCallId,
toolName: firstApproval.toolName,
input: firstApproval.args,
suspendPayload: firstApproval,
resumeSchema: standardApprovalResumeSchema,
},
],
});
const phase1Agent = new SdkAgent('agent-builder')
.model(
scriptedModel([
toolCallTurn('outer-tool-call-1', BUILDER_TOOLS.CALL_AGENT, {
message: 'Delete the record',
sessionId: 'session-1',
}),
]),
)
.instructions('Test the target agent.')
.tool(getCallAgentTool(service))
.checkpoint(checkpointStore);
const firstRun = await phase1Agent.stream('Test deleting a record');
const firstSuspensions = suspendedChunks(await collectChunks(firstRun.stream));
expect(firstSuspensions).toHaveLength(1);
const firstOuterSuspension = firstSuspensions[0];
outerRunId = firstOuterSuspension.runId;
outerToolCallId = firstOuterSuspension.toolCallId;
expect(firstOuterSuspension).toMatchObject({
toolName: BUILDER_TOOLS.CALL_AGENT,
suspendPayload: firstApproval,
});
expect(
(await checkpointStore.load(outerRunId))?.pendingToolCalls[outerToolCallId],
).toMatchObject({
suspended: true,
continuation: firstContinuation,
});
}
const secondApproval = {
type: 'approval' as const,
toolName: 'notify_owner',
displayName: 'Notify owner',
args: { ownerId: 'owner-1' },
};
const secondContinuation = {
runId: 'target-run-2',
toolCallId: 'target-tool-call-2',
sessionId: 'session-2',
response: 'Deletion approved. I need approval to notify the owner.',
};
const { service, agentTestRunService } = makeService();
agentTestRunService.resumeDraftRun.mockResolvedValue({
status: 'suspended',
response: secondContinuation.response,
sessionId: secondContinuation.sessionId,
executionId: 'execution-2',
suspensions: [
{
runId: secondContinuation.runId,
toolCallId: secondContinuation.toolCallId,
toolName: secondApproval.toolName,
input: secondApproval.args,
suspendPayload: secondApproval,
resumeSchema: standardApprovalResumeSchema,
},
],
});
const phase2Agent = new SdkAgent('agent-builder')
.model(scriptedModel([textTurn('This run should remain suspended.')]))
.instructions('Test the target agent.')
.tool(getCallAgentTool(service))
.checkpoint(checkpointStore);
const resumedRun = await phase2Agent.resume(
'stream',
{ approved: true },
{ runId: outerRunId, toolCallId: outerToolCallId },
);
const secondSuspensions = suspendedChunks(await collectChunks(resumedRun.stream));
expect(agentTestRunService.resumeDraftRun).toHaveBeenCalledWith({
agentId,
projectId,
sessionId: firstContinuation.sessionId,
runId: firstContinuation.runId,
toolCallId: firstContinuation.toolCallId,
resumeData: { approved: true },
user,
source: 'instance-ai',
response: firstContinuation.response,
abortSignal: expect.any(AbortSignal),
});
expect(secondSuspensions).toHaveLength(1);
const secondOuterSuspension = secondSuspensions[0];
expect(secondOuterSuspension).toMatchObject({
toolName: BUILDER_TOOLS.CALL_AGENT,
suspendPayload: secondApproval,
});
expect(
(await checkpointStore.load(secondOuterSuspension.runId))?.pendingToolCalls[
secondOuterSuspension.toolCallId
],
).toMatchObject({
suspended: true,
continuation: secondContinuation,
});
});
it('cancels approval-shaped custom suspensions and directs the user to Preview', async () => {
const { service, agentTestRunService } = makeService();
vi.spyOn(checkAccess, 'userHasScopes').mockResolvedValue(true);
agentTestRunService.executeDraftRun.mockResolvedValue({
status: 'suspended',
response: 'I need approval.',
response: 'Choose a date.',
sessionId: 'session-1',
executionId: 'execution-1',
suspensions: [
{
runId: 'run-1',
toolCallId: 'tool-call-1',
toolName: 'delete_record',
input: { id: 'record-1' },
suspendPayload: { type: 'approval' },
toolName: 'schedule_record',
suspendPayload: {
type: 'approval',
toolName: 'schedule_record',
args: { date: 'tomorrow' },
},
resumeSchema: {
...standardApprovalResumeSchema,
allOf: [{ properties: { approved: { const: true } } }],
},
},
],
});
agentTestRunService.cancelSuspendedRun.mockResolvedValue(true);
const result = await getCallAgentTool(service).handler!(
{ message: 'Delete it', sessionId: 'session-1' },
ctx,
);
const result = await getCallAgentTool(service).handler!({ message: 'Schedule it' }, ctx);
expect(agentTestRunService.executeDraftRun).toHaveBeenCalledWith({
agentId,
projectId,
message: 'Delete it',
sessionId: 'session-1',
credentialProvider,
user,
source: 'instance-ai',
});
expect(agentTestRunService.cancelSuspendedRun).toHaveBeenCalledWith({
agentId,
runId: 'run-1',
@@ -1811,10 +2030,10 @@ describe('AgentsBuilderToolsService', () => {
});
expect(result).toEqual({
status: 'approval_required',
response: 'I need approval.',
response: 'Choose a date.',
sessionId: 'session-1',
executionId: 'execution-1',
suspensions: [{ runId: 'run-1', toolCallId: 'tool-call-1', toolName: 'delete_record' }],
suspensions: [{ runId: 'run-1', toolCallId: 'tool-call-1', toolName: 'schedule_record' }],
previewPath: '/projects/project-1/agents/agent-1/preview',
});
});
@@ -84,6 +84,10 @@ export interface ResumeForChatConfig {
runId: string;
toolCallId: string;
resumeData: unknown;
/** Expected memory scope used to prevent resuming another user's or thread's checkpoint. */
expectedMemory?: Partial<AgentMemoryScope>;
/** Identifies the surface that resumed the execution. */
source?: string;
/**
* The calling n8n user for in-app preview chat resumes used to gate
* node/workflow tools by their access. Absent for published/integration
@@ -304,6 +308,8 @@ export class AgentExecutionOrchestratorService {
runId,
toolCallId,
resumeData,
expectedMemory,
source,
integrationType,
user,
usePublishedVersion = true,
@@ -327,6 +333,14 @@ export class AgentExecutionOrchestratorService {
if (memoryScope.delegated === true) {
throw new UserError('Delegated actions must be resumed through their parent agent');
}
if (
(expectedMemory?.threadId !== undefined &&
memoryScope.threadId !== expectedMemory.threadId) ||
(expectedMemory?.resourceId !== undefined &&
memoryScope.resourceId !== expectedMemory.resourceId)
) {
throw new UserError(`Checkpoint ${runId} does not belong to this chat`);
}
const threadId = memoryScope.threadId;
@@ -354,22 +368,25 @@ export class AgentExecutionOrchestratorService {
});
const startedAt = recorder.startedAt;
const runType: AgentRunTelemetryType = usePublishedVersion ? 'production' : 'test';
let executionSource = source;
try {
// A resume request carries no `source` of its own — recover it from
// the suspended run being resumed so tracing stays consistent across
// the suspend/resume cycle. Skipped entirely when tracing is disabled,
// since `build()` would discard the result anyway.
const suspendedExecution = this.agentRunTracingService.enabled
? await this.agentExecutionService.findLatestSuspendedRun(threadId)
: undefined;
const suspendedExecution =
this.agentRunTracingService.enabled && source === undefined
? await this.agentExecutionService.findLatestSuspendedRun(threadId)
: undefined;
executionSource ??= suspendedExecution?.source ?? undefined;
const tracing = await this.agentRunTracingService.build({
agentId,
projectId,
threadId,
userId: user?.id,
source: suspendedExecution?.source ?? 'unknown',
source: executionSource ?? 'unknown',
modelId: modelIdFromSnapshot(agentInstance.snapshot.model),
});
@@ -390,6 +407,7 @@ export class AgentExecutionOrchestratorService {
agentName: agentInstance.name,
projectId,
userMessage: null,
...(executionSource !== undefined ? { source: executionSource } : {}),
telemetry: {
runType,
configuration: runtime.telemetryConfiguration,
@@ -423,6 +441,7 @@ export class AgentExecutionOrchestratorService {
agentName: agentInstance.name,
projectId,
userMessage: null,
...(executionSource !== undefined ? { source: executionSource } : {}),
record: messageRecord,
hitlStatus: recorder.suspended ? 'suspended' : 'resumed',
telemetry: {
@@ -1,4 +1,5 @@
import type { CredentialProvider, StreamChunk } from '@n8n/agents';
import { N8N_CHAT_INTEGRATION_TYPE } from '@n8n/api-types';
import type { User } from '@n8n/db';
import { Service } from '@n8n/di';
import { randomUUID } from 'node:crypto';
@@ -40,6 +41,19 @@ interface ExecuteDraftRunInput extends PrepareDraftRunInput {
abortSignal?: AbortSignal;
}
interface ResumeDraftRunInput {
agentId: string;
projectId: string;
sessionId: string;
runId: string;
toolCallId: string;
resumeData: unknown;
user: User;
source?: string;
response: string;
abortSignal?: AbortSignal;
}
export interface AgentTestRunSuspension {
runId: string;
toolCallId: string;
@@ -61,6 +75,8 @@ export type AgentTestRunResult =
| { status: 'session_not_found' }
| { status: 'agent_misconfigured'; missing: string[] };
type CollectedDraftRunResult = Extract<AgentTestRunResult, { status: 'completed' | 'suspended' }>;
@Service()
export class AgentTestRunService {
constructor(
@@ -123,9 +139,7 @@ export class AgentTestRunService {
const prepared = await this.prepareDraftRun(input);
if (prepared.status !== 'ready') return prepared;
let response = '';
let executionId: string | undefined;
const suspensions: AgentTestRunSuspension[] = [];
const stream = this.streamDraftRun({
...input,
sessionId: prepared.sessionId,
@@ -134,6 +148,64 @@ export class AgentTestRunService {
},
});
return await this.collectDraftRun(stream, prepared.sessionId, '', () => executionId);
}
async resumeDraftRun(input: ResumeDraftRunInput): Promise<AgentTestRunResult> {
const existing = await this.agentExecutionService.findThreadById(input.sessionId);
if (existing && !threadBelongsTo(existing, input.projectId, input.agentId)) {
return { status: 'session_not_found' };
}
let executionId: string | undefined;
const stream = this.agentExecutionOrchestratorService.resumeForChat({
agentId: input.agentId,
projectId: input.projectId,
runId: input.runId,
toolCallId: input.toolCallId,
resumeData: input.resumeData,
user: input.user,
usePublishedVersion: false,
integrationType: N8N_CHAT_INTEGRATION_TYPE,
expectedMemory: {
threadId: input.sessionId,
resourceId: draftChatMemoryResourceId(input.user.id),
},
source: input.source,
onExecutionRecorded: (id) => {
executionId = id;
},
...(input.abortSignal ? { abortSignal: input.abortSignal } : {}),
});
return await this.collectDraftRun(stream, input.sessionId, input.response, () => executionId);
}
async cancelSuspendedRun({
agentId,
runId,
userId,
}: {
agentId: string;
runId: string;
userId: string;
}): Promise<boolean> {
return await this.agentExecutionOrchestratorService.cancelChatRun({
agentId,
runId,
resourceId: draftChatMemoryResourceId(userId),
});
}
private async collectDraftRun(
stream: AsyncIterable<StreamChunk>,
sessionId: string,
initialResponse: string,
getExecutionId: () => string | undefined,
): Promise<CollectedDraftRunResult> {
let response = initialResponse;
const suspensions: AgentTestRunSuspension[] = [];
for await (const chunk of stream) {
if (chunk.type === 'error') {
throw chunk.error;
@@ -152,30 +224,14 @@ export class AgentTestRunService {
}
}
const executionId = getExecutionId();
const metadata = {
response,
sessionId: prepared.sessionId,
sessionId,
...(executionId ? { executionId } : {}),
};
if (suspensions.length > 0) {
return { status: 'suspended', ...metadata, suspensions };
}
return { status: 'completed', ...metadata };
}
async cancelSuspendedRun({
agentId,
runId,
userId,
}: {
agentId: string;
runId: string;
userId: string;
}): Promise<boolean> {
return await this.agentExecutionOrchestratorService.cancelChatRun({
agentId,
runId,
resourceId: draftChatMemoryResourceId(userId),
});
return suspensions.length > 0
? { status: 'suspended', ...metadata, suspensions }
: { status: 'completed', ...metadata };
}
}
@@ -142,7 +142,8 @@ const agentsSdkMocks = vi.hoisted(() => {
};
});
vi.mock('@n8n/agents', () => ({
vi.mock('@n8n/agents', async (importOriginal) => ({
...(await importOriginal<typeof import('@n8n/agents')>()),
Agent: agentsSdkMocks.MockAgent,
Memory: agentsSdkMocks.MockMemory,
createObservationLogObserveFn: agentsSdkMocks.createObservationLogObserveFn,
@@ -43,8 +43,9 @@ Use \`call_agent\` to verify the agent's behavior, not its channel integrations.
After configuring a channel, tell the user that they must publish the agent and
message it from the connected platform to verify the channel.
If the result is \`approval_required\`, explain that the action needs approval
and direct the user to [Preview](${agentPreviewPath}) to run it again.
Standard tool approvals pause \`call_agent\` until the user approves or rejects them in this chat.
If it returns \`approval_required\` for an unsupported interaction, explain that it cannot be
completed here and direct the user to [Preview](${agentPreviewPath}) to run it again.
After a successful build or config change that leaves the agent ready to try,
include the same [Preview](${agentPreviewPath}) markdown link in your wrap-up
@@ -1,6 +1,18 @@
import type { BuiltTool, CredentialProvider } from '@n8n/agents';
import { isAbortError } from '@n8n/agents';
import { Tool } from '@n8n/agents/tool';
import { isDeepStrictEqual } from 'node:util';
import {
isAbortError,
zodToJsonSchema,
type BuiltTool,
type CredentialProvider,
type InterruptibleToolContext,
} from '@n8n/agents';
import {
APPROVAL_RESUME_SCHEMA,
APPROVAL_SUSPEND_SCHEMA,
Tool,
type ApprovalResumePayload,
type ApprovalSuspendPayload,
} from '@n8n/agents/tool';
import {
applyNativeWebSearchDefaultOn,
getProviderPrefix,
@@ -48,7 +60,11 @@ import { AgentIntegrationPersistenceService } from '../agent-integration-persist
import { AgentPublishService } from '../agent-publish.service';
import { AgentSkillsService } from '../agent-skills.service';
import { AgentTaskService } from '../agent-task.service';
import { AgentTestRunService } from '../agent-test-run.service';
import {
AgentTestRunService,
type AgentTestRunResult,
type AgentTestRunSuspension,
} from '../agent-test-run.service';
import { AgentsToolsService } from '../agents-tools.service';
import { AgentsService } from '../agents.service';
import { AttachableWorkflowsService } from '../attachable-workflows.service';
@@ -82,6 +98,30 @@ const STALE_CONFIG_ERROR: ConfigValidationError = {
'Agent config changed since you last read it. Call read_config, then retry using the config and configHash it returns.',
};
const callAgentContinuationSchema = z
.object({
runId: z.string(),
toolCallId: z.string(),
sessionId: z.string(),
response: z.string(),
})
.strict();
const expectedApprovalResumeJsonSchema = zodToJsonSchema(APPROVAL_RESUME_SCHEMA);
function parseStandardApprovalSuspension(
suspension: AgentTestRunSuspension,
): ApprovalSuspendPayload | undefined {
const payload = APPROVAL_SUSPEND_SCHEMA.safeParse(suspension.suspendPayload);
if (
!payload.success ||
!isDeepStrictEqual(suspension.resumeSchema, expectedApprovalResumeJsonSchema)
) {
return undefined;
}
return payload.data;
}
/** LLM-facing follow-up guidance for this builder surface (CLI skill-based tools). */
const CLI_AGENT_CONFIG_MESSAGES: AgentConfigValidationMessages = {
emptyInstructionsFollowUp: 'saving the config again.',
@@ -620,8 +660,8 @@ export class AgentsBuilderToolsService {
'Tests the draft agent through built-in Preview chat. It does not test configured channel integrations, including their triggers, platform context, message delivery, or replies. ' +
'Pass the returned sessionId on later calls to continue the same conversation; omit it to start a new one. ' +
'The draft uses its real configured tools and credentials, so external side effects are possible. ' +
'Returns status completed with response/session identifiers, approval_required with a Preview path, or error. ' +
'If the result requires approval, direct the user to the returned Preview path.',
'Standard tool approvals pause this test until the user approves or rejects them in chat. ' +
'Unsupported interactive requests return approval_required with a Preview path.',
)
.input(
z.object({
@@ -634,90 +674,141 @@ export class AgentsBuilderToolsService {
.describe('Session ID from a previous call_agent result'),
}),
)
.handler(async ({ message, sessionId }: { message: string; sessionId?: string }, ctx) => {
if (!(await userHasScopes(user, ['agent:execute'], false, { projectId }))) {
return {
status: 'error',
code: 'forbidden',
message: 'You do not have permission to run agents in this project.',
};
}
const previewPath = buildAgentPreviewPath(projectId, agentId);
try {
const result = await this.agentTestRunService.executeDraftRun({
agentId,
projectId,
message,
sessionId,
credentialProvider,
user,
source: 'instance-ai',
...(ctx.abortSignal ? { abortSignal: ctx.abortSignal } : {}),
});
if (result.status === 'session_not_found') {
.suspend(APPROVAL_SUSPEND_SCHEMA)
.resume(APPROVAL_RESUME_SCHEMA)
.handler(
async (
{ message, sessionId }: { message: string; sessionId?: string },
ctx: InterruptibleToolContext<ApprovalSuspendPayload, ApprovalResumePayload>,
) => {
if (!(await userHasScopes(user, ['agent:execute'], false, { projectId }))) {
return {
status: 'error',
code: 'session_not_found',
message: 'Session not found.',
code: 'forbidden',
message: 'You do not have permission to run agents in this project.',
};
}
if (result.status === 'agent_misconfigured') {
return {
status: 'error',
code: 'agent_misconfigured',
message: "This agent isn't ready to run yet. Finish configuring it and try again.",
missing: result.missing,
};
}
if (result.status === 'completed') return result;
const runIds = [...new Set(result.suspensions.map(({ runId }) => runId))];
const cancellations = await Promise.all(
runIds.map(
async (runId) =>
await this.agentTestRunService.cancelSuspendedRun({
agentId,
runId,
userId: user.id,
}),
),
).catch(() => []);
if (
cancellations.length !== runIds.length ||
cancellations.some((cancelled) => !cancelled)
) {
const previewPath = buildAgentPreviewPath(projectId, agentId);
try {
let result: AgentTestRunResult;
if (ctx.resumeData === undefined) {
result = await this.agentTestRunService.executeDraftRun({
agentId,
projectId,
message,
sessionId,
credentialProvider,
user,
source: 'instance-ai',
...(ctx.abortSignal ? { abortSignal: ctx.abortSignal } : {}),
});
} else {
const continuation = callAgentContinuationSchema.safeParse(ctx.continuation);
if (
!continuation.success ||
!APPROVAL_SUSPEND_SCHEMA.safeParse(ctx.suspendPayload).success
) {
return {
status: 'error',
code: 'invalid_checkpoint',
message: 'This test run can no longer be resumed.',
};
}
result = await this.agentTestRunService.resumeDraftRun({
agentId,
projectId,
sessionId: continuation.data.sessionId,
runId: continuation.data.runId,
toolCallId: continuation.data.toolCallId,
resumeData: { approved: ctx.resumeData.approved },
user,
source: 'instance-ai',
response: continuation.data.response,
...(ctx.abortSignal ? { abortSignal: ctx.abortSignal } : {}),
});
}
if (result.status === 'session_not_found') {
return {
status: 'error',
code: 'session_not_found',
message: 'Session not found.',
};
}
if (result.status === 'agent_misconfigured') {
return {
status: 'error',
code: 'agent_misconfigured',
message: "This agent isn't ready to run yet. Finish configuring it and try again.",
missing: result.missing,
};
}
if (result.status === 'completed') return result;
const approvals = result.suspensions.map(parseStandardApprovalSuspension);
if (approvals.every((approval) => approval !== undefined)) {
const firstSuspension = result.suspensions[0];
const firstApproval = approvals[0];
if (firstSuspension && firstApproval) {
return await ctx.suspend(firstApproval, {
continuation: {
runId: firstSuspension.runId,
toolCallId: firstSuspension.toolCallId,
sessionId: result.sessionId,
response: result.response,
},
});
}
}
const runIds = [...new Set(result.suspensions.map(({ runId }) => runId))];
const cancellations = await Promise.all(
runIds.map(
async (runId) =>
await this.agentTestRunService.cancelSuspendedRun({
agentId,
runId,
userId: user.id,
}),
),
).catch(() => []);
if (
cancellations.length !== runIds.length ||
cancellations.some((cancelled) => !cancelled)
) {
return {
status: 'error',
code: 'cancellation_failed',
message:
'This test needs approval, but its suspended run could not be cancelled. Open Preview before continuing this session.',
sessionId: result.sessionId,
previewPath,
};
}
return {
status: 'error',
code: 'cancellation_failed',
message:
'This test needs approval, but its suspended run could not be cancelled. Open Preview before continuing this session.',
status: 'approval_required',
response: result.response,
sessionId: result.sessionId,
...(result.executionId ? { executionId: result.executionId } : {}),
suspensions: result.suspensions.map(({ runId, toolCallId, toolName }) => ({
runId,
toolCallId,
toolName,
})),
previewPath,
};
} catch (error) {
if (ctx.abortSignal ? ctx.abortSignal.aborted : isAbortError(error)) throw error;
return {
status: 'error',
code: 'execution_failed',
message: error instanceof Error ? error.message : 'Agent test run failed.',
};
}
return {
status: 'approval_required',
response: result.response,
sessionId: result.sessionId,
...(result.executionId ? { executionId: result.executionId } : {}),
suspensions: result.suspensions.map(({ runId, toolCallId, toolName }) => ({
runId,
toolCallId,
toolName,
})),
previewPath,
};
} catch (error) {
if (ctx.abortSignal ? ctx.abortSignal.aborted : isAbortError(error)) throw error;
return {
status: 'error',
code: 'execution_failed',
message: error instanceof Error ? error.message : 'Agent test run failed.',
};
}
})
},
)
.build();
const modelLookup: ModelLookup = {
@@ -28,7 +28,7 @@ export const INSTANCE_AI_BUILDER_ADDENDUM = `## Instance AI session rules
You are running as a sub-agent inside n8n's instance AI chat; the user sees your questions as chat cards.
Preview links work in this chat. Include a markdown Preview link after a successful build and when \`call_agent\` reports \`approval_required\`, using the exact relative path from "When To Build vs When To Converse" (form: \`[Preview](<path>)\`). Do not invent absolute URLs. Do not omit the link and describe the path in plain text instead.
Preview links work in this chat. Include a markdown Preview link after a successful build and when \`call_agent\` reports an unsupported interaction as \`approval_required\`, using the exact relative path from "When To Build vs When To Converse" (form: \`[Preview](<path>)\`). Do not invent absolute URLs. Do not omit the link and describe the path in plain text instead.
You can publish and unpublish the target agent with \`publish_agent\` and \`unpublish_agent\`. Never tell the user to open the agent editor and click Publish.
@@ -20,6 +20,9 @@ vi.mock('@n8n/i18n', async (importOriginal) => ({
...(await importOriginal()),
useI18n: () => ({
baseText: (key: string, opts?: { interpolate?: Record<string, string> }) => {
if (key === 'agents.chat.approval.description') {
return `The agent wants to run the ${opts?.interpolate?.toolName ?? ''} tool.`;
}
if (opts?.interpolate) {
return Object.entries(opts.interpolate).reduce(
(str, [k, v]) => str.replace(`{${k}}`, v),
@@ -359,6 +362,48 @@ describe('InstanceAiConfirmationPanel telemetry', () => {
);
});
it('shows target approval details and only submits a one-time decision', async () => {
injectPendingConfirmation(thread, {
requestId: 'req-target',
severity: 'warning',
message: 'Confirmation required',
targetApproval: {
toolName: 'delete_record',
displayName: 'Delete record',
args: { id: 'record-1' },
},
});
const confirmSpy = vi.spyOn(thread, 'confirmAction').mockResolvedValue(true);
const addKeySpy = vi.spyOn(thread, 'addAlwaysAllowKey');
const { getByTestId, getByText, queryByTestId } = renderComponent({
props: { kind: 'floating' },
});
expect(getByText('The agent wants to run the Delete record tool.')).toBeVisible();
expect(getByTestId('instance-ai-target-approval-args')).toHaveTextContent('"id": "record-1"');
expect(queryByTestId('instance-ai-panel-confirm-always-allow')).toBeNull();
await userEvent.click(getByTestId('instance-ai-panel-confirm-approve'));
expect(confirmSpy).toHaveBeenCalledWith('req-target', {
kind: 'approval',
approved: true,
});
expect(addKeySpy).not.toHaveBeenCalled();
expect(mockTelemetryTrack).toHaveBeenCalledWith(
'User finished providing input',
expect.objectContaining({
provided_inputs: [
expect.objectContaining({
options: ['approve', 'deny'],
option_chosen: 'approve',
}),
],
}),
);
});
it('renders nothing when mounted as inline for a floating-eligible confirmation', () => {
injectPendingConfirmation(thread, {
requestId: 'req-inline-skip',
@@ -7,7 +7,7 @@ import { mockedStore } from '@/__tests__/utils';
import { useWorkflowsListStore } from '@/app/stores/workflowsList.store';
import { fetchThreadMessages, fetchThreadStatus } from '../instanceAi.memory.api';
import { ensureThread, postMessage, postConfirmation, postCancel } from '../instanceAi.api';
import { INSTANCE_AI_THREAD_SOURCE_FALLBACK } from '@n8n/api-types';
import { INSTANCE_AI_THREAD_SOURCE_FALLBACK, type InstanceAiTargetApproval } from '@n8n/api-types';
import {
createThreadRuntime,
getAgentBuilderTargetFromThreadMetadata,
@@ -1668,6 +1668,7 @@ describe('createThreadRuntime - session always-allow', () => {
args?: Record<string, unknown>;
severity?: 'info' | 'warning' | 'destructive';
channelConfig?: { integrationType: string; agentId: string };
targetApproval?: InstanceAiTargetApproval;
},
): void {
runtime.messages.push({
@@ -1697,6 +1698,7 @@ describe('createThreadRuntime - session always-allow', () => {
severity: opts.severity ?? 'info',
message: 'Approve?',
...(opts.channelConfig ? { channelConfig: opts.channelConfig } : {}),
...(opts.targetApproval ? { targetApproval: opts.targetApproval } : {}),
},
},
],
@@ -1758,6 +1760,25 @@ describe('createThreadRuntime - session always-allow', () => {
expect(mockPostConfirmation).not.toHaveBeenCalled();
});
it('does not auto-approve target-agent approvals even when the outer tool key matches', async () => {
const runtime = registry.getOrCreateRuntime(activeThreadId);
runtime.addAlwaysAllowKey('build-agent', {});
pushPendingApproval(runtime, {
messageId: 'msg-target-approval',
requestId: 'req-target-approval',
toolName: 'build-agent',
targetApproval: {
toolName: 'delete_record',
args: { id: 'record-1' },
},
});
await new Promise((resolve) => setTimeout(resolve, 10));
expect(runtime.resolvedConfirmationIds.has('req-target-approval')).toBe(false);
expect(mockPostConfirmation).not.toHaveBeenCalled();
});
it('distinguishes submit-workflow create vs update grants by workflowId presence', async () => {
const runtime = registry.getOrCreateRuntime(activeThreadId);
runtime.addAlwaysAllowKey('submit-workflow', {});
@@ -132,6 +132,9 @@ function isDestructive(item: PendingConfirmationItem): boolean {
* English strings over the wire.
*/
function buildApprovalTitle(item: PendingConfirmationItem): string {
if (item.toolCall.confirmation.targetApproval) {
return i18n.baseText('agents.chat.approval.title');
}
const { toolName, args } = item.toolCall;
const action = typeof args?.action === 'string' ? args.action : undefined;
const imperativeKey = (
@@ -154,6 +157,12 @@ function buildApprovalTitle(item: PendingConfirmationItem): string {
* message includes a trailing explanation doesn't bloat the card.
*/
function buildApprovalSubtitle(item: PendingConfirmationItem): string {
const targetApproval = item.toolCall.confirmation.targetApproval;
if (targetApproval) {
return i18n.baseText('agents.chat.approval.description', {
interpolate: { toolName: targetApproval.displayName ?? targetApproval.toolName },
});
}
const message = item.toolCall.confirmation.message ?? '';
const idx = message.indexOf('?');
return idx === -1 ? message : message.slice(0, idx + 1);
@@ -167,7 +176,7 @@ function buildApprovalSubtitle(item: PendingConfirmationItem): string {
function buildApprovalOptions(item: PendingConfirmationItem): ApprovalOption[] {
const destructive = isDestructive(item);
const options: ApprovalOption[] = [];
if (!destructive) {
if (!destructive && !item.toolCall.confirmation.targetApproval) {
options.push({
key: 'always-allow',
icon: 'check-check',
@@ -192,6 +201,16 @@ function buildApprovalOptions(item: PendingConfirmationItem): ApprovalOption[] {
return options;
}
function formatTargetApprovalArgs(conf: InstanceAiConfirmation): string {
const args = conf.targetApproval?.args;
if (args === undefined) return '';
try {
return JSON.stringify(args, null, 2) ?? String(args);
} catch {
return String(args);
}
}
function handleApprovalSelect(item: PendingConfirmationItem, key: string) {
switch (key) {
case 'always-allow':
@@ -228,7 +247,7 @@ async function handleConfirm(item: PendingConfirmationItem, approved: boolean) {
// "Always allow" is offered alongside Approve/Deny for non-destructive
// generic approvals; include it in the option set so telemetry reflects
// what the user actually chose between.
const alwaysAllowAvailable = !isDestructive(item);
const alwaysAllowAvailable = !isDestructive(item) && !conf.targetApproval;
trackInputCompleted(
conf,
[
@@ -581,6 +600,13 @@ function handlePlanDeny(conf: InstanceAiConfirmation, numTasks: number) {
{{ buildApprovalTitle(chunk.item) }}
</N8nText>
<ConfirmationPreview>{{ buildApprovalSubtitle(chunk.item) }}</ConfirmationPreview>
<ConfirmationPreview
v-if="formatTargetApprovalArgs(chunk.item.toolCall.confirmation)"
:class="$style.targetApprovalArgs"
data-test-id="instance-ai-target-approval-args"
>
{{ formatTargetApprovalArgs(chunk.item.toolCall.confirmation) }}
</ConfirmationPreview>
</div>
<ApprovalOptionList
@@ -622,6 +648,11 @@ function handlePlanDeny(conf: InstanceAiConfirmation, numTasks: number) {
}
}
.targetApprovalArgs {
white-space: pre-wrap;
word-break: break-word;
}
.approvalRow {
display: flex;
flex-direction: column;
@@ -593,6 +593,7 @@ export function createThreadRuntime(
function isGenericApprovalEligible(item: PendingConfirmationItem): boolean {
const conf = item.toolCall.confirmation;
if (conf.targetApproval) return false;
if (conf.severity === 'destructive') return false;
if (conf.domainAccess) return false;
if (conf.inputType) return false;