mirror of
https://github.com/n8n-io/n8n.git
synced 2026-10-29 19:10:43 +08:00
fix(core): Bound oversized agent tool results (#35759)
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -24,6 +24,7 @@ import {
|
||||
type DelegateSubAgentRunner,
|
||||
} from '../tools/delegate-sub-agent-tool';
|
||||
import { toAiSdkTools } from '../tools/tool-adapter';
|
||||
import { MAX_MODEL_TOOL_RESULT_CHARS } from '../tools/tool-result-guard';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Module mocks
|
||||
@@ -7360,3 +7361,202 @@ describe('AgentRuntime — MCP connection failure warnings', () => {
|
||||
expect(systemText).not.toContain('<mcp-connection-status>');
|
||||
});
|
||||
});
|
||||
|
||||
describe('AgentRuntime — oversized tool results', () => {
|
||||
type TruncationEnvelope = {
|
||||
_truncated: true;
|
||||
originalCharCount: number;
|
||||
estimatedTokenCount: number;
|
||||
head: string;
|
||||
tail: string;
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
function parseEnvelope(value: string): TruncationEnvelope {
|
||||
try {
|
||||
return JSON.parse(value) as TruncationEnvelope;
|
||||
} catch {
|
||||
throw new Error('Expected a valid truncation envelope');
|
||||
}
|
||||
}
|
||||
|
||||
function getModelToolResult(callIndex = 1): unknown {
|
||||
const call = generateText.mock.calls[callIndex][0] as {
|
||||
messages: Array<{
|
||||
role: string;
|
||||
content: Array<{ type: string; output?: { type: string; value: unknown } }>;
|
||||
}>;
|
||||
};
|
||||
const toolMessage = call.messages.find((message) => message.role === 'tool');
|
||||
return toolMessage?.content.find((part) => part.type === 'tool-result')?.output?.value;
|
||||
}
|
||||
|
||||
it('leaves a small transformed result unchanged', async () => {
|
||||
const tool: BuiltTool = {
|
||||
name: 'small_result',
|
||||
description: 'Return a small result',
|
||||
inputSchema: z.object({}),
|
||||
handler: async () => await Promise.resolve({ raw: 'value' }),
|
||||
toModelOutput: () => ({ summary: 'small' }),
|
||||
};
|
||||
const { runtime } = createRuntimeWithTools([tool], 1);
|
||||
generateText
|
||||
.mockResolvedValueOnce(
|
||||
makeGenerateWithToolCalls([{ toolCallId: 'tc-small', toolName: tool.name, args: {} }]),
|
||||
)
|
||||
.mockResolvedValueOnce(makeGenerateSuccess());
|
||||
|
||||
await runtime.generate('run');
|
||||
|
||||
expect(getModelToolResult()).toEqual({ summary: 'small' });
|
||||
});
|
||||
|
||||
it('bounds an oversized transformed result while preserving raw output', async () => {
|
||||
const rawOutput = {
|
||||
value: `RAW_HEAD${'r'.repeat(MAX_MODEL_TOOL_RESULT_CHARS)}RAW_TAIL`,
|
||||
};
|
||||
const transformedOutput = `TRANSFORM_HEAD${'t'.repeat(MAX_MODEL_TOOL_RESULT_CHARS)}TRANSFORM_TAIL`;
|
||||
const tool: BuiltTool = {
|
||||
name: 'large_result',
|
||||
description: 'Return a large result',
|
||||
inputSchema: z.object({}),
|
||||
handler: async () => await Promise.resolve(rawOutput),
|
||||
toModelOutput: () => transformedOutput,
|
||||
};
|
||||
const { runtime } = createRuntimeWithTools([tool], 1);
|
||||
generateText
|
||||
.mockResolvedValueOnce(
|
||||
makeGenerateWithToolCalls([{ toolCallId: 'tc-large', toolName: tool.name, args: {} }]),
|
||||
)
|
||||
.mockResolvedValueOnce(makeGenerateSuccess());
|
||||
|
||||
const result = await runtime.generate('run');
|
||||
const envelope = getModelToolResult() as TruncationEnvelope;
|
||||
|
||||
expect(JSON.stringify(envelope).length).toBeLessThanOrEqual(MAX_MODEL_TOOL_RESULT_CHARS);
|
||||
expect(envelope).toMatchObject({
|
||||
_truncated: true,
|
||||
originalCharCount: JSON.stringify(transformedOutput).length,
|
||||
});
|
||||
expect(envelope.head).toContain('TRANSFORM_HEAD');
|
||||
expect(envelope.tail).toContain('TRANSFORM_TAIL');
|
||||
expect(result.toolCalls?.[0]?.output).toEqual(rawOutput);
|
||||
|
||||
const { runtime: streamRuntime } = createRuntimeWithTools([tool], 1);
|
||||
streamText
|
||||
.mockReturnValueOnce({
|
||||
stream: makeChunkStream([]),
|
||||
finishReason: Promise.resolve('tool-calls'),
|
||||
usage: Promise.resolve({ inputTokens: 10, outputTokens: 5, totalTokens: 15 }),
|
||||
response: Promise.resolve({
|
||||
messages: [
|
||||
{
|
||||
role: 'assistant',
|
||||
content: [
|
||||
{
|
||||
type: 'tool-call',
|
||||
toolCallId: 'tc-large-stream',
|
||||
toolName: tool.name,
|
||||
args: {},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
}),
|
||||
toolCalls: Promise.resolve([
|
||||
{ toolCallId: 'tc-large-stream', toolName: tool.name, input: {} },
|
||||
]),
|
||||
})
|
||||
.mockReturnValueOnce(makeStreamSuccess());
|
||||
|
||||
const { stream } = await streamRuntime.stream('run');
|
||||
const chunks = await collectChunks(stream);
|
||||
const streamResult = chunks.find(
|
||||
(chunk) => chunk.type === 'tool-result' && chunk.toolCallId === 'tc-large-stream',
|
||||
) as (StreamChunk & { type: 'tool-result' }) | undefined;
|
||||
|
||||
expect(streamResult?.output).toEqual(envelope);
|
||||
});
|
||||
|
||||
it('bounds an oversized error without changing its rejected state', async () => {
|
||||
const tool: BuiltTool = {
|
||||
name: 'large_error',
|
||||
description: 'Throw a large error',
|
||||
inputSchema: z.object({}),
|
||||
handler: () => {
|
||||
throw new Error(`ERROR_HEAD${'e'.repeat(MAX_MODEL_TOOL_RESULT_CHARS)}ERROR_TAIL`);
|
||||
},
|
||||
};
|
||||
const { runtime } = createRuntimeWithTools([tool], 1);
|
||||
generateText
|
||||
.mockResolvedValueOnce(
|
||||
makeGenerateWithToolCalls([{ toolCallId: 'tc-error', toolName: tool.name, args: {} }]),
|
||||
)
|
||||
.mockResolvedValueOnce(makeGenerateSuccess('recovered'));
|
||||
|
||||
const result = await runtime.generate('run');
|
||||
const toolCall = result.messages
|
||||
.flatMap((message) => ('content' in message ? message.content : []))
|
||||
.find(
|
||||
(content): content is ContentToolCall =>
|
||||
content.type === 'tool-call' && content.toolCallId === 'tc-error',
|
||||
);
|
||||
const envelope = parseEnvelope(toolCall?.state === 'rejected' ? toolCall.error : '');
|
||||
|
||||
expect(result.finishReason).toBe('stop');
|
||||
expect(toolCall?.state).toBe('rejected');
|
||||
expect(JSON.stringify(envelope).length).toBeLessThanOrEqual(MAX_MODEL_TOOL_RESULT_CHARS);
|
||||
expect(envelope.head).toContain('ERROR_HEAD');
|
||||
expect(envelope.tail).toContain('ERROR_TAIL');
|
||||
});
|
||||
|
||||
it('bounds aggregate toMessage text while preserving file content', async () => {
|
||||
const fileData = Buffer.from('file').toString('base64');
|
||||
const textBlockLength = Math.floor(MAX_MODEL_TOOL_RESULT_CHARS / 2);
|
||||
const tool: BuiltTool = {
|
||||
name: 'large_message',
|
||||
description: 'Return a large message',
|
||||
inputSchema: z.object({}),
|
||||
handler: async () => await Promise.resolve({ ok: true }),
|
||||
toMessage: () => ({
|
||||
role: 'assistant',
|
||||
content: [
|
||||
{ type: 'text', text: `MESSAGE_HEAD${'m'.repeat(textBlockLength)}` },
|
||||
{ type: 'file', mediaType: 'text/plain', data: fileData },
|
||||
{ type: 'text', text: `${'n'.repeat(textBlockLength)}MESSAGE_TAIL` },
|
||||
],
|
||||
}),
|
||||
};
|
||||
const { runtime } = createRuntimeWithTools([tool], 1);
|
||||
generateText
|
||||
.mockResolvedValueOnce(
|
||||
makeGenerateWithToolCalls([{ toolCallId: 'tc-message', toolName: tool.name, args: {} }]),
|
||||
)
|
||||
.mockResolvedValueOnce(makeGenerateSuccess());
|
||||
|
||||
const result = await runtime.generate('run');
|
||||
const message = result.messages.find(
|
||||
(candidate) =>
|
||||
'content' in candidate &&
|
||||
candidate.content.some((content) => content.type === 'file' && content.data === fileData),
|
||||
);
|
||||
const textBlocks =
|
||||
message && 'content' in message
|
||||
? message.content.filter((content) => content.type === 'text')
|
||||
: [];
|
||||
const fileBlock =
|
||||
message && 'content' in message
|
||||
? message.content.find((content) => content.type === 'file')
|
||||
: undefined;
|
||||
const envelope = parseEnvelope(textBlocks[0]?.text ?? '');
|
||||
|
||||
expect(textBlocks).toHaveLength(1);
|
||||
expect(JSON.stringify(envelope).length).toBeLessThanOrEqual(MAX_MODEL_TOOL_RESULT_CHARS);
|
||||
expect(envelope.head).toContain('MESSAGE_HEAD');
|
||||
expect(envelope.tail).toContain('MESSAGE_TAIL');
|
||||
expect(fileBlock).toMatchObject({ type: 'file', mediaType: 'text/plain', data: fileData });
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import { toJsonValue } from '@n8n/utils/json/to-json-value';
|
||||
import { zodToJsonSchema, type JsonSchema7Type } from 'zod-to-json-schema';
|
||||
|
||||
import {
|
||||
@@ -7,6 +6,11 @@ import {
|
||||
} from './delegate-sub-agent-tool';
|
||||
import { DEFAULT_SUB_AGENT_MAX_CHILDREN } from './sub-agent-task-path';
|
||||
import { executeTool, isSuspendedToolResult, type SuspendedToolResult } from './tool-adapter';
|
||||
import {
|
||||
guardToolErrorForModel,
|
||||
guardToolMessageForModel,
|
||||
guardToolResultForModel,
|
||||
} from './tool-result-guard';
|
||||
import { isAbortError, raceWithAbort } from '../../sdk/abort';
|
||||
import { isCancellation } from '../../sdk/cancellation';
|
||||
import { isLlmMessage } from '../../sdk/message';
|
||||
@@ -24,6 +28,7 @@ import type { JSONObject, JSONValue } from '../../types/utils/json';
|
||||
import { parseWithSchema } from '../../utils/parse';
|
||||
import { isZodSchema } from '../../utils/zod';
|
||||
import { incrementToolCallCount } from '../loop/execution-counter';
|
||||
import { stringifyError } from '../loop/runtime-helpers';
|
||||
import type { AgentMessageList } from '../model/message-list';
|
||||
import { normalizeToolInputForModel } from '../model/messages';
|
||||
import type { AgentEventBus } from '../state/event-bus';
|
||||
@@ -850,7 +855,7 @@ export class ToolCallExecutor {
|
||||
result: error,
|
||||
isError: true,
|
||||
});
|
||||
params.list.setToolCallError(params.toolCallId, error);
|
||||
params.list.setToolCallError(params.toolCallId, guardToolErrorForModel(stringifyError(error)));
|
||||
return { outcome: 'error', error };
|
||||
}
|
||||
|
||||
@@ -1012,6 +1017,7 @@ export class ToolCallExecutor {
|
||||
} catch (error) {
|
||||
return this.toolError(params, error);
|
||||
}
|
||||
const guardedResult = guardToolResultForModel(modelResult);
|
||||
|
||||
this.eventBus.emit({
|
||||
type: AgentEvent.ToolExecutionEnd,
|
||||
@@ -1021,11 +1027,14 @@ export class ToolCallExecutor {
|
||||
isError: false,
|
||||
});
|
||||
|
||||
list.setToolCallResult(toolCallId, toJsonValue(modelResult));
|
||||
list.setToolCallResult(toolCallId, guardedResult.historyOutput);
|
||||
|
||||
const customMessage = builtTool.toMessage?.(toolResult);
|
||||
if (customMessage) {
|
||||
list.addResponse([customMessage]);
|
||||
const guardedCustomMessage = customMessage
|
||||
? guardToolMessageForModel(customMessage)
|
||||
: undefined;
|
||||
if (guardedCustomMessage) {
|
||||
list.addResponse([guardedCustomMessage]);
|
||||
}
|
||||
|
||||
return {
|
||||
@@ -1036,8 +1045,8 @@ export class ToolCallExecutor {
|
||||
output: toolResult,
|
||||
transformed: !!builtTool.toModelOutput,
|
||||
},
|
||||
modelOutput: modelResult,
|
||||
customMessage,
|
||||
modelOutput: guardedResult.wireOutput,
|
||||
customMessage: guardedCustomMessage,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
import { toJsonValue } from '@n8n/utils/json/to-json-value';
|
||||
|
||||
import type { AgentMessage, MessageContent } from '../../types/sdk/message';
|
||||
import { estimateObservationTokens } from '../../types/sdk/observation-log';
|
||||
import type { JSONObject, JSONValue } from '../../types/utils/json';
|
||||
|
||||
export const MAX_MODEL_TOOL_RESULT_TOKENS = 50_000;
|
||||
export const MAX_MODEL_TOOL_RESULT_CHARS = MAX_MODEL_TOOL_RESULT_TOKENS * 4;
|
||||
|
||||
interface TruncatedToolResult extends JSONObject {
|
||||
_truncated: true;
|
||||
originalCharCount: number;
|
||||
estimatedTokenCount: number;
|
||||
head: string;
|
||||
tail: string;
|
||||
}
|
||||
|
||||
export interface GuardedToolResult {
|
||||
historyOutput: JSONValue;
|
||||
wireOutput: unknown;
|
||||
truncated: boolean;
|
||||
}
|
||||
|
||||
export function guardToolResultForModel(output: unknown): GuardedToolResult {
|
||||
const historyOutput = toJsonValue(output);
|
||||
const serialized = JSON.stringify(historyOutput);
|
||||
|
||||
if (estimateObservationTokens(serialized) <= MAX_MODEL_TOOL_RESULT_TOKENS) {
|
||||
return { historyOutput, wireOutput: output, truncated: false };
|
||||
}
|
||||
|
||||
const truncated = buildTruncationEnvelope(serialized);
|
||||
return { historyOutput: truncated, wireOutput: truncated, truncated: true };
|
||||
}
|
||||
|
||||
export function guardToolErrorForModel(errorText: string): string {
|
||||
const guarded = guardToolResultForModel(errorText);
|
||||
return guarded.truncated ? JSON.stringify(guarded.historyOutput) : errorText;
|
||||
}
|
||||
|
||||
export function guardToolMessageForModel(message: AgentMessage): AgentMessage {
|
||||
if (!('content' in message)) return message;
|
||||
|
||||
const textBlocks = message.content.filter((block) => block.type === 'text');
|
||||
if (textBlocks.length === 0) return message;
|
||||
|
||||
const serialized = JSON.stringify(textBlocks.map(({ text }) => text));
|
||||
if (estimateObservationTokens(serialized) <= MAX_MODEL_TOOL_RESULT_TOKENS) return message;
|
||||
|
||||
const replacement = JSON.stringify(buildTruncationEnvelope(serialized));
|
||||
let replacedText = false;
|
||||
const content = message.content.flatMap((block): MessageContent[] => {
|
||||
if (block.type !== 'text') return [block];
|
||||
if (replacedText) return [];
|
||||
|
||||
replacedText = true;
|
||||
return [{ ...block, text: replacement }];
|
||||
});
|
||||
|
||||
return { ...message, content };
|
||||
}
|
||||
|
||||
function buildTruncationEnvelope(serialized: string): TruncatedToolResult {
|
||||
const base = {
|
||||
_truncated: true,
|
||||
originalCharCount: serialized.length,
|
||||
estimatedTokenCount: estimateObservationTokens(serialized),
|
||||
} as const;
|
||||
let low = 0;
|
||||
let high = Math.min(serialized.length, MAX_MODEL_TOOL_RESULT_CHARS);
|
||||
let best: TruncatedToolResult = { ...base, head: '', tail: '' };
|
||||
|
||||
while (low <= high) {
|
||||
const excerptLength = Math.floor((low + high) / 2);
|
||||
const { head, tail } = splitHeadAndTail(serialized, excerptLength);
|
||||
const candidate: TruncatedToolResult = { ...base, head, tail };
|
||||
|
||||
if (JSON.stringify(candidate).length <= MAX_MODEL_TOOL_RESULT_CHARS) {
|
||||
best = candidate;
|
||||
low = excerptLength + 1;
|
||||
} else {
|
||||
high = excerptLength - 1;
|
||||
}
|
||||
}
|
||||
|
||||
return best;
|
||||
}
|
||||
function splitHeadAndTail(value: string, excerptLength: number): { head: string; tail: string } {
|
||||
const headLength = Math.ceil(excerptLength / 2);
|
||||
const tailLength = Math.floor(excerptLength / 2);
|
||||
return {
|
||||
head: value.slice(0, headLength),
|
||||
tail: tailLength === 0 ? '' : value.slice(-tailLength),
|
||||
};
|
||||
}
|
||||
@@ -72,6 +72,9 @@ Custom tools are last resort and only for pure computation. Load
|
||||
|
||||
### Gotchas
|
||||
|
||||
- Generic web search must use \`config.webSearch\`; never add an HTTP Request
|
||||
Tool unless the user explicitly requests direct HTTP, API, or specific-page
|
||||
fetching.
|
||||
- Live crawling, fetching, and API integrations need MCP, workflow, or node tools, not custom tools.
|
||||
- Do not invent MCP servers, node type names, workflow names, credential ids, or provider tool keys.
|
||||
|
||||
|
||||
@@ -88,6 +88,17 @@ For callable (non-chat) services, call \`resolve_integration\` separately per
|
||||
service and follow the returned \`kind\`: \`"mcp"\` -> MCP Servers section
|
||||
below, \`"node"\` -> Node Tools section below.
|
||||
|
||||
## Web Search vs Direct HTTP Requests
|
||||
|
||||
Generic web search, browsing, research, current-information, and source-finding
|
||||
requests must use \`config.webSearch\` according to the system prompt's
|
||||
web-search rules. Do not call \`resolve_integration\` or \`search_nodes\` for
|
||||
these requests.
|
||||
|
||||
Never add \`n8n-nodes-base.httpRequestTool\` or
|
||||
\`@n8n/n8n-nodes-langchain.toolHttpRequest\` unless the user explicitly
|
||||
requests the HTTP Request Tool or direct HTTP, API, or specific-page fetching.
|
||||
|
||||
## Chat Integrations
|
||||
|
||||
The \`integrations\` array controls how the target agent is triggered.
|
||||
|
||||
Reference in New Issue
Block a user