diff --git a/packages/@n8n/agents/src/runtime/__tests__/agent-runtime.test.ts b/packages/@n8n/agents/src/runtime/__tests__/agent-runtime.test.ts index 3ce45008a0d..cf4201ddd94 100644 --- a/packages/@n8n/agents/src/runtime/__tests__/agent-runtime.test.ts +++ b/packages/@n8n/agents/src/runtime/__tests__/agent-runtime.test.ts @@ -24,6 +24,7 @@ import { type DelegateSubAgentRunner, } from '../tools/delegate-sub-agent-tool'; import { toAiSdkTools } from '../tools/tool-adapter'; +import { MAX_MODEL_TOOL_RESULT_CHARS } from '../tools/tool-result-guard'; // --------------------------------------------------------------------------- // Module mocks @@ -7360,3 +7361,202 @@ describe('AgentRuntime — MCP connection failure warnings', () => { expect(systemText).not.toContain(''); }); }); + +describe('AgentRuntime — oversized tool results', () => { + type TruncationEnvelope = { + _truncated: true; + originalCharCount: number; + estimatedTokenCount: number; + head: string; + tail: string; + }; + + beforeEach(() => { + vi.clearAllMocks(); + }); + + function parseEnvelope(value: string): TruncationEnvelope { + try { + return JSON.parse(value) as TruncationEnvelope; + } catch { + throw new Error('Expected a valid truncation envelope'); + } + } + + function getModelToolResult(callIndex = 1): unknown { + const call = generateText.mock.calls[callIndex][0] as { + messages: Array<{ + role: string; + content: Array<{ type: string; output?: { type: string; value: unknown } }>; + }>; + }; + const toolMessage = call.messages.find((message) => message.role === 'tool'); + return toolMessage?.content.find((part) => part.type === 'tool-result')?.output?.value; + } + + it('leaves a small transformed result unchanged', async () => { + const tool: BuiltTool = { + name: 'small_result', + description: 'Return a small result', + inputSchema: z.object({}), + handler: async () => await Promise.resolve({ raw: 'value' }), + toModelOutput: () => ({ summary: 'small' }), + }; + const { runtime } = createRuntimeWithTools([tool], 1); + generateText + .mockResolvedValueOnce( + makeGenerateWithToolCalls([{ toolCallId: 'tc-small', toolName: tool.name, args: {} }]), + ) + .mockResolvedValueOnce(makeGenerateSuccess()); + + await runtime.generate('run'); + + expect(getModelToolResult()).toEqual({ summary: 'small' }); + }); + + it('bounds an oversized transformed result while preserving raw output', async () => { + const rawOutput = { + value: `RAW_HEAD${'r'.repeat(MAX_MODEL_TOOL_RESULT_CHARS)}RAW_TAIL`, + }; + const transformedOutput = `TRANSFORM_HEAD${'t'.repeat(MAX_MODEL_TOOL_RESULT_CHARS)}TRANSFORM_TAIL`; + const tool: BuiltTool = { + name: 'large_result', + description: 'Return a large result', + inputSchema: z.object({}), + handler: async () => await Promise.resolve(rawOutput), + toModelOutput: () => transformedOutput, + }; + const { runtime } = createRuntimeWithTools([tool], 1); + generateText + .mockResolvedValueOnce( + makeGenerateWithToolCalls([{ toolCallId: 'tc-large', toolName: tool.name, args: {} }]), + ) + .mockResolvedValueOnce(makeGenerateSuccess()); + + const result = await runtime.generate('run'); + const envelope = getModelToolResult() as TruncationEnvelope; + + expect(JSON.stringify(envelope).length).toBeLessThanOrEqual(MAX_MODEL_TOOL_RESULT_CHARS); + expect(envelope).toMatchObject({ + _truncated: true, + originalCharCount: JSON.stringify(transformedOutput).length, + }); + expect(envelope.head).toContain('TRANSFORM_HEAD'); + expect(envelope.tail).toContain('TRANSFORM_TAIL'); + expect(result.toolCalls?.[0]?.output).toEqual(rawOutput); + + const { runtime: streamRuntime } = createRuntimeWithTools([tool], 1); + streamText + .mockReturnValueOnce({ + stream: makeChunkStream([]), + finishReason: Promise.resolve('tool-calls'), + usage: Promise.resolve({ inputTokens: 10, outputTokens: 5, totalTokens: 15 }), + response: Promise.resolve({ + messages: [ + { + role: 'assistant', + content: [ + { + type: 'tool-call', + toolCallId: 'tc-large-stream', + toolName: tool.name, + args: {}, + }, + ], + }, + ], + }), + toolCalls: Promise.resolve([ + { toolCallId: 'tc-large-stream', toolName: tool.name, input: {} }, + ]), + }) + .mockReturnValueOnce(makeStreamSuccess()); + + const { stream } = await streamRuntime.stream('run'); + const chunks = await collectChunks(stream); + const streamResult = chunks.find( + (chunk) => chunk.type === 'tool-result' && chunk.toolCallId === 'tc-large-stream', + ) as (StreamChunk & { type: 'tool-result' }) | undefined; + + expect(streamResult?.output).toEqual(envelope); + }); + + it('bounds an oversized error without changing its rejected state', async () => { + const tool: BuiltTool = { + name: 'large_error', + description: 'Throw a large error', + inputSchema: z.object({}), + handler: () => { + throw new Error(`ERROR_HEAD${'e'.repeat(MAX_MODEL_TOOL_RESULT_CHARS)}ERROR_TAIL`); + }, + }; + const { runtime } = createRuntimeWithTools([tool], 1); + generateText + .mockResolvedValueOnce( + makeGenerateWithToolCalls([{ toolCallId: 'tc-error', toolName: tool.name, args: {} }]), + ) + .mockResolvedValueOnce(makeGenerateSuccess('recovered')); + + const result = await runtime.generate('run'); + const toolCall = result.messages + .flatMap((message) => ('content' in message ? message.content : [])) + .find( + (content): content is ContentToolCall => + content.type === 'tool-call' && content.toolCallId === 'tc-error', + ); + const envelope = parseEnvelope(toolCall?.state === 'rejected' ? toolCall.error : ''); + + expect(result.finishReason).toBe('stop'); + expect(toolCall?.state).toBe('rejected'); + expect(JSON.stringify(envelope).length).toBeLessThanOrEqual(MAX_MODEL_TOOL_RESULT_CHARS); + expect(envelope.head).toContain('ERROR_HEAD'); + expect(envelope.tail).toContain('ERROR_TAIL'); + }); + + it('bounds aggregate toMessage text while preserving file content', async () => { + const fileData = Buffer.from('file').toString('base64'); + const textBlockLength = Math.floor(MAX_MODEL_TOOL_RESULT_CHARS / 2); + const tool: BuiltTool = { + name: 'large_message', + description: 'Return a large message', + inputSchema: z.object({}), + handler: async () => await Promise.resolve({ ok: true }), + toMessage: () => ({ + role: 'assistant', + content: [ + { type: 'text', text: `MESSAGE_HEAD${'m'.repeat(textBlockLength)}` }, + { type: 'file', mediaType: 'text/plain', data: fileData }, + { type: 'text', text: `${'n'.repeat(textBlockLength)}MESSAGE_TAIL` }, + ], + }), + }; + const { runtime } = createRuntimeWithTools([tool], 1); + generateText + .mockResolvedValueOnce( + makeGenerateWithToolCalls([{ toolCallId: 'tc-message', toolName: tool.name, args: {} }]), + ) + .mockResolvedValueOnce(makeGenerateSuccess()); + + const result = await runtime.generate('run'); + const message = result.messages.find( + (candidate) => + 'content' in candidate && + candidate.content.some((content) => content.type === 'file' && content.data === fileData), + ); + const textBlocks = + message && 'content' in message + ? message.content.filter((content) => content.type === 'text') + : []; + const fileBlock = + message && 'content' in message + ? message.content.find((content) => content.type === 'file') + : undefined; + const envelope = parseEnvelope(textBlocks[0]?.text ?? ''); + + expect(textBlocks).toHaveLength(1); + expect(JSON.stringify(envelope).length).toBeLessThanOrEqual(MAX_MODEL_TOOL_RESULT_CHARS); + expect(envelope.head).toContain('MESSAGE_HEAD'); + expect(envelope.tail).toContain('MESSAGE_TAIL'); + expect(fileBlock).toMatchObject({ type: 'file', mediaType: 'text/plain', data: fileData }); + }); +}); diff --git a/packages/@n8n/agents/src/runtime/tools/tool-call-executor.ts b/packages/@n8n/agents/src/runtime/tools/tool-call-executor.ts index 7b109007b11..8ed4a07427a 100644 --- a/packages/@n8n/agents/src/runtime/tools/tool-call-executor.ts +++ b/packages/@n8n/agents/src/runtime/tools/tool-call-executor.ts @@ -1,4 +1,3 @@ -import { toJsonValue } from '@n8n/utils/json/to-json-value'; import { zodToJsonSchema, type JsonSchema7Type } from 'zod-to-json-schema'; import { @@ -7,6 +6,11 @@ import { } from './delegate-sub-agent-tool'; import { DEFAULT_SUB_AGENT_MAX_CHILDREN } from './sub-agent-task-path'; import { executeTool, isSuspendedToolResult, type SuspendedToolResult } from './tool-adapter'; +import { + guardToolErrorForModel, + guardToolMessageForModel, + guardToolResultForModel, +} from './tool-result-guard'; import { isAbortError, raceWithAbort } from '../../sdk/abort'; import { isCancellation } from '../../sdk/cancellation'; import { isLlmMessage } from '../../sdk/message'; @@ -24,6 +28,7 @@ import type { JSONObject, JSONValue } from '../../types/utils/json'; import { parseWithSchema } from '../../utils/parse'; import { isZodSchema } from '../../utils/zod'; import { incrementToolCallCount } from '../loop/execution-counter'; +import { stringifyError } from '../loop/runtime-helpers'; import type { AgentMessageList } from '../model/message-list'; import { normalizeToolInputForModel } from '../model/messages'; import type { AgentEventBus } from '../state/event-bus'; @@ -850,7 +855,7 @@ export class ToolCallExecutor { result: error, isError: true, }); - params.list.setToolCallError(params.toolCallId, error); + params.list.setToolCallError(params.toolCallId, guardToolErrorForModel(stringifyError(error))); return { outcome: 'error', error }; } @@ -1012,6 +1017,7 @@ export class ToolCallExecutor { } catch (error) { return this.toolError(params, error); } + const guardedResult = guardToolResultForModel(modelResult); this.eventBus.emit({ type: AgentEvent.ToolExecutionEnd, @@ -1021,11 +1027,14 @@ export class ToolCallExecutor { isError: false, }); - list.setToolCallResult(toolCallId, toJsonValue(modelResult)); + list.setToolCallResult(toolCallId, guardedResult.historyOutput); const customMessage = builtTool.toMessage?.(toolResult); - if (customMessage) { - list.addResponse([customMessage]); + const guardedCustomMessage = customMessage + ? guardToolMessageForModel(customMessage) + : undefined; + if (guardedCustomMessage) { + list.addResponse([guardedCustomMessage]); } return { @@ -1036,8 +1045,8 @@ export class ToolCallExecutor { output: toolResult, transformed: !!builtTool.toModelOutput, }, - modelOutput: modelResult, - customMessage, + modelOutput: guardedResult.wireOutput, + customMessage: guardedCustomMessage, }; } } diff --git a/packages/@n8n/agents/src/runtime/tools/tool-result-guard.ts b/packages/@n8n/agents/src/runtime/tools/tool-result-guard.ts new file mode 100644 index 00000000000..6b0820afed1 --- /dev/null +++ b/packages/@n8n/agents/src/runtime/tools/tool-result-guard.ts @@ -0,0 +1,95 @@ +import { toJsonValue } from '@n8n/utils/json/to-json-value'; + +import type { AgentMessage, MessageContent } from '../../types/sdk/message'; +import { estimateObservationTokens } from '../../types/sdk/observation-log'; +import type { JSONObject, JSONValue } from '../../types/utils/json'; + +export const MAX_MODEL_TOOL_RESULT_TOKENS = 50_000; +export const MAX_MODEL_TOOL_RESULT_CHARS = MAX_MODEL_TOOL_RESULT_TOKENS * 4; + +interface TruncatedToolResult extends JSONObject { + _truncated: true; + originalCharCount: number; + estimatedTokenCount: number; + head: string; + tail: string; +} + +export interface GuardedToolResult { + historyOutput: JSONValue; + wireOutput: unknown; + truncated: boolean; +} + +export function guardToolResultForModel(output: unknown): GuardedToolResult { + const historyOutput = toJsonValue(output); + const serialized = JSON.stringify(historyOutput); + + if (estimateObservationTokens(serialized) <= MAX_MODEL_TOOL_RESULT_TOKENS) { + return { historyOutput, wireOutput: output, truncated: false }; + } + + const truncated = buildTruncationEnvelope(serialized); + return { historyOutput: truncated, wireOutput: truncated, truncated: true }; +} + +export function guardToolErrorForModel(errorText: string): string { + const guarded = guardToolResultForModel(errorText); + return guarded.truncated ? JSON.stringify(guarded.historyOutput) : errorText; +} + +export function guardToolMessageForModel(message: AgentMessage): AgentMessage { + if (!('content' in message)) return message; + + const textBlocks = message.content.filter((block) => block.type === 'text'); + if (textBlocks.length === 0) return message; + + const serialized = JSON.stringify(textBlocks.map(({ text }) => text)); + if (estimateObservationTokens(serialized) <= MAX_MODEL_TOOL_RESULT_TOKENS) return message; + + const replacement = JSON.stringify(buildTruncationEnvelope(serialized)); + let replacedText = false; + const content = message.content.flatMap((block): MessageContent[] => { + if (block.type !== 'text') return [block]; + if (replacedText) return []; + + replacedText = true; + return [{ ...block, text: replacement }]; + }); + + return { ...message, content }; +} + +function buildTruncationEnvelope(serialized: string): TruncatedToolResult { + const base = { + _truncated: true, + originalCharCount: serialized.length, + estimatedTokenCount: estimateObservationTokens(serialized), + } as const; + let low = 0; + let high = Math.min(serialized.length, MAX_MODEL_TOOL_RESULT_CHARS); + let best: TruncatedToolResult = { ...base, head: '', tail: '' }; + + while (low <= high) { + const excerptLength = Math.floor((low + high) / 2); + const { head, tail } = splitHeadAndTail(serialized, excerptLength); + const candidate: TruncatedToolResult = { ...base, head, tail }; + + if (JSON.stringify(candidate).length <= MAX_MODEL_TOOL_RESULT_CHARS) { + best = candidate; + low = excerptLength + 1; + } else { + high = excerptLength - 1; + } + } + + return best; +} +function splitHeadAndTail(value: string, excerptLength: number): { head: string; tail: string } { + const headLength = Math.ceil(excerptLength / 2); + const tailLength = Math.floor(excerptLength / 2); + return { + head: value.slice(0, headLength), + tail: tailLength === 0 ? '' : value.slice(-tailLength), + }; +} diff --git a/packages/cli/src/modules/agents/builder/prompts/tools.prompt.ts b/packages/cli/src/modules/agents/builder/prompts/tools.prompt.ts index 0b2e13a9f62..e70d357b031 100644 --- a/packages/cli/src/modules/agents/builder/prompts/tools.prompt.ts +++ b/packages/cli/src/modules/agents/builder/prompts/tools.prompt.ts @@ -72,6 +72,9 @@ Custom tools are last resort and only for pure computation. Load ### Gotchas +- Generic web search must use \`config.webSearch\`; never add an HTTP Request + Tool unless the user explicitly requests direct HTTP, API, or specific-page + fetching. - Live crawling, fetching, and API integrations need MCP, workflow, or node tools, not custom tools. - Do not invent MCP servers, node type names, workflow names, credential ids, or provider tool keys. diff --git a/packages/cli/src/modules/agents/builder/skills/external-services.skill.ts b/packages/cli/src/modules/agents/builder/skills/external-services.skill.ts index aeeb9351a7c..76628b27cb2 100644 --- a/packages/cli/src/modules/agents/builder/skills/external-services.skill.ts +++ b/packages/cli/src/modules/agents/builder/skills/external-services.skill.ts @@ -88,6 +88,17 @@ For callable (non-chat) services, call \`resolve_integration\` separately per service and follow the returned \`kind\`: \`"mcp"\` -> MCP Servers section below, \`"node"\` -> Node Tools section below. +## Web Search vs Direct HTTP Requests + +Generic web search, browsing, research, current-information, and source-finding +requests must use \`config.webSearch\` according to the system prompt's +web-search rules. Do not call \`resolve_integration\` or \`search_nodes\` for +these requests. + +Never add \`n8n-nodes-base.httpRequestTool\` or +\`@n8n/n8n-nodes-langchain.toolHttpRequest\` unless the user explicitly +requests the HTTP Request Tool or direct HTTP, API, or specific-page fetching. + ## Chat Integrations The \`integrations\` array controls how the target agent is triggered.