fix(core): bound image payloads in read file results

sdk/packages/core/src/extensions/tools/definitions.test.ts

@@ -872,6 +872,44 @@ describe("default run_commands tool", () => {
 });
 
 describe("default read_files tool", () => {
+	const oneByOnePng = Buffer.from(
+		"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==",
+		"base64",
+	);
+
+	function paddedPngBase64(size: number): string {
+		return Buffer.concat([oneByOnePng, Buffer.alloc(size, 1)]).toString(
+			"base64",
+		);
+	}
+
+	function imageResult(data: string) {
+		return [
+			{ type: "text" as const, text: "Successfully read image" },
+			{ type: "image" as const, data, mediaType: "image/png" },
+		];
+	}
+
+	function countInlineImages(value: unknown): number {
+		if (!value || typeof value !== "object") {
+			return 0;
+		}
+		if (Array.isArray(value)) {
+			return value.reduce<number>(
+				(sum, item) => sum + countInlineImages(item),
+				0,
+			);
+		}
+		const record = value as Record<string, unknown>;
+		if (record.type === "image") {
+			return 1;
+		}
+		return Object.values(record).reduce<number>(
+			(sum, item) => sum + countInlineImages(item),
+			0,
+		);
+	}
+
 	it("validates ranged file requests and passes them to the executor", async () => {
 		const execute = vi.fn(async () => "selected lines");
 		const tool = createReadFilesTool(execute);
@@ -914,6 +952,46 @@ describe("default read_files tool", () => {
 		);
 	});
 
+	it("limits inline images in one read_files result and replaces overflow with metadata", async () => {
+		const firstImage = paddedPngBase64(128);
+		const omittedImage = paddedPngBase64(256);
+		const execute = vi
+			.fn()
+			.mockResolvedValueOnce(imageResult(firstImage))
+			.mockResolvedValueOnce(imageResult(omittedImage))
+			.mockResolvedValueOnce(imageResult(omittedImage));
+		const tool = createReadFilesTool(execute, {
+			readFilesMaxInlineImages: 1,
+			readFilesMaxInlineImagePayloadBytes: 10_000,
+		});
+
+		const result = await tool.execute(
+			{
+				files: [
+					{ path: "/tmp/frame-001.png" },
+					{ path: "/tmp/frame-002.png" },
+					{ path: "/tmp/frame-003.png" },
+				],
+			},
+			{
+				agentId: "agent-1",
+				conversationId: "conv-1",
+				iteration: 1,
+				metadata: { modelSupportsImages: true },
+			},
+		);
+
+		const serialized = JSON.stringify(result);
+		expect(countInlineImages(result)).toBe(1);
+		expect(serialized).toContain(firstImage);
+		expect(serialized).not.toContain(omittedImage);
+		expect(serialized).toContain("Image omitted due to budget.");
+		expect(serialized).toContain("/tmp/frame-002.png");
+		expect(serialized).toContain("Media type: image/png");
+		expect(serialized).toContain("Byte size:");
+		expect(serialized).toContain("Dimensions: 1x1");
+	});
+
 	it("accepts string union inputs reading full file content", async () => {
 		const execute = vi.fn(async () => "full file");
 		const tool = createReadFilesTool(execute);

sdk/packages/core/src/extensions/tools/definitions.ts

@@ -13,6 +13,11 @@ import {
 } from "@cline/shared";
 import { captureRunCommandsTimeout } from "../../services/telemetry/core-events";
 import { getToolContextTelemetry } from "../../services/telemetry/tool-context";
+import {
+	DEFAULT_MAX_IMAGE_PAYLOAD_BYTES_PER_TOOL_RESULT,
+	DEFAULT_MAX_INLINE_IMAGES_PER_TOOL_RESULT,
+	limitReadFilesToolOperationImages,
+} from "./executors/output-limits";
 import {
 	formatError,
 	formatReadFileQuery,
@@ -115,9 +120,20 @@ function captureRunCommandsTimeoutFromContext(
  */
 export function createReadFilesTool(
 	executor: FileReadExecutor,
-	config: Pick<DefaultToolsConfig, "fileReadTimeoutMs"> = {},
+	config: Pick<
+		DefaultToolsConfig,
+		| "fileReadTimeoutMs"
+		| "readFilesMaxInlineImages"
+		| "readFilesMaxInlineImagePayloadBytes"
+	> = {},
 ): AgentTool<ReadFilesInput, ToolOperationResult[]> {
 	const timeoutMs = config.fileReadTimeoutMs ?? 10000;
+	const maxInlineImages =
+		config.readFilesMaxInlineImages ??
+		DEFAULT_MAX_INLINE_IMAGES_PER_TOOL_RESULT;
+	const maxImagePayloadBytes =
+		config.readFilesMaxInlineImagePayloadBytes ??
+		DEFAULT_MAX_IMAGE_PAYLOAD_BYTES_PER_TOOL_RESULT;
 
 	return createTool<ReadFilesInput, ToolOperationResult[]>({
 		name: "read_files",
@@ -161,7 +177,7 @@ export function createReadFilesTool(
 				requests = [validate];
 			}
 
-			return Promise.all(
+			const results = await Promise.all(
 				requests.map(async (request): Promise<ToolOperationResult> => {
 					const rangeError = getReadFileRangeError(request);
 					if (rangeError) {
@@ -195,6 +211,10 @@ export function createReadFilesTool(
 					}
 				}),
 			);
+			return limitReadFilesToolOperationImages(results, {
+				maxInlineImages,
+				maxImagePayloadBytes,
+			});
 		},
 	});
 }

sdk/packages/core/src/extensions/tools/executors/output-limits.ts

@@ -0,0 +1,235 @@
+import type { ImageContent, TextContent } from "@cline/shared";
+import type { ToolOperationResult } from "../types";
+
+export const DEFAULT_MAX_INLINE_IMAGES_PER_TOOL_RESULT = 3;
+export const DEFAULT_MAX_IMAGE_PAYLOAD_BYTES_PER_TOOL_RESULT = 512 * 1024;
+
+interface LimitReadFilesImageOptions {
+	maxInlineImages: number;
+	maxImagePayloadBytes: number;
+}
+
+interface ImageDimensions {
+	width: number;
+	height: number;
+}
+
+export function limitReadFilesToolOperationImages(
+	operations: ToolOperationResult[],
+	options: LimitReadFilesImageOptions,
+): ToolOperationResult[] {
+	const maxInlineImages = normalizeLimit(options.maxInlineImages);
+	const maxImagePayloadBytes = normalizeLimit(options.maxImagePayloadBytes);
+	let inlineImages = 0;
+	let imagePayloadBytes = 0;
+	let changed = false;
+
+	const next = operations.map((operation) => {
+		if (!operation.success || !Array.isArray(operation.result)) {
+			return operation;
+		}
+
+		let omittedImage = false;
+		const result: unknown[] = [];
+		for (const item of operation.result) {
+			if (!isImageContent(item)) {
+				result.push(item);
+				continue;
+			}
+
+			const payloadBytes = imagePayloadByteLength(item);
+			if (
+				inlineImages < maxInlineImages &&
+				imagePayloadBytes + payloadBytes <= maxImagePayloadBytes
+			) {
+				inlineImages += 1;
+				imagePayloadBytes += payloadBytes;
+				result.push(item);
+				continue;
+			}
+
+			omittedImage = true;
+			changed = true;
+			result.push(imageOmittedTextBlock(item, operation.query));
+		}
+
+		if (!omittedImage) {
+			return operation;
+		}
+
+		return {
+			...operation,
+			result: result.filter(
+				(item) =>
+					!(
+						isTextContent(item) &&
+						item.text.trim() === "Successfully read image"
+					),
+			),
+		};
+	});
+
+	return changed ? next : operations;
+}
+
+export function imagePayloadByteLength(
+	image: Pick<ImageContent, "data">,
+): number {
+	return Buffer.byteLength(image.data, "utf8");
+}
+
+function imageOmittedTextBlock(
+	image: ImageContent,
+	filePath: string,
+): TextContent {
+	const bytes = Buffer.from(image.data, "base64");
+	const lines = [
+		"Image omitted due to budget.",
+		`Path: ${filePath}`,
+		`Media type: ${image.mediaType}`,
+		`Byte size: ${bytes.byteLength}`,
+	];
+	const dimensions = getImageDimensions(bytes, image.mediaType);
+	if (dimensions) {
+		lines.push(`Dimensions: ${dimensions.width}x${dimensions.height}`);
+	}
+	return { type: "text", text: lines.join("\n") };
+}
+
+function isImageContent(value: unknown): value is ImageContent {
+	if (!value || typeof value !== "object") {
+		return false;
+	}
+	const record = value as Record<string, unknown>;
+	return (
+		record.type === "image" &&
+		typeof record.data === "string" &&
+		typeof record.mediaType === "string"
+	);
+}
+
+function isTextContent(value: unknown): value is TextContent {
+	if (!value || typeof value !== "object") {
+		return false;
+	}
+	const record = value as Record<string, unknown>;
+	return record.type === "text" && typeof record.text === "string";
+}
+
+function normalizeLimit(value: number): number {
+	return Number.isFinite(value) && value > 0 ? Math.floor(value) : 0;
+}
+
+function getImageDimensions(
+	buffer: Buffer,
+	mediaType: string,
+): ImageDimensions | undefined {
+	if (mediaType === "image/png" && isPng(buffer)) {
+		return {
+			width: buffer.readUInt32BE(16),
+			height: buffer.readUInt32BE(20),
+		};
+	}
+	if (mediaType === "image/gif" && isGif(buffer)) {
+		return {
+			width: buffer.readUInt16LE(6),
+			height: buffer.readUInt16LE(8),
+		};
+	}
+	if (mediaType === "image/jpeg") {
+		return getJpegDimensions(buffer);
+	}
+	if (mediaType === "image/webp") {
+		return getWebpDimensions(buffer);
+	}
+	return undefined;
+}
+
+function isPng(buffer: Buffer): boolean {
+	return (
+		buffer.length >= 24 &&
+		buffer.readUInt32BE(0) === 0x89504e47 &&
+		buffer.readUInt32BE(4) === 0x0d0a1a0a
+	);
+}
+
+function isGif(buffer: Buffer): boolean {
+	return (
+		buffer.length >= 10 &&
+		(buffer.toString("ascii", 0, 6) === "GIF87a" ||
+			buffer.toString("ascii", 0, 6) === "GIF89a")
+	);
+}
+
+function getJpegDimensions(buffer: Buffer): ImageDimensions | undefined {
+	if (buffer.length < 4 || buffer[0] !== 0xff || buffer[1] !== 0xd8) {
+		return undefined;
+	}
+
+	let offset = 2;
+	while (offset + 9 < buffer.length) {
+		if (buffer[offset] !== 0xff) {
+			offset += 1;
+			continue;
+		}
+
+		const marker = buffer[offset + 1];
+		offset += 2;
+		if (marker === 0xd9 || marker === 0xda || offset + 2 > buffer.length) {
+			break;
+		}
+		const segmentLength = buffer.readUInt16BE(offset);
+		if (segmentLength < 2 || offset + segmentLength > buffer.length) {
+			break;
+		}
+		if (isJpegStartOfFrame(marker) && segmentLength >= 7) {
+			return {
+				height: buffer.readUInt16BE(offset + 3),
+				width: buffer.readUInt16BE(offset + 5),
+			};
+		}
+		offset += segmentLength;
+	}
+	return undefined;
+}
+
+function isJpegStartOfFrame(marker: number): boolean {
+	return (
+		(marker >= 0xc0 && marker <= 0xc3) ||
+		(marker >= 0xc5 && marker <= 0xc7) ||
+		(marker >= 0xc9 && marker <= 0xcb) ||
+		(marker >= 0xcd && marker <= 0xcf)
+	);
+}
+
+function getWebpDimensions(buffer: Buffer): ImageDimensions | undefined {
+	if (
+		buffer.length < 30 ||
+		buffer.toString("ascii", 0, 4) !== "RIFF" ||
+		buffer.toString("ascii", 8, 12) !== "WEBP"
+	) {
+		return undefined;
+	}
+
+	const chunkType = buffer.toString("ascii", 12, 16);
+	if (chunkType === "VP8X") {
+		return {
+			width: 1 + buffer.readUIntLE(24, 3),
+			height: 1 + buffer.readUIntLE(27, 3),
+		};
+	}
+	if (chunkType === "VP8 ") {
+		return {
+			width: buffer.readUInt16LE(26) & 0x3fff,
+			height: buffer.readUInt16LE(28) & 0x3fff,
+		};
+	}
+	if (chunkType === "VP8L" && buffer[20] === 0x2f) {
+		const bits = buffer.readUInt32LE(21);
+		return {
+			width: (bits & 0x3fff) + 1,
+			height: ((bits >> 14) & 0x3fff) + 1,
+		};
+	}
+	return undefined;
+}

sdk/packages/core/src/extensions/tools/types.ts

@@ -303,6 +303,18 @@ export interface DefaultToolsConfig {
 	 */
 	fileReadTimeoutMs?: number;
 
+	/**
+	 * Maximum image files to keep inline in one read_files result
+	 * @default 3
+	 */
+	readFilesMaxInlineImages?: number;
+
+	/**
+	 * Maximum base64 image payload bytes to keep inline in one read_files result
+	 * @default 524288
+	 */
+	readFilesMaxInlineImagePayloadBytes?: number;
+
 	/**
 	 * Timeout for bash command execution in milliseconds
 	 * @default 30000

sdk/packages/core/src/session/services/message-builder.test.ts

@@ -416,6 +416,24 @@ describe("MessageBuilder with structured ToolOperationResult content", () => {
 		return `${HEAD_MARKER}${filler}${MIDDLE_SENTINEL}${filler}${TAIL_MARKER}`;
 	}
 
+	const oneByOnePng = Buffer.from(
+		"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==",
+		"base64",
+	);
+
+	function paddedPngBase64(size: number, fill: number): string {
+		return Buffer.concat([oneByOnePng, Buffer.alloc(size, fill)]).toString(
+			"base64",
+		);
+	}
+
+	function imageReadResult(data: string) {
+		return [
+			{ type: "text" as const, text: "Successfully read image" },
+			{ type: "image" as const, data, mediaType: "image/png" },
+		];
+	}
+
 	function toolUseMessage(
 		id: string,
 		name: string,
@@ -671,4 +689,42 @@ describe("MessageBuilder with structured ToolOperationResult content", () => {
 		expect(serialized).toContain(HEAD_MARKER);
 		expect(serialized).toContain(TAIL_MARKER);
 	});
+
+	it("counts nested read_files image bytes in the aggregate provider budget", () => {
+		const inlineImage = paddedPngBase64(30_000, 1);
+		const builder = new MessageBuilder(50_000, new Set(["read_files"]), 40_000);
+		const messages: Message[] = [
+			toolUseMessage("call_1", "read_files", {
+				files: [{ path: "/tmp/frame-001.png" }],
+			}),
+			structuredToolResultMessage("call_1", "read_files", [
+				{
+					query: "/tmp/frame-001.png",
+					result: [
+						...imageReadResult(inlineImage),
+						{
+							type: "text" as const,
+							text: `ocr:${"x".repeat(20_000)}`,
+						},
+					],
+					success: true,
+				},
+			]),
+		];
+
+		const built = builder.buildForApi(messages);
+		const agentMessages = messagesToAgentMessages(built);
+		const aiSdkMessages = formatMessagesForAiSdk(
+			undefined,
+			agentMessages.map(({ role, content }) => ({
+				role,
+				content,
+			})) as unknown as AiSdkFormatterMessage[],
+		);
+		const serialized = JSON.stringify(aiSdkMessages);
+
+		expect(serialized).toContain(inlineImage);
+		expect(serialized).toContain("provider request budget");
+		expect(serialized.length).toBeLessThan(75_000);
+	});
 });

sdk/packages/core/src/session/services/message-builder.ts

@@ -11,11 +11,13 @@
 
 import {
 	type ContentBlock,
+	type ImageContent,
 	type Message,
 	normalizeUserInput,
 	type TextContent,
 	type ToolResultContent,
 } from "@cline/shared";
+import { imagePayloadByteLength } from "../../extensions/tools/executors/output-limits";
 
 const DEFAULT_MAX_TOOL_RESULT_CHARS = 50_000;
 const DEFAULT_MAX_TOTAL_TEXT_BYTES = 6_000_000;
@@ -799,7 +801,7 @@ export class MessageBuilder {
 			return changed ? next : value;
 		}
 		if (value !== null && typeof value === "object") {
-			if (isImageContentLike(value)) {
+			if (isImageContentBlock(value)) {
 				return value;
 			}
 			let changed = false;
@@ -829,7 +831,7 @@ export class MessageBuilder {
 			return messages;
 		}
 
-		let totalBytes = this.countMessageTextBytes(messages);
+		let totalBytes = this.countMessageProviderBytes(messages);
 		if (totalBytes <= this.maxTotalTextBytes) {
 			return messages;
 		}
@@ -872,7 +874,7 @@ export class MessageBuilder {
 		return next;
 	}
 
-	private countMessageTextBytes(messages: Message[]): number {
+	private countMessageProviderBytes(messages: Message[]): number {
 		let total = 0;
 		for (const message of messages) {
 			if (typeof message.content === "string") {
@@ -886,6 +888,8 @@ export class MessageBuilder {
 					total += utf8ByteLength(block.thinking);
 				} else if (block.type === "file") {
 					total += utf8ByteLength(block.content);
+				} else if (block.type === "image") {
+					total += imagePayloadByteLength(block);
 				} else if (block.type === "tool_result") {
 					if (typeof block.content === "string") {
 						total += utf8ByteLength(block.content);
@@ -895,8 +899,10 @@ export class MessageBuilder {
 								total += utf8ByteLength(entry.text);
 							} else if (entry.type === "file") {
 								total += utf8ByteLength(entry.content);
+							} else if (entry.type === "image") {
+								total += imagePayloadByteLength(entry);
 							} else if (isStructuredToolResultEntry(entry)) {
-								total += countNestedStringBytes(entry);
+								total += countNestedProviderBytes(entry);
 							}
 						}
 					}
@@ -1042,28 +1048,36 @@ function isStructuredToolResultEntry(entry: unknown): boolean {
 	return type !== "text" && type !== "image" && type !== "file";
 }
 
-function isImageContentLike(value: object): boolean {
-	return (value as { type?: unknown }).type === "image";
+function isImageContentBlock(value: unknown): value is ImageContent {
+	if (!value || typeof value !== "object") {
+		return false;
+	}
+	const record = value as Record<string, unknown>;
+	return (
+		record.type === "image" &&
+		typeof record.data === "string" &&
+		typeof record.mediaType === "string"
+	);
 }
 
-function countNestedStringBytes(value: unknown): number {
+function countNestedProviderBytes(value: unknown): number {
 	if (typeof value === "string") {
 		return utf8ByteLength(value);
 	}
 	if (Array.isArray(value)) {
 		let total = 0;
 		for (const item of value) {
-			total += countNestedStringBytes(item);
+			total += countNestedProviderBytes(item);
 		}
 		return total;
 	}
 	if (value !== null && typeof value === "object") {
-		if (isImageContentLike(value)) {
-			return 0;
+		if (isImageContentBlock(value)) {
+			return imagePayloadByteLength(value);
 		}
 		let total = 0;
 		for (const item of Object.values(value)) {
-			total += countNestedStringBytes(item);
+			total += countNestedProviderBytes(item);
 		}
 		return total;
 	}
@@ -1091,7 +1105,7 @@ function collectNestedStringCandidates(
 		return;
 	}
 	if (container !== null && typeof container === "object") {
-		if (isImageContentLike(container)) {
+		if (isImageContentBlock(container)) {
 			return;
 		}
 		const record = container as Record<string, unknown>;