From 4fc366df5f685285844c8e724ff407f9a9a676ae Mon Sep 17 00:00:00 2001 From: Saoud Rizwan <7799382+saoudrizwan@users.noreply.github.com> Date: Fri, 12 Jun 2026 17:23:25 -0700 Subject: [PATCH] fix(sdk): allow ranged reads on large files (#11511) * fix(sdk): allow ranged reads on large files * fix(sdk): bound ranged file reads * fix(sdk): bound streamed file reads * fix(sdk): simplify file read streaming bounds --- .../tools/executors/file-read.test.ts | 120 +++++++++++ .../extensions/tools/executors/file-read.ts | 198 +++++++++++++----- 2 files changed, 271 insertions(+), 47 deletions(-) diff --git a/sdk/packages/core/src/extensions/tools/executors/file-read.test.ts b/sdk/packages/core/src/extensions/tools/executors/file-read.test.ts index bf7307089f..2581c793c8 100644 --- a/sdk/packages/core/src/extensions/tools/executors/file-read.test.ts +++ b/sdk/packages/core/src/extensions/tools/executors/file-read.test.ts @@ -50,6 +50,17 @@ describe("createFileReadExecutor", () => { ); }); + it("bounds line counting for unranged reads after the capture window", async () => { + const result = await readTempFile(numberedLines(60_000)); + + expect(result).toContain("1 | line 1"); + expect(result).toContain("2000 | line 2000"); + expect(result).not.toContain("line 2001"); + expect(result).toContain( + "[Showing lines 1-2000 of 50000+ lines. Use start_line/end_line to read other sections.]", + ); + }); + it("honors explicit ranges beyond the default window start", async () => { const result = await readTempFile(numberedLines(2500), { start_line: 2400, @@ -59,6 +70,60 @@ describe("createFileReadExecutor", () => { expect(result).toBe("2400 | line 2400\n2401 | line 2401\n2402 | line 2402"); }); + it("reports the requested finite end line when a range exceeds the line cap", async () => { + const result = await readTempFile(numberedLines(3000), { + start_line: 1, + end_line: 3000, + }); + + expect(result).toContain("1 | line 1"); + expect(result).toContain("2000 | line 2000"); + expect(result).not.toContain("line 2001"); + expect(result).toContain( + "[Showing lines 1-2000 of 3000. Use start_line/end_line to read other sections.]", + ); + }); + + it("reads a requested range from a text file larger than the size gate", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "agents-file-read-")); + const filePath = path.join(dir, "large.txt"); + await fs.writeFile(filePath, numberedLines(100), "utf-8"); + + try { + const readFile = createFileReadExecutor({ maxFileSizeBytes: 10 }); + const result = (await readFile( + { path: filePath, start_line: 50, end_line: 52 }, + { agentId: "agent-1", conversationId: "conv-1", iteration: 1 }, + )) as string; + + expect(result).toBe("50 | line 50\n51 | line 51\n52 | line 52"); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } + }); + + it("returns a window for an unranged text file larger than the size gate", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "agents-file-read-")); + const filePath = path.join(dir, "large.txt"); + await fs.writeFile(filePath, numberedLines(2500), "utf-8"); + + try { + const readFile = createFileReadExecutor({ maxFileSizeBytes: 10 }); + const result = (await readFile( + { path: filePath }, + { agentId: "agent-1", conversationId: "conv-1", iteration: 1 }, + )) as string; + + expect(result).toContain("1 | line 1"); + expect(result).toContain("2000 | line 2000"); + expect(result).toContain( + "[Showing lines 1-2000 of 2500. Use start_line/end_line to read other sections.]", + ); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } + }); + it("truncates very long lines", async () => { const result = await readTempFile(`short\n${"y".repeat(5000)}\nend`); @@ -68,6 +133,52 @@ describe("createFileReadExecutor", () => { expect(result.length).toBeLessThan(2500); }); + it("rejects very large text files instead of streaming them", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "agents-file-read-")); + const filePath = path.join(dir, "huge.txt"); + await fs.writeFile(filePath, "hello", "utf-8"); + await fs.truncate(filePath, 100_000_001); + + try { + const readFile = createFileReadExecutor(); + + await expect( + readFile( + { path: filePath, start_line: 1, end_line: 1 }, + { agentId: "agent-1", conversationId: "conv-1", iteration: 1 }, + ), + ).rejects.toThrow("Text file too large to stream safely"); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } + }); + + it("rejects when the abort signal fires before reading", async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "agents-file-read-")); + const filePath = path.join(dir, "example.txt"); + await fs.writeFile(filePath, "hello", "utf-8"); + + try { + const controller = new AbortController(); + controller.abort(new Error("stop reading")); + const readFile = createFileReadExecutor(); + + await expect( + readFile( + { path: filePath }, + { + agentId: "agent-1", + conversationId: "conv-1", + iteration: 1, + signal: controller.signal, + }, + ), + ).rejects.toThrow("stop reading"); + } finally { + await fs.rm(dir, { recursive: true, force: true }); + } + }); + it("caps the returned window by characters for dense files", async () => { // 1500 lines of 100 chars each is ~150k chars, well over the ~50k cap // while staying under the 2000-line cap. @@ -81,6 +192,15 @@ describe("createFileReadExecutor", () => { expect(result).toContain("of 1500. Use start_line/end_line"); }); + it("keeps dense output capped when total line-number width grows after capture", async () => { + const result = await readTempFile( + Array.from({ length: 10_000 }, () => "z".repeat(100)).join("\n"), + ); + + expect(result.length).toBeLessThanOrEqual(50_000); + expect(result).toContain("of 10000. Use start_line/end_line"); + }); + it("returns image blocks for image files when the model supports images", async () => { const dir = await fs.mkdtemp(path.join(os.tmpdir(), "agents-file-read-")); const filePath = path.join(dir, "example.png"); diff --git a/sdk/packages/core/src/extensions/tools/executors/file-read.ts b/sdk/packages/core/src/extensions/tools/executors/file-read.ts index f52f32ad71..60097bca42 100644 --- a/sdk/packages/core/src/extensions/tools/executors/file-read.ts +++ b/sdk/packages/core/src/extensions/tools/executors/file-read.ts @@ -4,8 +4,10 @@ * Built-in implementation for reading files using Node.js fs module. */ +import { createReadStream } from "node:fs"; import * as fs from "node:fs/promises"; import * as path from "node:path"; +import { createInterface } from "node:readline"; import type { AgentToolContext } from "@cline/shared"; import { resolveExistingFilePath } from "@cline/shared/storage"; import type { ReadFileRequest } from "../schemas"; @@ -53,6 +55,139 @@ const DEFAULT_FILE_READ_OPTIONS: Required = { includeLineNumbers: true, // Include line numbers by default }; +const MAX_TEXT_STREAM_BYTES = 100_000_000; +const MAX_UNRANGED_LINE_SCAN = 50_000; + +interface CapturedLine { + lineNumber: number; + text: string; +} + +function getAbortError(signal: AbortSignal): Error { + const { reason } = signal; + if (reason instanceof Error) { + return reason; + } + if (reason !== undefined) { + return new Error(String(reason)); + } + return new Error("File read was aborted"); +} + +async function readTextWindow( + filePath: string, + encoding: BufferEncoding, + includeLineNumbers: boolean, + startLine: number | null | undefined, + endLine: number | null | undefined, + signal?: AbortSignal, +): Promise { + if (signal?.aborted) { + throw getAbortError(signal); + } + + const requestedStartLine = Math.max(startLine ?? 1, 1); + const requestedEndLine = endLine ?? Number.POSITIVE_INFINITY; + const hasFiniteEndLine = Number.isFinite(requestedEndLine); + const maxScannedLine = hasFiniteEndLine + ? requestedEndLine + : requestedStartLine + MAX_UNRANGED_LINE_SCAN - 1; + const captured: CapturedLine[] = []; + let chars = 0; + let totalLines = 0; + let capped = false; + let approximateTotalLines = false; + const maxCapturedLineNumber = Number.isFinite(requestedEndLine) + ? Math.min(requestedEndLine, requestedStartLine + MAX_READ_LINES - 1) + : requestedStartLine + MAX_READ_LINES - 1; + const lineNumberPrefixChars = includeLineNumbers + ? String(maxCapturedLineNumber).length + 3 + : 0; + + const stream = createReadStream(filePath, { encoding }); + const reader = createInterface({ + input: stream, + crlfDelay: Number.POSITIVE_INFINITY, + }); + const abortHandler = signal + ? () => stream.destroy(getAbortError(signal)) + : undefined; + + if (signal && abortHandler) { + signal.addEventListener("abort", abortHandler, { once: true }); + } + + try { + for await (const rawLine of reader) { + totalLines += 1; + if (totalLines > requestedEndLine) { + totalLines = requestedEndLine; + break; + } + if (!hasFiniteEndLine && capped && totalLines >= maxScannedLine) { + approximateTotalLines = true; + break; + } + if (totalLines < requestedStartLine || capped) { + continue; + } + if (captured.length >= MAX_READ_LINES) { + capped = true; + continue; + } + + let line = rawLine; + if (line.length > MAX_LINE_CHARS) { + line = `${line.slice(0, MAX_LINE_CHARS)} [line truncated]`; + } + + const nextChars = chars + line.length + lineNumberPrefixChars + 1; + if (nextChars > MAX_READ_OUTPUT_CHARS && captured.length > 0) { + capped = true; + continue; + } + + captured.push({ lineNumber: totalLines, text: line }); + chars = nextChars; + } + } finally { + if (signal && abortHandler) { + signal.removeEventListener("abort", abortHandler); + } + reader.close(); + stream.destroy(); + } + + const maxLineNumWidth = String( + captured[captured.length - 1]?.lineNumber ?? totalLines, + ).length; + const body = captured + .map(({ lineNumber, text }) => + includeLineNumbers + ? `${String(lineNumber).padStart(maxLineNumWidth, " ")} | ${text}` + : text, + ) + .join("\n"); + const lastCapturedLine = captured[captured.length - 1]?.lineNumber; + if (lastCapturedLine === undefined) { + return body; + } + + const effectiveEndLine = Math.min(requestedEndLine, totalLines); + if (lastCapturedLine >= effectiveEndLine) { + return body; + } + const totalLineText = approximateTotalLines + ? `${totalLines}+ lines` + : effectiveEndLine; + + return ( + `${body}\n\n` + + `[Showing lines ${requestedStartLine}-${lastCapturedLine} of ${totalLineText}. ` + + "Use start_line/end_line to read other sections.]" + ); +} + /** * Create a file read executor using Node.js fs module * @@ -93,15 +228,12 @@ export function createFileReadExecutor( throw new Error(`Path is not a file: ${resolvedPath}`); } - // Check file size - if (stat.size > maxFileSizeBytes) { - throw new Error( - `File too large: ${stat.size} bytes (max: ${maxFileSizeBytes} bytes). ` + - `Consider reading specific sections or using a different approach.`, - ); - } - if (imageMediaType) { + if (stat.size > maxFileSizeBytes) { + throw new Error( + `Image file too large: ${stat.size} bytes (max: ${maxFileSizeBytes} bytes).`, + ); + } if (context.metadata?.modelSupportsImages !== true) { throw new Error("Current model does not support image input"); } @@ -119,47 +251,19 @@ export function createFileReadExecutor( ]; } - // Read file content - const content = await fs.readFile(resolvedPath, encoding); - const allLines = content.split("\n"); - const rangeStart = Math.max((start_line ?? 1) - 1, 0); - const rangeEndExclusive = Math.min( - end_line ?? allLines.length, - allLines.length, - ); - const maxLineNumWidth = String(allLines.length).length; - - // Window the read: whole-file and oversized-range reads stop at the - // line and character caps so a single read cannot flood the - // conversation. The model pages through the rest with - // start_line/end_line. Line numbers are one-based for readability. - const lines: string[] = []; - let chars = 0; - let lineIndex = rangeStart; - while (lineIndex < rangeEndExclusive && lines.length < MAX_READ_LINES) { - let line = allLines[lineIndex]; - if (line.length > MAX_LINE_CHARS) { - line = `${line.slice(0, MAX_LINE_CHARS)} [line truncated]`; - } - if (includeLineNumbers) { - line = `${String(lineIndex + 1).padStart(maxLineNumWidth, " ")} | ${line}`; - } - if (chars + line.length + 1 > MAX_READ_OUTPUT_CHARS && lines.length > 0) { - break; - } - lines.push(line); - chars += line.length + 1; - lineIndex += 1; + if (stat.size > MAX_TEXT_STREAM_BYTES) { + throw new Error( + `Text file too large to stream safely: ${stat.size} bytes (max: ${MAX_TEXT_STREAM_BYTES} bytes). Use a targeted command such as sed, grep, head, or tail to inspect specific sections.`, + ); } - const body = lines.join("\n"); - if (lineIndex >= rangeEndExclusive) { - return body; - } - return ( - `${body}\n\n` + - `[Showing lines ${rangeStart + 1}-${lineIndex} of ${allLines.length}. ` + - "Use start_line/end_line to read other sections.]" + return readTextWindow( + resolvedPath, + encoding, + includeLineNumbers, + start_line, + end_line, + context.signal, ); }; }