Add changeset

This reverts commit ca17a06618, reversing
changes made to 6b9c2a1a45.

.changeset/late-cougars-hear.md

@@ -0,0 +1,5 @@
+---
+"claude-dev": patch
+---
+
+Add hard limit for file size Cline reads into context

.changeset/seven-flowers-lay.md

@@ -1,5 +0,0 @@
----
-"claude-dev": patch
----
-
-Fix a bug where cline crashes when reading large data from files

src/core/Cline.ts

@@ -62,7 +62,6 @@ import { ClineHandler } from "../api/providers/cline"
 import { ClineProvider, GlobalFileNames } from "./webview/ClineProvider"
 import { DEFAULT_LANGUAGE_SETTINGS, getLanguageKey, LanguageDisplay, LanguageKey } from "../shared/Languages"
 import { telemetryService } from "../services/telemetry/TelemetryService"
-import { getMaxAllowedSize } from "../utils/content-size"
 
 const cwd = vscode.workspace.workspaceFolders?.map((folder) => folder.uri.fsPath).at(0) ?? path.join(os.homedir(), "Desktop") // may or may not exist but fs checking existence would immediately ask for permission which would be bad UX, need to come up with a better solution
 
@@ -1354,8 +1353,25 @@ export class Cline {
 			if (previousRequest && previousRequest.text) {
 				const { tokensIn, tokensOut, cacheWrites, cacheReads }: ClineApiReqInfo = JSON.parse(previousRequest.text)
 				const totalTokens = (tokensIn || 0) + (tokensOut || 0) + (cacheWrites || 0) + (cacheReads || 0)
-				let contextWindow = this.api.getModel().info.contextWindow || 64_000 // minimum context (Deepseek)
-				const maxAllowedSize = getMaxAllowedSize(contextWindow)
+				let contextWindow = this.api.getModel().info.contextWindow || 128_000
+				// FIXME: hack to get anyone using openai compatible with deepseek to have the proper context window instead of the default 128k. We need a way for the user to specify the context window for models they input through openai compatible
+				if (this.api instanceof OpenAiHandler && this.api.getModel().id.toLowerCase().includes("deepseek")) {
+					contextWindow = 64_000
+				}
+				let maxAllowedSize: number
+				switch (contextWindow) {
+					case 64_000: // deepseek models
+						maxAllowedSize = contextWindow - 27_000
+						break
+					case 128_000: // most models
+						maxAllowedSize = contextWindow - 30_000
+						break
+					case 200_000: // claude models
+						maxAllowedSize = contextWindow - 40_000
+						break
+					default:
+						maxAllowedSize = Math.max(contextWindow - 40_000, contextWindow * 0.8) // for deepseek, 80% of 64k meant only ~10k buffer which was too small and resulted in users getting context window errors.
+				}
 
 				// This is the most reliable way to know when we're close to hitting the context window.
 				if (totalTokens >= maxAllowedSize) {
@@ -1999,17 +2015,15 @@ export class Cline {
 									}
 									telemetryService.captureToolUsage(this.taskId, block.name, false, true)
 								}
-								// Get context window and used context from API model
-								const contextWindow = this.api.getModel().info.contextWindow
-
-								// Pass the raw context window size - extractTextFromFile will calculate the appropriate limit
-								const content = await extractTextFromFile(absolutePath, contextWindow)
+								// now execute the tool like normal
+								const content = await extractTextFromFile(absolutePath)
 								pushToolResult(content)
 
 								break
 							}
 						} catch (error) {
 							await handleError("reading file", error)
+
 							break
 						}
 					}
@@ -3376,10 +3390,9 @@ export class Cline {
 							block.text.includes("<task>") ||
 							block.text.includes("<user_message>")
 						) {
-							let contextWindow = this.api.getModel().info.contextWindow
 							return {
 								...block,
-								text: await parseMentions(block.text, cwd, this.urlContentFetcher, contextWindow),
+								text: await parseMentions(block.text, cwd, this.urlContentFetcher),
 							}
 						}
 					}
@@ -3484,12 +3497,11 @@ export class Cline {
 				}
 			}
 		}
-
 		// only show inactive terminals if there's output to show
 		if (inactiveTerminals.length > 0) {
 			const inactiveTerminalOutputs = new Map<number, string>()
 			for (const inactiveTerminal of inactiveTerminals) {
-				const newOutput = await this.terminalManager.getUnretrievedOutput(inactiveTerminal.id)
+				const newOutput = this.terminalManager.getUnretrievedOutput(inactiveTerminal.id)
 				if (newOutput) {
 					inactiveTerminalOutputs.set(inactiveTerminal.id, newOutput)
 				}

src/core/mentions/index.ts

@@ -38,12 +38,7 @@ export function openMention(mention?: string): void {
 	}
 }
 
-export async function parseMentions(
-	text: string,
-	cwd: string,
-	urlContentFetcher: UrlContentFetcher,
-	contextWindow?: number,
-): Promise<string> {
+export async function parseMentions(text: string, cwd: string, urlContentFetcher: UrlContentFetcher): Promise<string> {
 	const mentions: Set<string> = new Set()
 	let parsedText = text.replace(mentionRegexGlobal, (match, mention) => {
 		mentions.add(mention)
@@ -95,7 +90,7 @@ export async function parseMentions(
 		} else if (mention.startsWith("/")) {
 			const mentionPath = mention.slice(1)
 			try {
-				const content = await getFileOrFolderContent(mentionPath, cwd, contextWindow)
+				const content = await getFileOrFolderContent(mentionPath, cwd)
 				if (mention.endsWith("/")) {
 					parsedText += `\n\n<folder_content path="${mentionPath}">\n${content}\n</folder_content>`
 				} else {
@@ -150,7 +145,7 @@ export async function parseMentions(
 	return parsedText
 }
 
-async function getFileOrFolderContent(mentionPath: string, cwd: string, contextWindow?: number): Promise<string> {
+async function getFileOrFolderContent(mentionPath: string, cwd: string): Promise<string> {
 	const absPath = path.resolve(cwd, mentionPath)
 
 	try {
@@ -161,7 +156,7 @@ async function getFileOrFolderContent(mentionPath: string, cwd: string, contextW
 			if (isBinary) {
 				return "(Binary file, unable to display content)"
 			}
-			const content = await extractTextFromFile(absPath, contextWindow)
+			const content = await extractTextFromFile(absPath)
 			return content
 		} else if (stats.isDirectory()) {
 			const entries = await fs.readdir(absPath, { withFileTypes: true })
@@ -182,7 +177,7 @@ async function getFileOrFolderContent(mentionPath: string, cwd: string, contextW
 								if (isBinary) {
 									return undefined
 								}
-								const content = await extractTextFromFile(absoluteFilePath, contextWindow)
+								const content = await extractTextFromFile(absoluteFilePath)
 								return `<file_content path="${filePath.toPosix()}">\n${content}\n</file_content>`
 							} catch (error) {
 								return undefined

src/integrations/misc/extract-text.test.ts

@@ -1,67 +0,0 @@
-import { expect } from "chai"
-import { extractTextFromFile } from "./extract-text"
-import fs from "fs/promises"
-import path from "path"
-import os from "os"
-import { ContentTooLargeError } from "../../shared/errors"
-
-const CONTEXT_LIMIT = 1000 // Context limit of 1000 tokens means max allowed size is 500 tokens
-
-describe("extract-text", () => {
-	let tempFilePath: string
-
-	beforeEach(async () => {
-		tempFilePath = path.join(os.tmpdir(), "test-file.txt")
-	})
-
-	afterEach(async () => {
-		await fs.unlink(tempFilePath).catch(() => {})
-	})
-
-	it("throws error for non-existent file", async () => {
-		const nonExistentPath = path.join(os.tmpdir(), "non-existent.txt")
-		try {
-			await extractTextFromFile(nonExistentPath, CONTEXT_LIMIT)
-			throw new Error("Should have thrown error")
-		} catch (error) {
-			expect(error.message).to.include("File not found")
-		}
-	})
-
-	it("throws ContentTooLargeError when file would exceed max allowed size", async () => {
-		// Create content that would exceed max allowed size (37k tokens)
-		const largeContent = "x".repeat(148000) // 37k tokens
-		await fs.writeFile(tempFilePath, largeContent)
-
-		try {
-			await extractTextFromFile(tempFilePath, 37_000) // Pass pre-processed maxAllowedSize
-			throw new Error("Should have thrown error")
-		} catch (error) {
-			expect(error).to.be.instanceOf(ContentTooLargeError)
-			expect(error.details.type).to.equal("file")
-			expect(error.details.path).to.equal(tempFilePath)
-			expect(error.details.size.wouldExceedLimit).to.equal(true)
-		}
-	})
-
-	it("reads text file content when within size limit", async () => {
-		const content = "Hello world"
-		await fs.writeFile(tempFilePath, content)
-
-		const result = await extractTextFromFile(tempFilePath, CONTEXT_LIMIT)
-		expect(result).to.equal(content)
-	})
-
-	it("throws error for binary files", async () => {
-		// Create a simple binary file
-		const buffer = new Uint8Array([0x89, 0x50, 0x4e, 0x47]) // PNG file header
-		await fs.writeFile(tempFilePath, buffer, { encoding: "binary" })
-
-		try {
-			await extractTextFromFile(tempFilePath, CONTEXT_LIMIT)
-			throw new Error("Should have thrown error")
-		} catch (error) {
-			expect(error.message).to.include("Cannot read text for file type")
-		}
-	})
-})

src/integrations/misc/extract-text.ts

@@ -4,102 +4,35 @@ import pdf from "pdf-parse/lib/pdf-parse"
 import mammoth from "mammoth"
 import fs from "fs/promises"
 import { isBinaryFile } from "isbinaryfile"
-import { estimateContentSize, estimateFileSize, wouldExceedSizeLimit, getMaxAllowedSize } from "../../utils/content-size"
-import { ContentTooLargeError } from "../../shared/errors"
+import { getFileSizeInKB } from "../../utils/fs"
 
-/**
- * Checks if terminal output would exceed size limits and returns the content if safe
- * @param content The terminal output content to check
- * @param contextWindow Context window limit in tokens
- * @param command The command that generated this output (for error reporting)
- * @returns The validated content
- * @throws ContentTooLargeError if content exceeds size limit
- */
-export async function extractTextFromTerminal(content: string | Buffer, contextWindow: number, command: string): Promise<string> {
-	console.debug(`[TERMINAL_SIZE_CHECK] Checking size for command output: ${command}`)
-
-	// Convert to string but don't trim yet
-	const rawContent = content.toString()
-	console.debug(`[TERMINAL_SIZE_CHECK] Raw content length: ${rawContent.length}`)
-
-	// Check size before trimming
-	const sizeEstimate = estimateContentSize(rawContent, contextWindow)
-	console.debug(`[TERMINAL_SIZE_CHECK] Content size: ${sizeEstimate.bytes} bytes`)
-	console.debug(`[TERMINAL_SIZE_CHECK] Estimated tokens: ${sizeEstimate.estimatedTokens}`)
-	console.debug(`[TERMINAL_SIZE_CHECK] Context window: ${contextWindow}`)
-
-	if (sizeEstimate.wouldExceedLimit) {
-		console.debug(`[TERMINAL_SIZE_CHECK] Output exceeds size limit`)
-		throw new ContentTooLargeError({
-			type: "terminal",
-			command,
-			size: sizeEstimate,
-		})
-	}
-
-	// Only trim after size check passes
-	const cleanContent = rawContent.trim()
-	console.debug(`[TERMINAL_SIZE_CHECK] Clean content length: ${cleanContent.length}`)
-	console.debug(`[TERMINAL_SIZE_CHECK] Size check passed`)
-	return cleanContent
-}
-
-export async function extractTextFromFile(
-	filePath: string,
-	contextWindow: number = 64_000 /* minimum context (Deepseek) */,
-): Promise<string> {
+export async function extractTextFromFile(filePath: string): Promise<string> {
 	try {
 		await fs.access(filePath)
 	} catch (error) {
 		throw new Error(`File not found: ${filePath}`)
 	}
-
-	console.debug(`[FILE_READ_CHECK] Checking size for file: ${filePath}`)
-
-	// Get file stats to check size
-	const stats = await fs.stat(filePath)
-	console.debug(`[FILE_SIZE_CHECK] File size: ${stats.size} bytes`)
-
-	// Calculate max allowed size from context window
-	const maxAllowedSize = getMaxAllowedSize(contextWindow)
-	console.debug(`[FILE_SIZE_CHECK] Max allowed size: ${maxAllowedSize} tokens`)
-
-	// Check if file size would exceed limit before attempting to read
-	// This is more efficient than creating a full SizeEstimate object when we just need a boolean check
-	if (wouldExceedSizeLimit(stats.size, contextWindow)) {
-		console.debug(`[FILE_SIZE_CHECK] File exceeds size limit`)
-		// Only create the full size estimate when we need it for the error
-		const sizeEstimate = await estimateFileSize(filePath, maxAllowedSize)
-		throw new ContentTooLargeError({
-			type: "file",
-			path: filePath,
-			size: sizeEstimate,
-		})
-	}
-	console.debug(`[FILE_SIZE_CHECK] File size check passed`)
 	const fileExtension = path.extname(filePath).toLowerCase()
-	console.debug(`[FILE_READ] Reading file: ${filePath}`)
-	let content: string
 	switch (fileExtension) {
 		case ".pdf":
-			content = await extractTextFromPDF(filePath)
-			break
+			return extractTextFromPDF(filePath)
 		case ".docx":
-			content = await extractTextFromDOCX(filePath)
-			break
+			return extractTextFromDOCX(filePath)
 		case ".ipynb":
-			content = await extractTextFromIPYNB(filePath)
-			break
+			return extractTextFromIPYNB(filePath)
 		default:
 			const isBinary = await isBinaryFile(filePath).catch(() => false)
 			if (!isBinary) {
-				content = await fs.readFile(filePath, "utf8")
+				// If file is over 300KB, throw an error
+				const fileSizeInKB = await getFileSizeInKB(filePath)
+				if (fileSizeInKB > 300) {
+					throw new Error(`File is too large to read into context.`)
+				}
+				return await fs.readFile(filePath, "utf8")
 			} else {
 				throw new Error(`Cannot read text for file type: ${fileExtension}`)
 			}
 	}
-	console.debug(`[FILE_READ_COMPLETE] File read complete. Content length: ${content.length} chars`)
-	return content
 }
 
 async function extractTextFromPDF(filePath: string): Promise<string> {

src/shared/errors.ts

@@ -1,18 +0,0 @@
-import { SizeEstimate } from "../utils/content-size"
-
-/**
- * Error thrown when content would exceed the model's context window limit
- */
-export class ContentTooLargeError extends Error {
-	constructor(
-		public details: {
-			type: "file" | "terminal"
-			path?: string
-			command?: string
-			size: SizeEstimate
-		},
-	) {
-		super("Content too large for context window")
-		this.name = "ContentTooLargeError"
-	}
-}

src/utils/content-size.test.ts

@@ -1,90 +0,0 @@
-import { expect } from "chai"
-import { estimateContentSize, estimateFileSize, estimateTokens, wouldExceedSizeLimit } from "./content-size"
-import fs from "fs/promises"
-import path from "path"
-import os from "os"
-
-const CONTEXT_LIMIT = 1000 // Context limit of 1000 tokens means max allowed size is 500 tokens
-
-describe("content-size", () => {
-	describe("estimateTokens", () => {
-		it("estimates tokens based on byte count", () => {
-			expect(estimateTokens(100)).to.equal(50) // 100 bytes / 2 chars per token = 50 tokens
-			expect(estimateTokens(7)).to.equal(4) // Should round up for partial tokens
-		})
-	})
-
-	describe("wouldExceedSizeLimit", () => {
-		it("checks if byte count would exceed max allowed size", () => {
-			// For deepseek (64k - 27k = 37k tokens)
-			expect(wouldExceedSizeLimit(100, 37_000)).to.equal(false) // 50 tokens < 37k tokens
-			expect(wouldExceedSizeLimit(148000, 37_000)).to.equal(true) // 74k tokens > 37k tokens
-			// For standard models (128k - 30k = 98k tokens)
-			expect(wouldExceedSizeLimit(392000, 98_000)).to.equal(true) // 196k tokens > 98k tokens
-		})
-	})
-
-	describe("estimateContentSize", () => {
-		it("estimates size for string content", () => {
-			const content = "Hello world" // 11 bytes
-			const result = estimateContentSize(content, CONTEXT_LIMIT)
-
-			expect(result.bytes).to.equal(11)
-			expect(result.estimatedTokens).to.equal(6)
-			expect(result.wouldExceedLimit).to.equal(false)
-		})
-
-		it("estimates size for buffer content", () => {
-			const content = Buffer.from("Hello world") // 11 bytes
-			const result = estimateContentSize(content, CONTEXT_LIMIT)
-
-			expect(result.bytes).to.equal(11)
-			expect(result.estimatedTokens).to.equal(6)
-			expect(result.wouldExceedLimit).to.equal(false)
-		})
-
-		it("detects when content would exceed max allowed size", () => {
-			// Create content that would exceed max allowed size for deepseek (64k - 27k tokens)
-			const largeContent = "x".repeat(148000) // 74k tokens > (64k - 27k) tokens
-			const result = estimateContentSize(largeContent, 64_000)
-
-			expect(result.wouldExceedLimit).to.equal(true)
-		})
-	})
-
-	describe("estimateFileSize", () => {
-		let tempFilePath: string
-
-		beforeEach(async () => {
-			tempFilePath = path.join(os.tmpdir(), "test-file.txt")
-			await fs.writeFile(tempFilePath, "Hello world") // 11 bytes
-		})
-
-		afterEach(async () => {
-			await fs.unlink(tempFilePath).catch(() => {})
-		})
-
-		it("estimates size for existing file", async () => {
-			const result = await estimateFileSize(tempFilePath, CONTEXT_LIMIT)
-
-			expect(result.bytes).to.equal(11)
-			expect(result.estimatedTokens).to.equal(6)
-			expect(result.wouldExceedLimit).to.equal(false)
-		})
-
-		/**
-		 * Verifies that estimateFileSize properly throws an error when given a
-		 * non-existent file path, which is important for error handling in the application.
-		 * The test expects fs.stat() inside estimateFileSize to fail and throw.
-		 */
-		it("throws error for non-existent file", async () => {
-			const nonExistentPath = path.join(os.tmpdir(), "non-existent.txt")
-			try {
-				await estimateFileSize(nonExistentPath, CONTEXT_LIMIT)
-				throw new Error("Should have thrown error")
-			} catch (error) {
-				expect(error).to.be.instanceOf(Error)
-			}
-		})
-	})
-})

src/utils/content-size.ts

@@ -1,79 +0,0 @@
-import { stat } from "fs/promises"
-
-// Rough approximation: 1 token ≈ 2 characters for English text
-const CHARS_PER_TOKEN = 2
-
-export interface SizeEstimate {
-	bytes: number
-	estimatedTokens: number
-	wouldExceedLimit: boolean
-}
-
-/**
- * Estimates tokens from byte count using a simple character ratio
- * This is a rough approximation - actual token count may vary
- */
-export function estimateTokens(bytes: number): number {
-	return Math.ceil(bytes / CHARS_PER_TOKEN)
-}
-
-/**
- * Checks if the given byte count would exceed the size limit
- * More efficient than creating a buffer just to check size
- */
-export function wouldExceedSizeLimit(byteCount: number, maxAllowedSize: number): boolean {
-	const estimatedTokenCount = estimateTokens(byteCount)
-	console.debug(`[WOULD_EXCEED_SIZE_CHECK] Estimated Token Count: ${estimatedTokenCount} tokens`)
-	return estimatedTokenCount >= maxAllowedSize
-}
-
-/**
- * Estimates size metrics for a string or buffer without loading entire content
- */
-export function estimateContentSize(content: string | Buffer, contextLimit: number): SizeEstimate {
-	const bytes = Buffer.isBuffer(content) ? content.length : Buffer.from(content).length
-	const estimatedTokenCount = estimateTokens(bytes)
-
-	return {
-		bytes,
-		estimatedTokens: estimatedTokenCount,
-		wouldExceedLimit: estimatedTokenCount >= getMaxAllowedSize(contextLimit),
-	}
-}
-
-/**
- * Gets size metrics for a file without reading its contents
- */
-export async function estimateFileSize(filePath: string, contextLimit: number): Promise<SizeEstimate> {
-	const stats = await stat(filePath)
-	const bytes = stats.size
-	const estimatedTokenCount = estimateTokens(bytes)
-
-	return {
-		bytes,
-		estimatedTokens: estimatedTokenCount,
-		wouldExceedLimit: estimatedTokenCount >= getMaxAllowedSize(contextLimit),
-	}
-}
-
-/**
- * Gets the maximum allowed size for the API context window
- */
-export function getMaxAllowedSize(contextWindow: number): number {
-	// Get context window and used context from API model
-	let maxAllowedSize: number
-	switch (contextWindow) {
-		case 64_000: // deepseek models
-			maxAllowedSize = contextWindow - 27_000
-			break
-		case 128_000: // most models
-			maxAllowedSize = contextWindow - 30_000
-			break
-		case 200_000: // claude models
-			maxAllowedSize = contextWindow - 40_000
-			break
-		default:
-			maxAllowedSize = Math.max(contextWindow - 40_000, contextWindow * 0.8)
-	}
-	return maxAllowedSize
-}

src/utils/fs.ts

@@ -1,6 +1,5 @@
 import fs from "fs/promises"
 import * as path from "path"
-import { extractTextFromFile } from "../integrations/misc/extract-text"
 
 /**
  * Asynchronously creates all non-existing subdirectories for a given file path
@@ -33,37 +32,10 @@ export async function createDirectoriesForFile(filePath: string): Promise<string
 }
 
 /**
- * Safely reads a configuration file with size checking
- * @param filePath Path to the configuration file
- * @param contextWindow Context window limit in tokens
- * @returns The file contents as a string
- */
-export async function readConfigFile(filePath: string, contextWindow: number): Promise<string> {
-	return await extractTextFromFile(filePath, contextWindow)
-}
-
-/**
- * Safely reads and parses a JSON configuration file
- * @param filePath Path to the JSON configuration file
- * @param contextWindow Context window limit in tokens
- * @returns The parsed configuration object
- */
-export async function readJsonConfigFile<T>(filePath: string, contextWindow: number): Promise<T> {
-	try {
-		const content = await readConfigFile(filePath, contextWindow)
-		return JSON.parse(content) as T
-	} catch (error) {
-		if (error instanceof SyntaxError) {
-			throw new Error(`Invalid JSON in config file ${filePath}: ${error.message}`, { cause: error })
-		}
-		throw new Error(`Failed to read config file ${filePath}: ${error instanceof Error ? error.message : String(error)}`)
-	}
-}
-
-/**
- * Helper function to check if a path exists
- * @param filePath The path to check
- * @returns A promise that resolves to true if the path exists, false otherwise
+ * Helper function to check if a path exists.
+ *
+ * @param path - The path to check.
+ * @returns A promise that resolves to true if the path exists, false otherwise.
  */
 export async function fileExistsAtPath(filePath: string): Promise<boolean> {
 	try {
@@ -87,3 +59,18 @@ export async function isDirectory(filePath: string): Promise<boolean> {
 		return false
 	}
 }
+
+/**
+ * Gets the size of a file in kilobytes
+ * @param filePath - Path to the file to check
+ * @returns Promise<number> - Size of the file in KB, or 0 if file doesn't exist
+ */
+export async function getFileSizeInKB(filePath: string): Promise<number> {
+	try {
+		const stats = await fs.stat(filePath)
+		const fileSizeInKB = stats.size / 1000 // Convert bytes to KB (decimal) - matches OS file size display
+		return fileSizeInKB
+	} catch {
+		return 0
+	}
+}