From 1b56c3fc11182967d47e436ee97c7f926625ba4e Mon Sep 17 00:00:00 2001 From: Ocasta Date: Sun, 23 Feb 2025 11:15:40 -0800 Subject: [PATCH 01/19] new attempt --- src/core/Cline.ts | 42 ++++++++++- src/core/mentions/index.ts | 4 +- src/integrations/misc/extract-text.test.ts | 67 +++++++++++++++++ src/integrations/misc/extract-text.ts | 14 +++- src/integrations/terminal/TerminalManager.ts | 67 +++++++++-------- src/integrations/terminal/TerminalProcess.ts | 50 ++++++++++++- src/shared/errors.ts | 18 +++++ src/utils/content-size.test.ts | 79 ++++++++++++++++++++ src/utils/content-size.ts | 72 ++++++++++++++++++ tsconfig.test.json | 2 +- 10 files changed, 376 insertions(+), 39 deletions(-) create mode 100644 src/integrations/misc/extract-text.test.ts create mode 100644 src/shared/errors.ts create mode 100644 src/utils/content-size.test.ts create mode 100644 src/utils/content-size.ts diff --git a/src/core/Cline.ts b/src/core/Cline.ts index 0fbc3faeac..9fc08e9277 100644 --- a/src/core/Cline.ts +++ b/src/core/Cline.ts @@ -61,6 +61,7 @@ import { getNextTruncationRange, getTruncatedMessages } from "./sliding-window" import { ClineProvider, GlobalFileNames } from "./webview/ClineProvider" import { DEFAULT_LANGUAGE_SETTINGS, getLanguageKey, LanguageDisplay, LanguageKey } from "../shared/Languages" import { telemetryService } from "../services/telemetry/TelemetryService" +import { getMaxAllowedSize } from "../utils/content-size" const cwd = vscode.workspace.workspaceFolders?.map((folder) => folder.uri.fsPath).at(0) ?? path.join(os.homedir(), "Desktop") // may or may not exist but fs checking existence would immediately ask for permission which would be bad UX, need to come up with a better solution @@ -1166,9 +1167,26 @@ export class Cline { // Tools async executeCommandTool(command: string): Promise<[boolean, ToolResponse]> { + const contextWindow = this.api.getModel().info.contextWindow || 128_000 + const maxAllowedSize = getMaxAllowedSize(contextWindow) + const usedContext = this.apiConversationHistory.reduce((total, msg) => { + if (Array.isArray(msg.content)) { + return ( + total + + msg.content.reduce((acc, block) => { + if (block.type === "text") { + return acc + block.text.length / 4 // Rough estimate of tokens + } + return acc + }, 0) + ) + } + return total + (typeof msg.content === "string" ? msg.content.length / 4 : 0) + }, 0) + const terminalInfo = await this.terminalManager.getOrCreateTerminal(cwd) terminalInfo.terminal.show() // weird visual bug when creating new terminals (even manually) where there's an empty space at the top. - const process = this.terminalManager.runCommand(terminalInfo, command) + const process = this.terminalManager.runCommand(terminalInfo, command, maxAllowedSize, usedContext) let userFeedback: { text?: string; images?: string[] } | undefined let didContinue = false @@ -1994,8 +2012,28 @@ export class Cline { break } } + // Get context window and used context from API model + const contextWindow = this.api.getModel().info.contextWindow || 128_000 + const maxAllowedSize = getMaxAllowedSize(contextWindow) + + // Calculate used context from current conversation + const usedContext = this.apiConversationHistory.reduce((total, msg) => { + if (Array.isArray(msg.content)) { + return ( + total + + msg.content.reduce((acc, block) => { + if (block.type === "text") { + return acc + block.text.length / 4 // Rough estimate of tokens + } + return acc + }, 0) + ) + } + return total + (typeof msg.content === "string" ? msg.content.length / 4 : 0) + }, 0) + // now execute the tool like normal - const content = await extractTextFromFile(absolutePath) + const content = await extractTextFromFile(absolutePath, maxAllowedSize, usedContext) pushToolResult(content) break diff --git a/src/core/mentions/index.ts b/src/core/mentions/index.ts index 146e823f34..4c67d8fb4d 100644 --- a/src/core/mentions/index.ts +++ b/src/core/mentions/index.ts @@ -156,7 +156,7 @@ async function getFileOrFolderContent(mentionPath: string, cwd: string): Promise if (isBinary) { return "(Binary file, unable to display content)" } - const content = await extractTextFromFile(absPath) + const content = await extractTextFromFile(absPath, 128_000) // Use standard context window size return content } else if (stats.isDirectory()) { const entries = await fs.readdir(absPath, { withFileTypes: true }) @@ -177,7 +177,7 @@ async function getFileOrFolderContent(mentionPath: string, cwd: string): Promise if (isBinary) { return undefined } - const content = await extractTextFromFile(absoluteFilePath) + const content = await extractTextFromFile(absoluteFilePath, 128_000) // Use standard context window size return `\n${content}\n` } catch (error) { return undefined diff --git a/src/integrations/misc/extract-text.test.ts b/src/integrations/misc/extract-text.test.ts new file mode 100644 index 0000000000..f3798cb6be --- /dev/null +++ b/src/integrations/misc/extract-text.test.ts @@ -0,0 +1,67 @@ +import { expect } from "chai" +import { extractTextFromFile } from "./extract-text" +import fs from "fs/promises" +import path from "path" +import os from "os" +import { ContentTooLargeError } from "../../shared/errors" + +const CONTEXT_LIMIT = 1000 +const USED_CONTEXT = 200 + +describe("extract-text", () => { + let tempFilePath: string + + beforeEach(async () => { + tempFilePath = path.join(os.tmpdir(), "test-file.txt") + }) + + afterEach(async () => { + await fs.unlink(tempFilePath).catch(() => {}) + }) + + it("throws error for non-existent file", async () => { + const nonExistentPath = path.join(os.tmpdir(), "non-existent.txt") + try { + await extractTextFromFile(nonExistentPath, CONTEXT_LIMIT, USED_CONTEXT) + throw new Error("Should have thrown error") + } catch (error) { + expect(error.message).to.include("File not found") + } + }) + + it("throws ContentTooLargeError when file would exceed limit", async () => { + const largeContent = "x".repeat(3000) // 3000 bytes = ~750 tokens + await fs.writeFile(tempFilePath, largeContent) + + try { + await extractTextFromFile(tempFilePath, CONTEXT_LIMIT, USED_CONTEXT) + throw new Error("Should have thrown error") + } catch (error) { + expect(error).to.be.instanceOf(ContentTooLargeError) + expect(error.details.type).to.equal("file") + expect(error.details.path).to.equal(tempFilePath) + expect(error.details.size.wouldExceedLimit).to.equal(true) + } + }) + + it("reads text file content when within size limit", async () => { + const content = "Hello world" + await fs.writeFile(tempFilePath, content) + + const result = await extractTextFromFile(tempFilePath, CONTEXT_LIMIT, USED_CONTEXT) + expect(result).to.equal(content) + }) + + it("throws error for binary files", async () => { + // Create a simple binary file + const buffer = new Uint8Array([0x89, 0x50, 0x4e, 0x47]) // PNG file header + await fs.writeFile(tempFilePath, buffer, { encoding: "binary" }) + + try { + await extractTextFromFile(tempFilePath, CONTEXT_LIMIT, USED_CONTEXT) + throw new Error("Should have thrown error") + } catch (error) { + expect(error.message).to.include("Cannot read text for file type") + } + }) +}) diff --git a/src/integrations/misc/extract-text.ts b/src/integrations/misc/extract-text.ts index 67a580af9b..81b66ca813 100644 --- a/src/integrations/misc/extract-text.ts +++ b/src/integrations/misc/extract-text.ts @@ -4,13 +4,25 @@ import pdf from "pdf-parse/lib/pdf-parse" import mammoth from "mammoth" import fs from "fs/promises" import { isBinaryFile } from "isbinaryfile" +import { estimateFileSize } from "../../utils/content-size" +import { ContentTooLargeError } from "../../shared/errors" -export async function extractTextFromFile(filePath: string): Promise { +export async function extractTextFromFile(filePath: string, contextLimit: number, usedContext: number = 0): Promise { try { await fs.access(filePath) } catch (error) { throw new Error(`File not found: ${filePath}`) } + + // Check file size before attempting to read + const sizeEstimate = await estimateFileSize(filePath, contextLimit, usedContext) + if (sizeEstimate.wouldExceedLimit) { + throw new ContentTooLargeError({ + type: "file", + path: filePath, + size: sizeEstimate, + }) + } const fileExtension = path.extname(filePath).toLowerCase() switch (fileExtension) { case ".pdf": diff --git a/src/integrations/terminal/TerminalManager.ts b/src/integrations/terminal/TerminalManager.ts index eb640b8c9a..33e22212b9 100644 --- a/src/integrations/terminal/TerminalManager.ts +++ b/src/integrations/terminal/TerminalManager.ts @@ -1,5 +1,33 @@ import pWaitFor from "p-wait-for" import * as vscode from "vscode" + +/* +The new shellIntegration API gives us access to terminal command execution output handling. +However, we don't update our VSCode type definitions or engine requirements to maintain compatibility +with older VSCode versions. Users on older versions will automatically fall back to using sendText +for terminal command execution. +Interestingly, some environments like Cursor enable these APIs even without the latest VSCode engine. +This approach allows us to leverage advanced features when available while ensuring broad compatibility. +*/ +declare module "vscode" { + // https://github.com/microsoft/vscode/blob/f0417069c62e20f3667506f4b7e53ca0004b4e3e/src/vscode-dts/vscode.d.ts#L7442 + interface Terminal { + shellIntegration?: { + cwd?: vscode.Uri + executeCommand?: (command: string) => { + read: () => AsyncIterable + } + } + } + // https://github.com/microsoft/vscode/blob/f0417069c62e20f3667506f4b7e53ca0004b4e3e/src/vscode-dts/vscode.d.ts#L10794 + interface Window { + onDidStartTerminalShellExecution?: ( + listener: (e: any) => any, + thisArgs?: any, + disposables?: vscode.Disposable[], + ) => vscode.Disposable + } +} import { arePathsEqual } from "../../utils/path" import { mergePromise, TerminalProcess, TerminalProcessResultPromise } from "./TerminalProcess" import { TerminalInfo, TerminalRegistry } from "./TerminalRegistry" @@ -61,34 +89,6 @@ Resources: - https://github.com/microsoft/vscode-extension-samples/blob/main/shell-integration-sample/src/extension.ts */ -/* -The new shellIntegration API gives us access to terminal command execution output handling. -However, we don't update our VSCode type definitions or engine requirements to maintain compatibility -with older VSCode versions. Users on older versions will automatically fall back to using sendText -for terminal command execution. -Interestingly, some environments like Cursor enable these APIs even without the latest VSCode engine. -This approach allows us to leverage advanced features when available while ensuring broad compatibility. -*/ -declare module "vscode" { - // https://github.com/microsoft/vscode/blob/f0417069c62e20f3667506f4b7e53ca0004b4e3e/src/vscode-dts/vscode.d.ts#L7442 - interface Terminal { - shellIntegration?: { - cwd?: vscode.Uri - executeCommand?: (command: string) => { - read: () => AsyncIterable - } - } - } - // https://github.com/microsoft/vscode/blob/f0417069c62e20f3667506f4b7e53ca0004b4e3e/src/vscode-dts/vscode.d.ts#L10794 - interface Window { - onDidStartTerminalShellExecution?: ( - listener: (e: any) => any, - thisArgs?: any, - disposables?: vscode.Disposable[], - ) => vscode.Disposable - } -} - export class TerminalManager { private terminalIds: Set = new Set() private processes: Map = new Map() @@ -109,7 +109,12 @@ export class TerminalManager { } } - runCommand(terminalInfo: TerminalInfo, command: string): TerminalProcessResultPromise { + runCommand( + terminalInfo: TerminalInfo, + command: string, + contextLimit?: number, + usedContext?: number, + ): TerminalProcessResultPromise { terminalInfo.busy = true terminalInfo.lastCommand = command const process = new TerminalProcess() @@ -141,14 +146,14 @@ export class TerminalManager { // if shell integration is already active, run the command immediately if (terminalInfo.terminal.shellIntegration) { process.waitForShellIntegration = false - process.run(terminalInfo.terminal, command) + process.run(terminalInfo.terminal, command, contextLimit, usedContext) } else { // docs recommend waiting 3s for shell integration to activate pWaitFor(() => terminalInfo.terminal.shellIntegration !== undefined, { timeout: 4000 }).finally(() => { const existingProcess = this.processes.get(terminalInfo.id) if (existingProcess && existingProcess.waitForShellIntegration) { existingProcess.waitForShellIntegration = false - existingProcess.run(terminalInfo.terminal, command) + existingProcess.run(terminalInfo.terminal, command, contextLimit, usedContext) } }) } diff --git a/src/integrations/terminal/TerminalProcess.ts b/src/integrations/terminal/TerminalProcess.ts index 5597350db3..fa2997aeb2 100644 --- a/src/integrations/terminal/TerminalProcess.ts +++ b/src/integrations/terminal/TerminalProcess.ts @@ -1,6 +1,8 @@ import { EventEmitter } from "events" -import stripAnsi from "strip-ansi" +import * as stripAnsi from "strip-ansi" import * as vscode from "vscode" +import { ContentTooLargeError } from "../../shared/errors" +import { estimateContentSize } from "../../utils/content-size" export interface TerminalProcessEvents { line: [line: string] @@ -22,11 +24,22 @@ export class TerminalProcess extends EventEmitter { private lastRetrievedIndex: number = 0 isHot: boolean = false private hotTimer: NodeJS.Timeout | null = null + private totalBytes: number = 0 + private contextLimit: number = 100000 // Default context window size + private usedContext: number = 0 + private lastCommand: string = "" // constructor() { // super() - async run(terminal: vscode.Terminal, command: string) { + async run(terminal: vscode.Terminal, command: string, contextLimit?: number, usedContext?: number) { + if (contextLimit) { + this.contextLimit = contextLimit + } + if (usedContext) { + this.usedContext = usedContext + } + this.lastCommand = command if (terminal.shellIntegration && terminal.shellIntegration.executeCommand) { const execution = terminal.shellIntegration.executeCommand(command) const stream = execution.read() @@ -35,6 +48,24 @@ export class TerminalProcess extends EventEmitter { let didOutputNonCommand = false let didEmitEmptyLine = false for await (let data of stream) { + // Add to total bytes before checking size + const dataBytes = Buffer.from(data).length + this.totalBytes += dataBytes + + // Check total accumulated size + const sizeEstimate = estimateContentSize(Buffer.alloc(this.totalBytes), this.contextLimit, this.usedContext) + if (sizeEstimate.wouldExceedLimit) { + this.emit( + "error", + new ContentTooLargeError({ + type: "terminal", + command, + size: sizeEstimate, + }), + ) + return + } + // 1. Process chunk and remove artifacts if (isFirstChunk) { /* @@ -184,6 +215,21 @@ export class TerminalProcess extends EventEmitter { // Inspired by https://github.com/sindresorhus/execa/blob/main/lib/transform/split.js private emitIfEol(chunk: string) { + // Check size before adding to buffer + const newBufferSize = this.buffer.length + chunk.length + const sizeEstimate = estimateContentSize(Buffer.alloc(newBufferSize), this.contextLimit, this.usedContext) + if (sizeEstimate.wouldExceedLimit) { + this.emit( + "error", + new ContentTooLargeError({ + type: "terminal", + command: this.lastCommand, + size: sizeEstimate, + }), + ) + return + } + this.buffer += chunk let lineEndIndex: number while ((lineEndIndex = this.buffer.indexOf("\n")) !== -1) { diff --git a/src/shared/errors.ts b/src/shared/errors.ts new file mode 100644 index 0000000000..741e749383 --- /dev/null +++ b/src/shared/errors.ts @@ -0,0 +1,18 @@ +import { SizeEstimate } from "../utils/content-size" + +/** + * Error thrown when content would exceed the model's context window limit + */ +export class ContentTooLargeError extends Error { + constructor( + public details: { + type: "file" | "terminal" + path?: string + command?: string + size: SizeEstimate + }, + ) { + super("Content too large for context window") + this.name = "ContentTooLargeError" + } +} diff --git a/src/utils/content-size.test.ts b/src/utils/content-size.test.ts new file mode 100644 index 0000000000..d59510a654 --- /dev/null +++ b/src/utils/content-size.test.ts @@ -0,0 +1,79 @@ +import { expect } from "chai" +import { estimateContentSize, estimateFileSize, estimateTokens } from "./content-size" +import fs from "fs/promises" +import path from "path" +import os from "os" + +const CONTEXT_LIMIT = 1000 +const USED_CONTEXT = 200 + +describe("content-size", () => { + describe("estimateTokens", () => { + it("estimates tokens based on byte count", () => { + expect(estimateTokens(100)).to.equal(25) // 100 bytes / 4 chars per token = 25 tokens + expect(estimateTokens(7)).to.equal(2) // Should round up for partial tokens + }) + }) + + describe("estimateContentSize", () => { + it("estimates size for string content", () => { + const content = "Hello world" // 11 bytes + const result = estimateContentSize(content, CONTEXT_LIMIT, USED_CONTEXT) + + expect(result.bytes).to.equal(11) + expect(result.estimatedTokens).to.equal(3) + expect(result.remainingContextSize).to.equal(800) + expect(result.wouldExceedLimit).to.equal(false) + }) + + it("estimates size for buffer content", () => { + const content = Buffer.from("Hello world") // 11 bytes + const result = estimateContentSize(content, CONTEXT_LIMIT, USED_CONTEXT) + + expect(result.bytes).to.equal(11) + expect(result.estimatedTokens).to.equal(3) + expect(result.remainingContextSize).to.equal(800) + expect(result.wouldExceedLimit).to.equal(false) + }) + + it("detects when content would exceed limit", () => { + const largeContent = "x".repeat(3000) // 3000 bytes = ~750 tokens + const result = estimateContentSize(largeContent, CONTEXT_LIMIT, USED_CONTEXT) + + expect(result.wouldExceedLimit).to.equal(true) + expect(result.remainingContextSize).to.equal(800) + }) + }) + + describe("estimateFileSize", () => { + let tempFilePath: string + + beforeEach(async () => { + tempFilePath = path.join(os.tmpdir(), "test-file.txt") + await fs.writeFile(tempFilePath, "Hello world") // 11 bytes + }) + + afterEach(async () => { + await fs.unlink(tempFilePath).catch(() => {}) + }) + + it("estimates size for existing file", async () => { + const result = await estimateFileSize(tempFilePath, CONTEXT_LIMIT, USED_CONTEXT) + + expect(result.bytes).to.equal(11) + expect(result.estimatedTokens).to.equal(3) + expect(result.remainingContextSize).to.equal(800) + expect(result.wouldExceedLimit).to.equal(false) + }) + + it("throws error for non-existent file", async () => { + const nonExistentPath = path.join(os.tmpdir(), "non-existent.txt") + try { + await estimateFileSize(nonExistentPath, CONTEXT_LIMIT, USED_CONTEXT) + throw new Error("Should have thrown error") + } catch (error) { + expect(error).to.be.instanceOf(Error) + } + }) + }) +}) diff --git a/src/utils/content-size.ts b/src/utils/content-size.ts new file mode 100644 index 0000000000..f420045d07 --- /dev/null +++ b/src/utils/content-size.ts @@ -0,0 +1,72 @@ +import fs from "fs/promises" +import { stat } from "fs/promises" + +// Rough approximation: 1 token ≈ 4 characters for English text +const CHARS_PER_TOKEN = 4 + +export interface SizeEstimate { + bytes: number + estimatedTokens: number + wouldExceedLimit: boolean + remainingContextSize: number +} + +/** + * Estimates tokens from byte count using a simple character ratio + * This is a rough approximation - actual token count may vary + */ +export function estimateTokens(bytes: number): number { + return Math.ceil(bytes / CHARS_PER_TOKEN) +} + +/** + * Estimates size metrics for a string or buffer without loading entire content + */ +export function estimateContentSize(content: string | Buffer, contextLimit: number, usedContext: number = 0): SizeEstimate { + const bytes = Buffer.isBuffer(content) ? content.length : Buffer.from(content).length + const estimatedTokenCount = estimateTokens(bytes) + const remainingContext = contextLimit - usedContext + + return { + bytes, + estimatedTokens: estimatedTokenCount, + wouldExceedLimit: estimatedTokenCount > remainingContext, + remainingContextSize: remainingContext, + } +} + +/** + * Gets size metrics for a file without reading its contents + */ +export async function estimateFileSize(filePath: string, contextLimit: number, usedContext: number = 0): Promise { + const stats = await stat(filePath) + const bytes = stats.size + const estimatedTokenCount = estimateTokens(bytes) + const remainingContext = contextLimit - usedContext + + return { + bytes, + estimatedTokens: estimatedTokenCount, + wouldExceedLimit: estimatedTokenCount > remainingContext, + remainingContextSize: remainingContext, + } +} + +export function getMaxAllowedSize(contextWindow: number): number { + // Get context window and used context from API model + let maxAllowedSize: number + switch (contextWindow) { + case 64_000: // deepseek models + maxAllowedSize = contextWindow - 27_000 + break + case 128_000: // most models + maxAllowedSize = contextWindow - 30_000 + break + case 200_000: // claude models + maxAllowedSize = contextWindow - 40_000 + break + default: + maxAllowedSize = Math.max(contextWindow - 40_000, contextWindow * 0.8) + } + return maxAllowedSize +} diff --git a/tsconfig.test.json b/tsconfig.test.json index 92f67542f5..a60e9ddfc6 100644 --- a/tsconfig.test.json +++ b/tsconfig.test.json @@ -14,5 +14,5 @@ "rootDir": "src" }, "include": ["src/**/*.test.ts"], - "exclude": ["src/test/**/*.js"] + "exclude": ["src/test/**/*.js", "src/integrations/terminal/**/*"] } From 9f418b10f6622dc30f793a01f12be3768bb3a753 Mon Sep 17 00:00:00 2001 From: Ocasta Date: Sun, 23 Feb 2025 11:18:30 -0800 Subject: [PATCH 02/19] cleanup vscode module declaration --- src/integrations/terminal/TerminalManager.ts | 56 ++++++++++---------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/src/integrations/terminal/TerminalManager.ts b/src/integrations/terminal/TerminalManager.ts index 33e22212b9..6d8c2a36c9 100644 --- a/src/integrations/terminal/TerminalManager.ts +++ b/src/integrations/terminal/TerminalManager.ts @@ -1,33 +1,5 @@ import pWaitFor from "p-wait-for" import * as vscode from "vscode" - -/* -The new shellIntegration API gives us access to terminal command execution output handling. -However, we don't update our VSCode type definitions or engine requirements to maintain compatibility -with older VSCode versions. Users on older versions will automatically fall back to using sendText -for terminal command execution. -Interestingly, some environments like Cursor enable these APIs even without the latest VSCode engine. -This approach allows us to leverage advanced features when available while ensuring broad compatibility. -*/ -declare module "vscode" { - // https://github.com/microsoft/vscode/blob/f0417069c62e20f3667506f4b7e53ca0004b4e3e/src/vscode-dts/vscode.d.ts#L7442 - interface Terminal { - shellIntegration?: { - cwd?: vscode.Uri - executeCommand?: (command: string) => { - read: () => AsyncIterable - } - } - } - // https://github.com/microsoft/vscode/blob/f0417069c62e20f3667506f4b7e53ca0004b4e3e/src/vscode-dts/vscode.d.ts#L10794 - interface Window { - onDidStartTerminalShellExecution?: ( - listener: (e: any) => any, - thisArgs?: any, - disposables?: vscode.Disposable[], - ) => vscode.Disposable - } -} import { arePathsEqual } from "../../utils/path" import { mergePromise, TerminalProcess, TerminalProcessResultPromise } from "./TerminalProcess" import { TerminalInfo, TerminalRegistry } from "./TerminalRegistry" @@ -89,6 +61,34 @@ Resources: - https://github.com/microsoft/vscode-extension-samples/blob/main/shell-integration-sample/src/extension.ts */ +/* +The new shellIntegration API gives us access to terminal command execution output handling. +However, we don't update our VSCode type definitions or engine requirements to maintain compatibility +with older VSCode versions. Users on older versions will automatically fall back to using sendText +for terminal command execution. +Interestingly, some environments like Cursor enable these APIs even without the latest VSCode engine. +This approach allows us to leverage advanced features when available while ensuring broad compatibility. +*/ +declare module "vscode" { + // https://github.com/microsoft/vscode/blob/f0417069c62e20f3667506f4b7e53ca0004b4e3e/src/vscode-dts/vscode.d.ts#L7442 + interface Terminal { + shellIntegration?: { + cwd?: vscode.Uri + executeCommand?: (command: string) => { + read: () => AsyncIterable + } + } + } + // https://github.com/microsoft/vscode/blob/f0417069c62e20f3667506f4b7e53ca0004b4e3e/src/vscode-dts/vscode.d.ts#L10794 + interface Window { + onDidStartTerminalShellExecution?: ( + listener: (e: any) => any, + thisArgs?: any, + disposables?: vscode.Disposable[], + ) => vscode.Disposable + } +} + export class TerminalManager { private terminalIds: Set = new Set() private processes: Map = new Map() From 8313fa099c19845aa2bcbdbd1ff69719172ce440 Mon Sep 17 00:00:00 2001 From: ocasta181 Date: Tue, 25 Feb 2025 12:14:15 -0800 Subject: [PATCH 03/19] reduce complexity, get working --- src/core/Cline.ts | 42 +++++--------------- src/integrations/misc/extract-text.test.ts | 6 ++- src/integrations/misc/extract-text.ts | 13 ++++-- src/integrations/terminal/TerminalProcess.ts | 23 +++++++---- src/utils/content-size.test.ts | 30 ++++++++++++-- src/utils/content-size.ts | 30 ++++++++++++-- 6 files changed, 91 insertions(+), 53 deletions(-) diff --git a/src/core/Cline.ts b/src/core/Cline.ts index 9fc08e9277..44b7e3fc71 100644 --- a/src/core/Cline.ts +++ b/src/core/Cline.ts @@ -1166,10 +1166,8 @@ export class Cline { // Tools - async executeCommandTool(command: string): Promise<[boolean, ToolResponse]> { - const contextWindow = this.api.getModel().info.contextWindow || 128_000 - const maxAllowedSize = getMaxAllowedSize(contextWindow) - const usedContext = this.apiConversationHistory.reduce((total, msg) => { + private calculateUsedContext(): number { + return this.apiConversationHistory.reduce((total, msg) => { if (Array.isArray(msg.content)) { return ( total + @@ -1183,6 +1181,12 @@ export class Cline { } return total + (typeof msg.content === "string" ? msg.content.length / 4 : 0) }, 0) + } + + async executeCommandTool(command: string): Promise<[boolean, ToolResponse]> { + const contextWindow = this.api.getModel().info.contextWindow || 128_000 + const maxAllowedSize = getMaxAllowedSize(contextWindow) + const usedContext = this.calculateUsedContext() const terminalInfo = await this.terminalManager.getOrCreateTerminal(cwd) terminalInfo.terminal.show() // weird visual bug when creating new terminals (even manually) where there's an empty space at the top. @@ -1359,20 +1363,7 @@ export class Cline { if (this.api instanceof OpenAiHandler && this.api.getModel().id.toLowerCase().includes("deepseek")) { contextWindow = 64_000 } - let maxAllowedSize: number - switch (contextWindow) { - case 64_000: // deepseek models - maxAllowedSize = contextWindow - 27_000 - break - case 128_000: // most models - maxAllowedSize = contextWindow - 30_000 - break - case 200_000: // claude models - maxAllowedSize = contextWindow - 40_000 - break - default: - maxAllowedSize = Math.max(contextWindow - 40_000, contextWindow * 0.8) // for deepseek, 80% of 64k meant only ~10k buffer which was too small and resulted in users getting context window errors. - } + const maxAllowedSize = getMaxAllowedSize(contextWindow) // This is the most reliable way to know when we're close to hitting the context window. if (totalTokens >= maxAllowedSize) { @@ -2017,20 +2008,7 @@ export class Cline { const maxAllowedSize = getMaxAllowedSize(contextWindow) // Calculate used context from current conversation - const usedContext = this.apiConversationHistory.reduce((total, msg) => { - if (Array.isArray(msg.content)) { - return ( - total + - msg.content.reduce((acc, block) => { - if (block.type === "text") { - return acc + block.text.length / 4 // Rough estimate of tokens - } - return acc - }, 0) - ) - } - return total + (typeof msg.content === "string" ? msg.content.length / 4 : 0) - }, 0) + const usedContext = this.calculateUsedContext() // now execute the tool like normal const content = await extractTextFromFile(absolutePath, maxAllowedSize, usedContext) diff --git a/src/integrations/misc/extract-text.test.ts b/src/integrations/misc/extract-text.test.ts index f3798cb6be..9cf94aac97 100644 --- a/src/integrations/misc/extract-text.test.ts +++ b/src/integrations/misc/extract-text.test.ts @@ -4,6 +4,7 @@ import fs from "fs/promises" import path from "path" import os from "os" import { ContentTooLargeError } from "../../shared/errors" +import { calculateMaxAllowedSize } from "../../utils/content-size" const CONTEXT_LIMIT = 1000 const USED_CONTEXT = 200 @@ -29,8 +30,9 @@ describe("extract-text", () => { } }) - it("throws ContentTooLargeError when file would exceed limit", async () => { - const largeContent = "x".repeat(3000) // 3000 bytes = ~750 tokens + it("throws ContentTooLargeError when file would exceed half of context limit", async () => { + const halfContextLimit = calculateMaxAllowedSize(CONTEXT_LIMIT) // 500 tokens + const largeContent = "x".repeat(halfContextLimit * 4 + 4) // Just over half context limit in tokens await fs.writeFile(tempFilePath, largeContent) try { diff --git a/src/integrations/misc/extract-text.ts b/src/integrations/misc/extract-text.ts index 81b66ca813..807e7e7cae 100644 --- a/src/integrations/misc/extract-text.ts +++ b/src/integrations/misc/extract-text.ts @@ -4,7 +4,7 @@ import pdf from "pdf-parse/lib/pdf-parse" import mammoth from "mammoth" import fs from "fs/promises" import { isBinaryFile } from "isbinaryfile" -import { estimateFileSize } from "../../utils/content-size" +import { estimateFileSize, wouldExceedSizeLimit } from "../../utils/content-size" import { ContentTooLargeError } from "../../shared/errors" export async function extractTextFromFile(filePath: string, contextLimit: number, usedContext: number = 0): Promise { @@ -14,9 +14,14 @@ export async function extractTextFromFile(filePath: string, contextLimit: number throw new Error(`File not found: ${filePath}`) } - // Check file size before attempting to read - const sizeEstimate = await estimateFileSize(filePath, contextLimit, usedContext) - if (sizeEstimate.wouldExceedLimit) { + // Get file stats to check size + const stats = await fs.stat(filePath) + + // Check if file size would exceed limit before attempting to read + // This is more efficient than creating a full SizeEstimate object when we just need a boolean check + if (wouldExceedSizeLimit(stats.size, contextLimit)) { + // Only create the full size estimate when we need it for the error + const sizeEstimate = await estimateFileSize(filePath, contextLimit, usedContext) throw new ContentTooLargeError({ type: "file", path: filePath, diff --git a/src/integrations/terminal/TerminalProcess.ts b/src/integrations/terminal/TerminalProcess.ts index fa2997aeb2..b40fab4852 100644 --- a/src/integrations/terminal/TerminalProcess.ts +++ b/src/integrations/terminal/TerminalProcess.ts @@ -1,8 +1,8 @@ import { EventEmitter } from "events" -import * as stripAnsi from "strip-ansi" +import stripAnsi from "strip-ansi" import * as vscode from "vscode" import { ContentTooLargeError } from "../../shared/errors" -import { estimateContentSize } from "../../utils/content-size" +import { estimateContentSize, wouldExceedSizeLimit } from "../../utils/content-size" export interface TerminalProcessEvents { line: [line: string] @@ -52,9 +52,15 @@ export class TerminalProcess extends EventEmitter { const dataBytes = Buffer.from(data).length this.totalBytes += dataBytes - // Check total accumulated size - const sizeEstimate = estimateContentSize(Buffer.alloc(this.totalBytes), this.contextLimit, this.usedContext) - if (sizeEstimate.wouldExceedLimit) { + // Check total accumulated size against half of context limit + // Use wouldExceedSizeLimit to avoid creating unnecessary buffer + if (wouldExceedSizeLimit(this.totalBytes, this.contextLimit)) { + // Create size estimate only when needed for error details + const sizeEstimate = estimateContentSize( + Buffer.alloc(0, this.totalBytes), + this.contextLimit, + this.usedContext, + ) this.emit( "error", new ContentTooLargeError({ @@ -215,10 +221,11 @@ export class TerminalProcess extends EventEmitter { // Inspired by https://github.com/sindresorhus/execa/blob/main/lib/transform/split.js private emitIfEol(chunk: string) { - // Check size before adding to buffer + // Check size before adding to buffer against half of context limit const newBufferSize = this.buffer.length + chunk.length - const sizeEstimate = estimateContentSize(Buffer.alloc(newBufferSize), this.contextLimit, this.usedContext) - if (sizeEstimate.wouldExceedLimit) { + if (wouldExceedSizeLimit(newBufferSize, this.contextLimit)) { + // Create size estimate only when needed for error details + const sizeEstimate = estimateContentSize(Buffer.alloc(0, newBufferSize), this.contextLimit, this.usedContext) this.emit( "error", new ContentTooLargeError({ diff --git a/src/utils/content-size.test.ts b/src/utils/content-size.test.ts index d59510a654..5795f0c53c 100644 --- a/src/utils/content-size.test.ts +++ b/src/utils/content-size.test.ts @@ -1,5 +1,11 @@ import { expect } from "chai" -import { estimateContentSize, estimateFileSize, estimateTokens } from "./content-size" +import { + estimateContentSize, + estimateFileSize, + estimateTokens, + calculateMaxAllowedSize, + wouldExceedSizeLimit, +} from "./content-size" import fs from "fs/promises" import path from "path" import os from "os" @@ -8,6 +14,13 @@ const CONTEXT_LIMIT = 1000 const USED_CONTEXT = 200 describe("content-size", () => { + describe("calculateMaxAllowedSize", () => { + it("calculates half of the context limit", () => { + expect(calculateMaxAllowedSize(1000)).to.equal(500) + expect(calculateMaxAllowedSize(128000)).to.equal(64000) + }) + }) + describe("estimateTokens", () => { it("estimates tokens based on byte count", () => { expect(estimateTokens(100)).to.equal(25) // 100 bytes / 4 chars per token = 25 tokens @@ -15,6 +28,14 @@ describe("content-size", () => { }) }) + describe("wouldExceedSizeLimit", () => { + it("checks if byte count would exceed half of context limit", () => { + expect(wouldExceedSizeLimit(100, 1000)).to.equal(false) // 25 tokens < 500 tokens + expect(wouldExceedSizeLimit(2000, 1000)).to.equal(true) // 500 tokens = 500 tokens (equal is considered exceeding) + expect(wouldExceedSizeLimit(2004, 1000)).to.equal(true) // 501 tokens > 500 tokens + }) + }) + describe("estimateContentSize", () => { it("estimates size for string content", () => { const content = "Hello world" // 11 bytes @@ -36,12 +57,13 @@ describe("content-size", () => { expect(result.wouldExceedLimit).to.equal(false) }) - it("detects when content would exceed limit", () => { - const largeContent = "x".repeat(3000) // 3000 bytes = ~750 tokens + it("detects when content would exceed half of context limit", () => { + const halfContextLimit = calculateMaxAllowedSize(CONTEXT_LIMIT) // 500 tokens + const largeContent = "x".repeat(halfContextLimit * 4 + 4) // Just over half context limit in tokens const result = estimateContentSize(largeContent, CONTEXT_LIMIT, USED_CONTEXT) expect(result.wouldExceedLimit).to.equal(true) - expect(result.remainingContextSize).to.equal(800) + expect(result.remainingContextSize).to.equal(800) // This is still contextLimit - usedContext }) }) diff --git a/src/utils/content-size.ts b/src/utils/content-size.ts index f420045d07..f07925a2be 100644 --- a/src/utils/content-size.ts +++ b/src/utils/content-size.ts @@ -1,4 +1,3 @@ -import fs from "fs/promises" import { stat } from "fs/promises" // Rough approximation: 1 token ≈ 4 characters for English text @@ -11,6 +10,14 @@ export interface SizeEstimate { remainingContextSize: number } +/** + * Calculates the maximum allowed size for a single content item (file or terminal output) + * We limit to half the context window to ensure no single item can consume too much context + */ +export function calculateMaxAllowedSize(contextLimit: number): number { + return Math.floor(contextLimit / 2) +} + /** * Estimates tokens from byte count using a simple character ratio * This is a rough approximation - actual token count may vary @@ -19,6 +26,16 @@ export function estimateTokens(bytes: number): number { return Math.ceil(bytes / CHARS_PER_TOKEN) } +/** + * Checks if the given byte count would exceed the size limit + * More efficient than creating a buffer just to check size + */ +export function wouldExceedSizeLimit(byteCount: number, contextLimit: number): boolean { + const estimatedTokenCount = estimateTokens(byteCount) + const maxAllowedSize = calculateMaxAllowedSize(contextLimit) + return estimatedTokenCount >= maxAllowedSize +} + /** * Estimates size metrics for a string or buffer without loading entire content */ @@ -26,11 +43,12 @@ export function estimateContentSize(content: string | Buffer, contextLimit: numb const bytes = Buffer.isBuffer(content) ? content.length : Buffer.from(content).length const estimatedTokenCount = estimateTokens(bytes) const remainingContext = contextLimit - usedContext + const maxAllowedSize = calculateMaxAllowedSize(contextLimit) return { bytes, estimatedTokens: estimatedTokenCount, - wouldExceedLimit: estimatedTokenCount > remainingContext, + wouldExceedLimit: estimatedTokenCount >= maxAllowedSize, remainingContextSize: remainingContext, } } @@ -43,15 +61,21 @@ export async function estimateFileSize(filePath: string, contextLimit: number, u const bytes = stats.size const estimatedTokenCount = estimateTokens(bytes) const remainingContext = contextLimit - usedContext + const maxAllowedSize = calculateMaxAllowedSize(contextLimit) return { bytes, estimatedTokens: estimatedTokenCount, - wouldExceedLimit: estimatedTokenCount > remainingContext, + wouldExceedLimit: estimatedTokenCount >= maxAllowedSize, remainingContextSize: remainingContext, } } +/** + * Gets the maximum allowed size for the API context window + * This is different from calculateMaxAllowedSize as it's for the entire context window + * rather than a single content item + */ export function getMaxAllowedSize(contextWindow: number): number { // Get context window and used context from API model let maxAllowedSize: number From af0d2f31a7a6a289a30501ee51cfc964f7c99531 Mon Sep 17 00:00:00 2001 From: ocasta181 Date: Tue, 25 Feb 2025 16:03:40 -0800 Subject: [PATCH 04/19] add changeset --- .changeset/seven-flowers-lay.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/seven-flowers-lay.md diff --git a/.changeset/seven-flowers-lay.md b/.changeset/seven-flowers-lay.md new file mode 100644 index 0000000000..bd1f90704b --- /dev/null +++ b/.changeset/seven-flowers-lay.md @@ -0,0 +1,5 @@ +--- +"claude-dev": patch +--- + +Fix a bug where cline crashes when reading large data from files or command outputs From 22b2e50210e67c79dc837478e6ec6c06dbc3d309 Mon Sep 17 00:00:00 2001 From: ocasta181 Date: Fri, 28 Feb 2025 18:50:28 -0800 Subject: [PATCH 05/19] use real context window size --- src/core/mentions/index.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/core/mentions/index.ts b/src/core/mentions/index.ts index 4c67d8fb4d..2f5e2a75f6 100644 --- a/src/core/mentions/index.ts +++ b/src/core/mentions/index.ts @@ -156,7 +156,8 @@ async function getFileOrFolderContent(mentionPath: string, cwd: string): Promise if (isBinary) { return "(Binary file, unable to display content)" } - const content = await extractTextFromFile(absPath, 128_000) // Use standard context window size + const contextWindow = this.api.getModel().info.contextWindow || 128_000 + const content = await extractTextFromFile(absPath, contextWindow) return content } else if (stats.isDirectory()) { const entries = await fs.readdir(absPath, { withFileTypes: true }) @@ -177,7 +178,8 @@ async function getFileOrFolderContent(mentionPath: string, cwd: string): Promise if (isBinary) { return undefined } - const content = await extractTextFromFile(absoluteFilePath, 128_000) // Use standard context window size + const contextWindow = this.api.getModel().info.contextWindow || 128_000 + const content = await extractTextFromFile(absoluteFilePath, contextWindow) return `\n${content}\n` } catch (error) { return undefined From ad8e0b19be61179e4e5fd36c46808a17cdeadaa7 Mon Sep 17 00:00:00 2001 From: ocasta181 Date: Fri, 28 Feb 2025 21:21:53 -0800 Subject: [PATCH 06/19] remove usedContext references --- src/core/Cline.ts | 7 ++----- src/integrations/misc/extract-text.test.ts | 9 ++++----- src/integrations/misc/extract-text.ts | 4 ++-- src/integrations/terminal/TerminalManager.ts | 11 +++-------- src/integrations/terminal/TerminalProcess.ts | 14 +++----------- src/utils/content-size.test.ts | 15 +++++---------- src/utils/content-size.ts | 9 ++------- 7 files changed, 21 insertions(+), 48 deletions(-) diff --git a/src/core/Cline.ts b/src/core/Cline.ts index 44b7e3fc71..496a80526a 100644 --- a/src/core/Cline.ts +++ b/src/core/Cline.ts @@ -1190,7 +1190,7 @@ export class Cline { const terminalInfo = await this.terminalManager.getOrCreateTerminal(cwd) terminalInfo.terminal.show() // weird visual bug when creating new terminals (even manually) where there's an empty space at the top. - const process = this.terminalManager.runCommand(terminalInfo, command, maxAllowedSize, usedContext) + const process = this.terminalManager.runCommand(terminalInfo, command, maxAllowedSize) let userFeedback: { text?: string; images?: string[] } | undefined let didContinue = false @@ -2007,11 +2007,8 @@ export class Cline { const contextWindow = this.api.getModel().info.contextWindow || 128_000 const maxAllowedSize = getMaxAllowedSize(contextWindow) - // Calculate used context from current conversation - const usedContext = this.calculateUsedContext() - // now execute the tool like normal - const content = await extractTextFromFile(absolutePath, maxAllowedSize, usedContext) + const content = await extractTextFromFile(absolutePath, maxAllowedSize) pushToolResult(content) break diff --git a/src/integrations/misc/extract-text.test.ts b/src/integrations/misc/extract-text.test.ts index 9cf94aac97..0e4622d7db 100644 --- a/src/integrations/misc/extract-text.test.ts +++ b/src/integrations/misc/extract-text.test.ts @@ -7,7 +7,6 @@ import { ContentTooLargeError } from "../../shared/errors" import { calculateMaxAllowedSize } from "../../utils/content-size" const CONTEXT_LIMIT = 1000 -const USED_CONTEXT = 200 describe("extract-text", () => { let tempFilePath: string @@ -23,7 +22,7 @@ describe("extract-text", () => { it("throws error for non-existent file", async () => { const nonExistentPath = path.join(os.tmpdir(), "non-existent.txt") try { - await extractTextFromFile(nonExistentPath, CONTEXT_LIMIT, USED_CONTEXT) + await extractTextFromFile(nonExistentPath, CONTEXT_LIMIT) throw new Error("Should have thrown error") } catch (error) { expect(error.message).to.include("File not found") @@ -36,7 +35,7 @@ describe("extract-text", () => { await fs.writeFile(tempFilePath, largeContent) try { - await extractTextFromFile(tempFilePath, CONTEXT_LIMIT, USED_CONTEXT) + await extractTextFromFile(tempFilePath, CONTEXT_LIMIT) throw new Error("Should have thrown error") } catch (error) { expect(error).to.be.instanceOf(ContentTooLargeError) @@ -50,7 +49,7 @@ describe("extract-text", () => { const content = "Hello world" await fs.writeFile(tempFilePath, content) - const result = await extractTextFromFile(tempFilePath, CONTEXT_LIMIT, USED_CONTEXT) + const result = await extractTextFromFile(tempFilePath, CONTEXT_LIMIT) expect(result).to.equal(content) }) @@ -60,7 +59,7 @@ describe("extract-text", () => { await fs.writeFile(tempFilePath, buffer, { encoding: "binary" }) try { - await extractTextFromFile(tempFilePath, CONTEXT_LIMIT, USED_CONTEXT) + await extractTextFromFile(tempFilePath, CONTEXT_LIMIT) throw new Error("Should have thrown error") } catch (error) { expect(error.message).to.include("Cannot read text for file type") diff --git a/src/integrations/misc/extract-text.ts b/src/integrations/misc/extract-text.ts index 807e7e7cae..f445567f00 100644 --- a/src/integrations/misc/extract-text.ts +++ b/src/integrations/misc/extract-text.ts @@ -7,7 +7,7 @@ import { isBinaryFile } from "isbinaryfile" import { estimateFileSize, wouldExceedSizeLimit } from "../../utils/content-size" import { ContentTooLargeError } from "../../shared/errors" -export async function extractTextFromFile(filePath: string, contextLimit: number, usedContext: number = 0): Promise { +export async function extractTextFromFile(filePath: string, contextLimit: number): Promise { try { await fs.access(filePath) } catch (error) { @@ -21,7 +21,7 @@ export async function extractTextFromFile(filePath: string, contextLimit: number // This is more efficient than creating a full SizeEstimate object when we just need a boolean check if (wouldExceedSizeLimit(stats.size, contextLimit)) { // Only create the full size estimate when we need it for the error - const sizeEstimate = await estimateFileSize(filePath, contextLimit, usedContext) + const sizeEstimate = await estimateFileSize(filePath, contextLimit) throw new ContentTooLargeError({ type: "file", path: filePath, diff --git a/src/integrations/terminal/TerminalManager.ts b/src/integrations/terminal/TerminalManager.ts index 6d8c2a36c9..f7542a0b3e 100644 --- a/src/integrations/terminal/TerminalManager.ts +++ b/src/integrations/terminal/TerminalManager.ts @@ -109,12 +109,7 @@ export class TerminalManager { } } - runCommand( - terminalInfo: TerminalInfo, - command: string, - contextLimit?: number, - usedContext?: number, - ): TerminalProcessResultPromise { + runCommand(terminalInfo: TerminalInfo, command: string, contextLimit?: number): TerminalProcessResultPromise { terminalInfo.busy = true terminalInfo.lastCommand = command const process = new TerminalProcess() @@ -146,14 +141,14 @@ export class TerminalManager { // if shell integration is already active, run the command immediately if (terminalInfo.terminal.shellIntegration) { process.waitForShellIntegration = false - process.run(terminalInfo.terminal, command, contextLimit, usedContext) + process.run(terminalInfo.terminal, command, contextLimit) } else { // docs recommend waiting 3s for shell integration to activate pWaitFor(() => terminalInfo.terminal.shellIntegration !== undefined, { timeout: 4000 }).finally(() => { const existingProcess = this.processes.get(terminalInfo.id) if (existingProcess && existingProcess.waitForShellIntegration) { existingProcess.waitForShellIntegration = false - existingProcess.run(terminalInfo.terminal, command, contextLimit, usedContext) + existingProcess.run(terminalInfo.terminal, command, contextLimit) } }) } diff --git a/src/integrations/terminal/TerminalProcess.ts b/src/integrations/terminal/TerminalProcess.ts index b40fab4852..671752704c 100644 --- a/src/integrations/terminal/TerminalProcess.ts +++ b/src/integrations/terminal/TerminalProcess.ts @@ -26,19 +26,15 @@ export class TerminalProcess extends EventEmitter { private hotTimer: NodeJS.Timeout | null = null private totalBytes: number = 0 private contextLimit: number = 100000 // Default context window size - private usedContext: number = 0 private lastCommand: string = "" // constructor() { // super() - async run(terminal: vscode.Terminal, command: string, contextLimit?: number, usedContext?: number) { + async run(terminal: vscode.Terminal, command: string, contextLimit?: number) { if (contextLimit) { this.contextLimit = contextLimit } - if (usedContext) { - this.usedContext = usedContext - } this.lastCommand = command if (terminal.shellIntegration && terminal.shellIntegration.executeCommand) { const execution = terminal.shellIntegration.executeCommand(command) @@ -56,11 +52,7 @@ export class TerminalProcess extends EventEmitter { // Use wouldExceedSizeLimit to avoid creating unnecessary buffer if (wouldExceedSizeLimit(this.totalBytes, this.contextLimit)) { // Create size estimate only when needed for error details - const sizeEstimate = estimateContentSize( - Buffer.alloc(0, this.totalBytes), - this.contextLimit, - this.usedContext, - ) + const sizeEstimate = estimateContentSize(Buffer.alloc(0, this.totalBytes), this.contextLimit) this.emit( "error", new ContentTooLargeError({ @@ -225,7 +217,7 @@ export class TerminalProcess extends EventEmitter { const newBufferSize = this.buffer.length + chunk.length if (wouldExceedSizeLimit(newBufferSize, this.contextLimit)) { // Create size estimate only when needed for error details - const sizeEstimate = estimateContentSize(Buffer.alloc(0, newBufferSize), this.contextLimit, this.usedContext) + const sizeEstimate = estimateContentSize(Buffer.alloc(0, newBufferSize), this.contextLimit) this.emit( "error", new ContentTooLargeError({ diff --git a/src/utils/content-size.test.ts b/src/utils/content-size.test.ts index 5795f0c53c..ae20072f07 100644 --- a/src/utils/content-size.test.ts +++ b/src/utils/content-size.test.ts @@ -11,7 +11,6 @@ import path from "path" import os from "os" const CONTEXT_LIMIT = 1000 -const USED_CONTEXT = 200 describe("content-size", () => { describe("calculateMaxAllowedSize", () => { @@ -39,31 +38,28 @@ describe("content-size", () => { describe("estimateContentSize", () => { it("estimates size for string content", () => { const content = "Hello world" // 11 bytes - const result = estimateContentSize(content, CONTEXT_LIMIT, USED_CONTEXT) + const result = estimateContentSize(content, CONTEXT_LIMIT) expect(result.bytes).to.equal(11) expect(result.estimatedTokens).to.equal(3) - expect(result.remainingContextSize).to.equal(800) expect(result.wouldExceedLimit).to.equal(false) }) it("estimates size for buffer content", () => { const content = Buffer.from("Hello world") // 11 bytes - const result = estimateContentSize(content, CONTEXT_LIMIT, USED_CONTEXT) + const result = estimateContentSize(content, CONTEXT_LIMIT) expect(result.bytes).to.equal(11) expect(result.estimatedTokens).to.equal(3) - expect(result.remainingContextSize).to.equal(800) expect(result.wouldExceedLimit).to.equal(false) }) it("detects when content would exceed half of context limit", () => { const halfContextLimit = calculateMaxAllowedSize(CONTEXT_LIMIT) // 500 tokens const largeContent = "x".repeat(halfContextLimit * 4 + 4) // Just over half context limit in tokens - const result = estimateContentSize(largeContent, CONTEXT_LIMIT, USED_CONTEXT) + const result = estimateContentSize(largeContent, CONTEXT_LIMIT) expect(result.wouldExceedLimit).to.equal(true) - expect(result.remainingContextSize).to.equal(800) // This is still contextLimit - usedContext }) }) @@ -80,18 +76,17 @@ describe("content-size", () => { }) it("estimates size for existing file", async () => { - const result = await estimateFileSize(tempFilePath, CONTEXT_LIMIT, USED_CONTEXT) + const result = await estimateFileSize(tempFilePath, CONTEXT_LIMIT) expect(result.bytes).to.equal(11) expect(result.estimatedTokens).to.equal(3) - expect(result.remainingContextSize).to.equal(800) expect(result.wouldExceedLimit).to.equal(false) }) it("throws error for non-existent file", async () => { const nonExistentPath = path.join(os.tmpdir(), "non-existent.txt") try { - await estimateFileSize(nonExistentPath, CONTEXT_LIMIT, USED_CONTEXT) + await estimateFileSize(nonExistentPath, CONTEXT_LIMIT) throw new Error("Should have thrown error") } catch (error) { expect(error).to.be.instanceOf(Error) diff --git a/src/utils/content-size.ts b/src/utils/content-size.ts index f07925a2be..f0343180ee 100644 --- a/src/utils/content-size.ts +++ b/src/utils/content-size.ts @@ -7,7 +7,6 @@ export interface SizeEstimate { bytes: number estimatedTokens: number wouldExceedLimit: boolean - remainingContextSize: number } /** @@ -39,35 +38,31 @@ export function wouldExceedSizeLimit(byteCount: number, contextLimit: number): b /** * Estimates size metrics for a string or buffer without loading entire content */ -export function estimateContentSize(content: string | Buffer, contextLimit: number, usedContext: number = 0): SizeEstimate { +export function estimateContentSize(content: string | Buffer, contextLimit: number): SizeEstimate { const bytes = Buffer.isBuffer(content) ? content.length : Buffer.from(content).length const estimatedTokenCount = estimateTokens(bytes) - const remainingContext = contextLimit - usedContext const maxAllowedSize = calculateMaxAllowedSize(contextLimit) return { bytes, estimatedTokens: estimatedTokenCount, wouldExceedLimit: estimatedTokenCount >= maxAllowedSize, - remainingContextSize: remainingContext, } } /** * Gets size metrics for a file without reading its contents */ -export async function estimateFileSize(filePath: string, contextLimit: number, usedContext: number = 0): Promise { +export async function estimateFileSize(filePath: string, contextLimit: number): Promise { const stats = await stat(filePath) const bytes = stats.size const estimatedTokenCount = estimateTokens(bytes) - const remainingContext = contextLimit - usedContext const maxAllowedSize = calculateMaxAllowedSize(contextLimit) return { bytes, estimatedTokens: estimatedTokenCount, wouldExceedLimit: estimatedTokenCount >= maxAllowedSize, - remainingContextSize: remainingContext, } } From a6f189c5c56c72d209a69fac7d809ae0f6afcc96 Mon Sep 17 00:00:00 2001 From: ocasta181 Date: Fri, 28 Feb 2025 21:26:06 -0800 Subject: [PATCH 07/19] remove calculateUsedContext --- src/core/Cline.ts | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/src/core/Cline.ts b/src/core/Cline.ts index 496a80526a..f0090b16e8 100644 --- a/src/core/Cline.ts +++ b/src/core/Cline.ts @@ -1166,27 +1166,9 @@ export class Cline { // Tools - private calculateUsedContext(): number { - return this.apiConversationHistory.reduce((total, msg) => { - if (Array.isArray(msg.content)) { - return ( - total + - msg.content.reduce((acc, block) => { - if (block.type === "text") { - return acc + block.text.length / 4 // Rough estimate of tokens - } - return acc - }, 0) - ) - } - return total + (typeof msg.content === "string" ? msg.content.length / 4 : 0) - }, 0) - } - async executeCommandTool(command: string): Promise<[boolean, ToolResponse]> { const contextWindow = this.api.getModel().info.contextWindow || 128_000 const maxAllowedSize = getMaxAllowedSize(contextWindow) - const usedContext = this.calculateUsedContext() const terminalInfo = await this.terminalManager.getOrCreateTerminal(cwd) terminalInfo.terminal.show() // weird visual bug when creating new terminals (even manually) where there's an empty space at the top. From 4846476f6c45db5a88c1bb54506b73a486200913 Mon Sep 17 00:00:00 2001 From: ocasta181 Date: Fri, 28 Feb 2025 21:47:12 -0800 Subject: [PATCH 08/19] remove more unused logic --- src/core/Cline.ts | 8 ++--- src/core/mentions/index.ts | 19 ++++++++--- src/integrations/misc/extract-text.test.ts | 11 +++---- src/integrations/terminal/TerminalProcess.ts | 2 +- src/utils/content-size.test.ts | 33 ++++++-------------- src/utils/content-size.ts | 18 ++--------- 6 files changed, 37 insertions(+), 54 deletions(-) diff --git a/src/core/Cline.ts b/src/core/Cline.ts index f0090b16e8..da05310210 100644 --- a/src/core/Cline.ts +++ b/src/core/Cline.ts @@ -1167,7 +1167,7 @@ export class Cline { // Tools async executeCommandTool(command: string): Promise<[boolean, ToolResponse]> { - const contextWindow = this.api.getModel().info.contextWindow || 128_000 + const contextWindow = this.api.getModel().info.contextWindow || 64_000 // minimum context (Deepseek) const maxAllowedSize = getMaxAllowedSize(contextWindow) const terminalInfo = await this.terminalManager.getOrCreateTerminal(cwd) @@ -1340,7 +1340,7 @@ export class Cline { if (previousRequest && previousRequest.text) { const { tokensIn, tokensOut, cacheWrites, cacheReads }: ClineApiReqInfo = JSON.parse(previousRequest.text) const totalTokens = (tokensIn || 0) + (tokensOut || 0) + (cacheWrites || 0) + (cacheReads || 0) - let contextWindow = this.api.getModel().info.contextWindow || 128_000 + let contextWindow = this.api.getModel().info.contextWindow || 64_000 // minimum context (Deepseek) // FIXME: hack to get anyone using openai compatible with deepseek to have the proper context window instead of the default 128k. We need a way for the user to specify the context window for models they input through openai compatible if (this.api instanceof OpenAiHandler && this.api.getModel().id.toLowerCase().includes("deepseek")) { contextWindow = 64_000 @@ -1986,7 +1986,7 @@ export class Cline { } } // Get context window and used context from API model - const contextWindow = this.api.getModel().info.contextWindow || 128_000 + const contextWindow = this.api.getModel().info.contextWindow || 64_000 // minimum context (Deepseek) const maxAllowedSize = getMaxAllowedSize(contextWindow) // now execute the tool like normal @@ -3357,7 +3357,7 @@ export class Cline { ) { return { ...block, - text: await parseMentions(block.text, cwd, this.urlContentFetcher), + text: await parseMentions(block.text, cwd, this.urlContentFetcher, this.api), } } } diff --git a/src/core/mentions/index.ts b/src/core/mentions/index.ts index 2f5e2a75f6..6bff6b6df5 100644 --- a/src/core/mentions/index.ts +++ b/src/core/mentions/index.ts @@ -38,7 +38,12 @@ export function openMention(mention?: string): void { } } -export async function parseMentions(text: string, cwd: string, urlContentFetcher: UrlContentFetcher): Promise { +export async function parseMentions( + text: string, + cwd: string, + urlContentFetcher: UrlContentFetcher, + api: { getModel: () => { info: { contextWindow?: number } } }, +): Promise { const mentions: Set = new Set() let parsedText = text.replace(mentionRegexGlobal, (match, mention) => { mentions.add(mention) @@ -90,7 +95,7 @@ export async function parseMentions(text: string, cwd: string, urlContentFetcher } else if (mention.startsWith("/")) { const mentionPath = mention.slice(1) try { - const content = await getFileOrFolderContent(mentionPath, cwd) + const content = await getFileOrFolderContent(mentionPath, cwd, api) if (mention.endsWith("/")) { parsedText += `\n\n\n${content}\n` } else { @@ -145,7 +150,11 @@ export async function parseMentions(text: string, cwd: string, urlContentFetcher return parsedText } -async function getFileOrFolderContent(mentionPath: string, cwd: string): Promise { +async function getFileOrFolderContent( + mentionPath: string, + cwd: string, + api: { getModel: () => { info: { contextWindow?: number } } }, +): Promise { const absPath = path.resolve(cwd, mentionPath) try { @@ -156,7 +165,7 @@ async function getFileOrFolderContent(mentionPath: string, cwd: string): Promise if (isBinary) { return "(Binary file, unable to display content)" } - const contextWindow = this.api.getModel().info.contextWindow || 128_000 + const contextWindow = api.getModel().info.contextWindow || 64_000 // minimum context (Deepseek) const content = await extractTextFromFile(absPath, contextWindow) return content } else if (stats.isDirectory()) { @@ -178,7 +187,7 @@ async function getFileOrFolderContent(mentionPath: string, cwd: string): Promise if (isBinary) { return undefined } - const contextWindow = this.api.getModel().info.contextWindow || 128_000 + const contextWindow = api.getModel().info.contextWindow || 64_000 // minimum context (Deepseek) const content = await extractTextFromFile(absoluteFilePath, contextWindow) return `\n${content}\n` } catch (error) { diff --git a/src/integrations/misc/extract-text.test.ts b/src/integrations/misc/extract-text.test.ts index 0e4622d7db..662d085f3a 100644 --- a/src/integrations/misc/extract-text.test.ts +++ b/src/integrations/misc/extract-text.test.ts @@ -4,9 +4,8 @@ import fs from "fs/promises" import path from "path" import os from "os" import { ContentTooLargeError } from "../../shared/errors" -import { calculateMaxAllowedSize } from "../../utils/content-size" -const CONTEXT_LIMIT = 1000 +const CONTEXT_LIMIT = 1000 // Context limit of 1000 tokens means max allowed size is 500 tokens describe("extract-text", () => { let tempFilePath: string @@ -29,13 +28,13 @@ describe("extract-text", () => { } }) - it("throws ContentTooLargeError when file would exceed half of context limit", async () => { - const halfContextLimit = calculateMaxAllowedSize(CONTEXT_LIMIT) // 500 tokens - const largeContent = "x".repeat(halfContextLimit * 4 + 4) // Just over half context limit in tokens + it("throws ContentTooLargeError when file would exceed max allowed size", async () => { + // Create content that would exceed max allowed size for deepseek (64k - 27k tokens) + const largeContent = "x".repeat(148000) // 37k tokens > (64k - 27k) tokens await fs.writeFile(tempFilePath, largeContent) try { - await extractTextFromFile(tempFilePath, CONTEXT_LIMIT) + await extractTextFromFile(tempFilePath, 64_000) throw new Error("Should have thrown error") } catch (error) { expect(error).to.be.instanceOf(ContentTooLargeError) diff --git a/src/integrations/terminal/TerminalProcess.ts b/src/integrations/terminal/TerminalProcess.ts index 671752704c..c1a8368c3d 100644 --- a/src/integrations/terminal/TerminalProcess.ts +++ b/src/integrations/terminal/TerminalProcess.ts @@ -25,7 +25,7 @@ export class TerminalProcess extends EventEmitter { isHot: boolean = false private hotTimer: NodeJS.Timeout | null = null private totalBytes: number = 0 - private contextLimit: number = 100000 // Default context window size + private contextLimit: number = 64_000 // minimum context (Deepseek) private lastCommand: string = "" // constructor() { diff --git a/src/utils/content-size.test.ts b/src/utils/content-size.test.ts index ae20072f07..a8fefa9d4d 100644 --- a/src/utils/content-size.test.ts +++ b/src/utils/content-size.test.ts @@ -1,25 +1,12 @@ import { expect } from "chai" -import { - estimateContentSize, - estimateFileSize, - estimateTokens, - calculateMaxAllowedSize, - wouldExceedSizeLimit, -} from "./content-size" +import { estimateContentSize, estimateFileSize, estimateTokens, wouldExceedSizeLimit } from "./content-size" import fs from "fs/promises" import path from "path" import os from "os" -const CONTEXT_LIMIT = 1000 +const CONTEXT_LIMIT = 1000 // Context limit of 1000 tokens means max allowed size is 500 tokens describe("content-size", () => { - describe("calculateMaxAllowedSize", () => { - it("calculates half of the context limit", () => { - expect(calculateMaxAllowedSize(1000)).to.equal(500) - expect(calculateMaxAllowedSize(128000)).to.equal(64000) - }) - }) - describe("estimateTokens", () => { it("estimates tokens based on byte count", () => { expect(estimateTokens(100)).to.equal(25) // 100 bytes / 4 chars per token = 25 tokens @@ -28,10 +15,10 @@ describe("content-size", () => { }) describe("wouldExceedSizeLimit", () => { - it("checks if byte count would exceed half of context limit", () => { - expect(wouldExceedSizeLimit(100, 1000)).to.equal(false) // 25 tokens < 500 tokens - expect(wouldExceedSizeLimit(2000, 1000)).to.equal(true) // 500 tokens = 500 tokens (equal is considered exceeding) - expect(wouldExceedSizeLimit(2004, 1000)).to.equal(true) // 501 tokens > 500 tokens + it("checks if byte count would exceed max allowed size", () => { + expect(wouldExceedSizeLimit(100, 64_000)).to.equal(false) // 25 tokens < (64k - 27k) tokens + expect(wouldExceedSizeLimit(148000, 64_000)).to.equal(true) // 37k tokens > (64k - 27k) tokens + expect(wouldExceedSizeLimit(392000, 128_000)).to.equal(true) // 98k tokens > (128k - 30k) tokens }) }) @@ -54,10 +41,10 @@ describe("content-size", () => { expect(result.wouldExceedLimit).to.equal(false) }) - it("detects when content would exceed half of context limit", () => { - const halfContextLimit = calculateMaxAllowedSize(CONTEXT_LIMIT) // 500 tokens - const largeContent = "x".repeat(halfContextLimit * 4 + 4) // Just over half context limit in tokens - const result = estimateContentSize(largeContent, CONTEXT_LIMIT) + it("detects when content would exceed max allowed size", () => { + // Create content that would exceed max allowed size for deepseek (64k - 27k tokens) + const largeContent = "x".repeat(148000) // 37k tokens > (64k - 27k) tokens + const result = estimateContentSize(largeContent, 64_000) expect(result.wouldExceedLimit).to.equal(true) }) diff --git a/src/utils/content-size.ts b/src/utils/content-size.ts index f0343180ee..641ac0dee8 100644 --- a/src/utils/content-size.ts +++ b/src/utils/content-size.ts @@ -9,14 +9,6 @@ export interface SizeEstimate { wouldExceedLimit: boolean } -/** - * Calculates the maximum allowed size for a single content item (file or terminal output) - * We limit to half the context window to ensure no single item can consume too much context - */ -export function calculateMaxAllowedSize(contextLimit: number): number { - return Math.floor(contextLimit / 2) -} - /** * Estimates tokens from byte count using a simple character ratio * This is a rough approximation - actual token count may vary @@ -31,7 +23,7 @@ export function estimateTokens(bytes: number): number { */ export function wouldExceedSizeLimit(byteCount: number, contextLimit: number): boolean { const estimatedTokenCount = estimateTokens(byteCount) - const maxAllowedSize = calculateMaxAllowedSize(contextLimit) + const maxAllowedSize = getMaxAllowedSize(contextLimit) return estimatedTokenCount >= maxAllowedSize } @@ -41,12 +33,11 @@ export function wouldExceedSizeLimit(byteCount: number, contextLimit: number): b export function estimateContentSize(content: string | Buffer, contextLimit: number): SizeEstimate { const bytes = Buffer.isBuffer(content) ? content.length : Buffer.from(content).length const estimatedTokenCount = estimateTokens(bytes) - const maxAllowedSize = calculateMaxAllowedSize(contextLimit) return { bytes, estimatedTokens: estimatedTokenCount, - wouldExceedLimit: estimatedTokenCount >= maxAllowedSize, + wouldExceedLimit: estimatedTokenCount >= getMaxAllowedSize(contextLimit), } } @@ -57,19 +48,16 @@ export async function estimateFileSize(filePath: string, contextLimit: number): const stats = await stat(filePath) const bytes = stats.size const estimatedTokenCount = estimateTokens(bytes) - const maxAllowedSize = calculateMaxAllowedSize(contextLimit) return { bytes, estimatedTokens: estimatedTokenCount, - wouldExceedLimit: estimatedTokenCount >= maxAllowedSize, + wouldExceedLimit: estimatedTokenCount >= getMaxAllowedSize(contextLimit), } } /** * Gets the maximum allowed size for the API context window - * This is different from calculateMaxAllowedSize as it's for the entire context window - * rather than a single content item */ export function getMaxAllowedSize(contextWindow: number): number { // Get context window and used context from API model From 0cdbdb057ed2299d6884aed075a3f0303a36fed1 Mon Sep 17 00:00:00 2001 From: akfoster Date: Fri, 28 Feb 2025 21:49:29 -0800 Subject: [PATCH 09/19] Update src/integrations/terminal/TerminalProcess.ts Co-authored-by: ellipsis-dev[bot] <65095814+ellipsis-dev[bot]@users.noreply.github.com> --- src/integrations/terminal/TerminalProcess.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/integrations/terminal/TerminalProcess.ts b/src/integrations/terminal/TerminalProcess.ts index c1a8368c3d..24ee0bc48b 100644 --- a/src/integrations/terminal/TerminalProcess.ts +++ b/src/integrations/terminal/TerminalProcess.ts @@ -52,7 +52,7 @@ export class TerminalProcess extends EventEmitter { // Use wouldExceedSizeLimit to avoid creating unnecessary buffer if (wouldExceedSizeLimit(this.totalBytes, this.contextLimit)) { // Create size estimate only when needed for error details - const sizeEstimate = estimateContentSize(Buffer.alloc(0, this.totalBytes), this.contextLimit) + const sizeEstimate = estimateContentSize(Buffer.alloc(this.totalBytes), this.contextLimit) this.emit( "error", new ContentTooLargeError({ From 9f5df9836c3df1390d3ecd2225acda5ee138a04c Mon Sep 17 00:00:00 2001 From: ocasta181 Date: Fri, 28 Feb 2025 21:58:08 -0800 Subject: [PATCH 10/19] remove exclusion from tsconfig --- tsconfig.test.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tsconfig.test.json b/tsconfig.test.json index a60e9ddfc6..92f67542f5 100644 --- a/tsconfig.test.json +++ b/tsconfig.test.json @@ -14,5 +14,5 @@ "rootDir": "src" }, "include": ["src/**/*.test.ts"], - "exclude": ["src/test/**/*.js", "src/integrations/terminal/**/*"] + "exclude": ["src/test/**/*.js"] } From f608b914b81a2e5b42f8343caa0afb9ca2dd3551 Mon Sep 17 00:00:00 2001 From: ocasta181 Date: Fri, 28 Feb 2025 22:09:54 -0800 Subject: [PATCH 11/19] remove uneccesary pass through getMaxAllowedSize; fix tests --- src/integrations/misc/extract-text.test.ts | 6 +++--- src/utils/content-size.test.ts | 8 +++++--- src/utils/content-size.ts | 3 +-- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/integrations/misc/extract-text.test.ts b/src/integrations/misc/extract-text.test.ts index 662d085f3a..046c71f0b3 100644 --- a/src/integrations/misc/extract-text.test.ts +++ b/src/integrations/misc/extract-text.test.ts @@ -29,12 +29,12 @@ describe("extract-text", () => { }) it("throws ContentTooLargeError when file would exceed max allowed size", async () => { - // Create content that would exceed max allowed size for deepseek (64k - 27k tokens) - const largeContent = "x".repeat(148000) // 37k tokens > (64k - 27k) tokens + // Create content that would exceed max allowed size (37k tokens) + const largeContent = "x".repeat(148000) // 37k tokens await fs.writeFile(tempFilePath, largeContent) try { - await extractTextFromFile(tempFilePath, 64_000) + await extractTextFromFile(tempFilePath, 37_000) // Pass pre-processed maxAllowedSize throw new Error("Should have thrown error") } catch (error) { expect(error).to.be.instanceOf(ContentTooLargeError) diff --git a/src/utils/content-size.test.ts b/src/utils/content-size.test.ts index a8fefa9d4d..2dc6fccb14 100644 --- a/src/utils/content-size.test.ts +++ b/src/utils/content-size.test.ts @@ -16,9 +16,11 @@ describe("content-size", () => { describe("wouldExceedSizeLimit", () => { it("checks if byte count would exceed max allowed size", () => { - expect(wouldExceedSizeLimit(100, 64_000)).to.equal(false) // 25 tokens < (64k - 27k) tokens - expect(wouldExceedSizeLimit(148000, 64_000)).to.equal(true) // 37k tokens > (64k - 27k) tokens - expect(wouldExceedSizeLimit(392000, 128_000)).to.equal(true) // 98k tokens > (128k - 30k) tokens + // For deepseek (64k - 27k = 37k tokens) + expect(wouldExceedSizeLimit(100, 37_000)).to.equal(false) // 25 tokens < 37k tokens + expect(wouldExceedSizeLimit(148000, 37_000)).to.equal(true) // 37k tokens = 37k tokens + // For standard models (128k - 30k = 98k tokens) + expect(wouldExceedSizeLimit(392000, 98_000)).to.equal(true) // 98k tokens = 98k tokens }) }) diff --git a/src/utils/content-size.ts b/src/utils/content-size.ts index 641ac0dee8..fb9bfdcd21 100644 --- a/src/utils/content-size.ts +++ b/src/utils/content-size.ts @@ -23,8 +23,7 @@ export function estimateTokens(bytes: number): number { */ export function wouldExceedSizeLimit(byteCount: number, contextLimit: number): boolean { const estimatedTokenCount = estimateTokens(byteCount) - const maxAllowedSize = getMaxAllowedSize(contextLimit) - return estimatedTokenCount >= maxAllowedSize + return estimatedTokenCount >= getMaxAllowedSize(contextLimit) } /** From 544d4503ac379aa9e138829887fafa9c40e0694b Mon Sep 17 00:00:00 2001 From: ocasta181 Date: Mon, 10 Mar 2025 19:32:38 -0700 Subject: [PATCH 12/19] current progress --- package-lock.json | 4 +- src/core/Cline.ts | 77 +++-- src/core/mentions/index.ts | 3 +- src/integrations/misc/extract-text.ts | 69 +++- src/integrations/terminal/TerminalManager.ts | 18 +- src/integrations/terminal/TerminalProcess.ts | 320 ++++++++---------- .../terminal/get-latest-output.ts | 7 +- src/services/mcp/McpHub.ts | 29 +- src/utils/content-size.ts | 9 +- src/utils/fs.ts | 29 +- 10 files changed, 317 insertions(+), 248 deletions(-) diff --git a/package-lock.json b/package-lock.json index 657ad84c79..95dcc37edc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "claude-dev", - "version": "3.6.0", + "version": "3.6.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "claude-dev", - "version": "3.6.0", + "version": "3.6.1", "license": "Apache-2.0", "dependencies": { "@anthropic-ai/bedrock-sdk": "^0.12.4", diff --git a/src/core/Cline.ts b/src/core/Cline.ts index f3c0b28e5e..ff874d3610 100644 --- a/src/core/Cline.ts +++ b/src/core/Cline.ts @@ -1140,7 +1140,7 @@ export class Cline { const contextWindow = this.api.getModel().info.contextWindow || 64_000 // minimum context (Deepseek) const maxAllowedSize = getMaxAllowedSize(contextWindow) - const terminalInfo = await this.terminalManager.getOrCreateTerminal(cwd) + const terminalInfo = await this.terminalManager.getOrCreateTerminal(cwd, contextWindow) terminalInfo.terminal.show() // weird visual bug when creating new terminals (even manually) where there's an empty space at the top. const process = this.terminalManager.runCommand(terminalInfo, command, maxAllowedSize) @@ -1188,7 +1188,7 @@ export class Cline { // the correct order of messages (although the webview is smart about // grouping command_output messages despite any gaps anyways) await delay(50) - + // AKF TODO -> investigate failed results result = result.trim() if (userFeedback) { @@ -1314,10 +1314,6 @@ export class Cline { const { tokensIn, tokensOut, cacheWrites, cacheReads }: ClineApiReqInfo = JSON.parse(previousRequest.text) const totalTokens = (tokensIn || 0) + (tokensOut || 0) + (cacheWrites || 0) + (cacheReads || 0) let contextWindow = this.api.getModel().info.contextWindow || 64_000 // minimum context (Deepseek) - // FIXME: hack to get anyone using openai compatible with deepseek to have the proper context window instead of the default 128k. We need a way for the user to specify the context window for models they input through openai compatible - if (this.api instanceof OpenAiHandler && this.api.getModel().id.toLowerCase().includes("deepseek")) { - contextWindow = 64_000 - } const maxAllowedSize = getMaxAllowedSize(contextWindow) // This is the most reliable way to know when we're close to hitting the context window. @@ -1964,17 +1960,15 @@ export class Cline { } // Get context window and used context from API model const contextWindow = this.api.getModel().info.contextWindow || 64_000 // minimum context (Deepseek) - const maxAllowedSize = getMaxAllowedSize(contextWindow) - // now execute the tool like normal - const content = await extractTextFromFile(absolutePath, maxAllowedSize) + // Pass the raw context window size - extractTextFromFile will calculate the appropriate limit + const content = await extractTextFromFile(absolutePath, contextWindow) pushToolResult(content) break } } catch (error) { await handleError("reading file", error) - break } } @@ -2969,6 +2963,16 @@ export class Cline { throw new Error("Cline instance aborted") } + // Log file content being passed to model + userContent.forEach((block) => { + if (block.type === "text" && block.text) { + // Look for file content markers + if (block.text.includes("= 3) { if (this.autoApprovalSettings.enabled && this.autoApprovalSettings.enableNotifications) { showSystemNotification({ @@ -3438,33 +3442,44 @@ export class Cline { if (busyTerminals.length > 0) { // terminals are cool, let's retrieve their output terminalDetails += "\n\n# Actively Running Terminals" - for (const busyTerminal of busyTerminals) { - terminalDetails += `\n## Original command: \`${busyTerminal.lastCommand}\`` - const newOutput = this.terminalManager.getUnretrievedOutput(busyTerminal.id) - if (newOutput) { - terminalDetails += `\n### New Output\n${newOutput}` - } else { - // details += `\n(Still running, no new output)` // don't want to show this right after running the command + const contextWindow = this.api.getModel().info.contextWindow || 64_000 // minimum context (Deepseek) + + // Get output from all busy terminals + const busyOutputs = await Promise.all( + busyTerminals.map(async (terminal) => { + const output = await this.terminalManager.getUnretrievedOutput(terminal.id, contextWindow) + return { terminal, output } + }), + ) + + // Add output to details + for (const { terminal, output } of busyOutputs) { + terminalDetails += `\n## Original command: \`${terminal.lastCommand}\`` + if (output) { + terminalDetails += `\n### New Output\n${output}` } } } + // only show inactive terminals if there's output to show if (inactiveTerminals.length > 0) { - const inactiveTerminalOutputs = new Map() - for (const inactiveTerminal of inactiveTerminals) { - const newOutput = this.terminalManager.getUnretrievedOutput(inactiveTerminal.id) - if (newOutput) { - inactiveTerminalOutputs.set(inactiveTerminal.id, newOutput) - } - } - if (inactiveTerminalOutputs.size > 0) { + const contextWindow = this.api.getModel().info.contextWindow || 64_000 // minimum context (Deepseek) + + // Get output from all inactive terminals + const inactiveOutputs = await Promise.all( + inactiveTerminals.map(async (terminal) => { + const output = await this.terminalManager.getUnretrievedOutput(terminal.id, contextWindow) + return { terminal, output } + }), + ) + + // Filter and add outputs that have content + const outputsWithContent = inactiveOutputs.filter(({ output }) => output) + if (outputsWithContent.length > 0) { terminalDetails += "\n\n# Inactive Terminals" - for (const [terminalId, newOutput] of inactiveTerminalOutputs) { - const inactiveTerminal = inactiveTerminals.find((t) => t.id === terminalId) - if (inactiveTerminal) { - terminalDetails += `\n## ${inactiveTerminal.lastCommand}` - terminalDetails += `\n### New Output\n${newOutput}` - } + for (const { terminal, output } of outputsWithContent) { + terminalDetails += `\n## ${terminal.lastCommand}` + terminalDetails += `\n### New Output\n${output}` } } } diff --git a/src/core/mentions/index.ts b/src/core/mentions/index.ts index 6bff6b6df5..c5db0f5359 100644 --- a/src/core/mentions/index.ts +++ b/src/core/mentions/index.ts @@ -117,7 +117,8 @@ export async function parseMentions( } } else if (mention === "terminal") { try { - const terminalOutput = await getLatestTerminalOutput() + const contextWindow = api.getModel().info.contextWindow || 64_000 // minimum context (Deepseek) + const terminalOutput = await getLatestTerminalOutput(contextWindow) parsedText += `\n\n\n${terminalOutput}\n` } catch (error) { parsedText += `\n\n\nError fetching terminal output: ${error.message}\n` diff --git a/src/integrations/misc/extract-text.ts b/src/integrations/misc/extract-text.ts index f445567f00..5a69cb319b 100644 --- a/src/integrations/misc/extract-text.ts +++ b/src/integrations/misc/extract-text.ts @@ -4,46 +4,99 @@ import pdf from "pdf-parse/lib/pdf-parse" import mammoth from "mammoth" import fs from "fs/promises" import { isBinaryFile } from "isbinaryfile" -import { estimateFileSize, wouldExceedSizeLimit } from "../../utils/content-size" +import { estimateContentSize, estimateFileSize, wouldExceedSizeLimit, getMaxAllowedSize } from "../../utils/content-size" import { ContentTooLargeError } from "../../shared/errors" -export async function extractTextFromFile(filePath: string, contextLimit: number): Promise { +/** + * Checks if terminal output would exceed size limits and returns the content if safe + * @param content The terminal output content to check + * @param contextWindow Context window limit in tokens + * @param command The command that generated this output (for error reporting) + * @returns The validated content + * @throws ContentTooLargeError if content exceeds size limit + */ +export async function extractTextFromTerminal(content: string | Buffer, contextWindow: number, command: string): Promise { + console.log(`[TERMINAL_SIZE_CHECK] Checking size for command output: ${command}`) + + // Convert to string but don't trim yet + const rawContent = content.toString() + console.log(`[TERMINAL_SIZE_CHECK] Raw content length: ${rawContent.length}`) + + // Check size before trimming + const sizeEstimate = estimateContentSize(rawContent, contextWindow) + console.log(`[TERMINAL_SIZE_CHECK] Content size: ${sizeEstimate.bytes} bytes`) + console.log(`[TERMINAL_SIZE_CHECK] Estimated tokens: ${sizeEstimate.estimatedTokens}`) + console.log(`[TERMINAL_SIZE_CHECK] Context window: ${contextWindow}`) + + if (sizeEstimate.wouldExceedLimit) { + console.log(`[TERMINAL_SIZE_CHECK] Output exceeds size limit`) + throw new ContentTooLargeError({ + type: "terminal", + command, + size: sizeEstimate, + }) + } + + // Only trim after size check passes + const cleanContent = rawContent.trim() + console.log(`[TERMINAL_SIZE_CHECK] Clean content length: ${cleanContent.length}`) + console.log(`[TERMINAL_SIZE_CHECK] Size check passed`) + return cleanContent +} + +export async function extractTextFromFile(filePath: string, contextWindow: number): Promise { try { await fs.access(filePath) } catch (error) { throw new Error(`File not found: ${filePath}`) } + console.log(`[FILE_READ_CHECK] Checking size for file: ${filePath}`) + // Get file stats to check size const stats = await fs.stat(filePath) + console.log(`[FILE_SIZE_CHECK] File size: ${stats.size} bytes`) + + // Calculate max allowed size from context window + const maxAllowedSize = getMaxAllowedSize(contextWindow) + console.log(`[FILE_SIZE_CHECK] Max allowed size: ${maxAllowedSize} tokens`) // Check if file size would exceed limit before attempting to read // This is more efficient than creating a full SizeEstimate object when we just need a boolean check - if (wouldExceedSizeLimit(stats.size, contextLimit)) { + if (wouldExceedSizeLimit(stats.size, contextWindow)) { + console.log(`[FILE_SIZE_CHECK] File exceeds size limit`) // Only create the full size estimate when we need it for the error - const sizeEstimate = await estimateFileSize(filePath, contextLimit) + const sizeEstimate = await estimateFileSize(filePath, maxAllowedSize) throw new ContentTooLargeError({ type: "file", path: filePath, size: sizeEstimate, }) } + console.log(`[FILE_SIZE_CHECK] File size check passed`) const fileExtension = path.extname(filePath).toLowerCase() + console.log(`[FILE_READ] Reading file: ${filePath}`) + let content: string switch (fileExtension) { case ".pdf": - return extractTextFromPDF(filePath) + content = await extractTextFromPDF(filePath) + break case ".docx": - return extractTextFromDOCX(filePath) + content = await extractTextFromDOCX(filePath) + break case ".ipynb": - return extractTextFromIPYNB(filePath) + content = await extractTextFromIPYNB(filePath) + break default: const isBinary = await isBinaryFile(filePath).catch(() => false) if (!isBinary) { - return await fs.readFile(filePath, "utf8") + content = await fs.readFile(filePath, "utf8") } else { throw new Error(`Cannot read text for file type: ${fileExtension}`) } } + console.log(`[FILE_READ_COMPLETE] File read complete. Content length: ${content.length} chars`) + return content } async function extractTextFromPDF(filePath: string): Promise { diff --git a/src/integrations/terminal/TerminalManager.ts b/src/integrations/terminal/TerminalManager.ts index f7542a0b3e..cdb15da0c5 100644 --- a/src/integrations/terminal/TerminalManager.ts +++ b/src/integrations/terminal/TerminalManager.ts @@ -3,6 +3,7 @@ import * as vscode from "vscode" import { arePathsEqual } from "../../utils/path" import { mergePromise, TerminalProcess, TerminalProcessResultPromise } from "./TerminalProcess" import { TerminalInfo, TerminalRegistry } from "./TerminalRegistry" +import { extractTextFromTerminal } from "../../integrations/misc/extract-text" /* TerminalManager: @@ -109,7 +110,7 @@ export class TerminalManager { } } - runCommand(terminalInfo: TerminalInfo, command: string, contextLimit?: number): TerminalProcessResultPromise { + runCommand(terminalInfo: TerminalInfo, command: string, contextLimit: number): TerminalProcessResultPromise { terminalInfo.busy = true terminalInfo.lastCommand = command const process = new TerminalProcess() @@ -156,7 +157,7 @@ export class TerminalManager { return mergePromise(process, promise) } - async getOrCreateTerminal(cwd: string): Promise { + async getOrCreateTerminal(cwd: string, contextLimit: number): Promise { const terminals = TerminalRegistry.getAllTerminals() // Find available terminal from our pool first (created for this task) @@ -179,7 +180,7 @@ export class TerminalManager { const availableTerminal = terminals.find((t) => !t.busy) if (availableTerminal) { // Navigate back to the desired directory - await this.runCommand(availableTerminal, `cd "${cwd}"`) + await this.runCommand(availableTerminal, `cd "${cwd}"`, contextLimit) this.terminalIds.add(availableTerminal.id) return availableTerminal } @@ -197,12 +198,19 @@ export class TerminalManager { .map((t) => ({ id: t.id, lastCommand: t.lastCommand })) } - getUnretrievedOutput(terminalId: number): string { + async getUnretrievedOutput(terminalId: number, contextLimit: number): Promise { if (!this.terminalIds.has(terminalId)) { return "" } const process = this.processes.get(terminalId) - return process ? process.getUnretrievedOutput() : "" + if (!process) { + return "" + } + const output = process.getUnretrievedOutput() + if (!output) { + return "" + } + return await extractTextFromTerminal(output, contextLimit, `Terminal ${terminalId} output`) } isProcessHot(terminalId: number): boolean { diff --git a/src/integrations/terminal/TerminalProcess.ts b/src/integrations/terminal/TerminalProcess.ts index c7cf08636c..a21bc12c02 100644 --- a/src/integrations/terminal/TerminalProcess.ts +++ b/src/integrations/terminal/TerminalProcess.ts @@ -1,8 +1,7 @@ import { EventEmitter } from "events" import { stripAnsi } from "./ansiUtils" import * as vscode from "vscode" -import { ContentTooLargeError } from "../../shared/errors" -import { estimateContentSize, wouldExceedSizeLimit } from "../../utils/content-size" +import { extractTextFromTerminal } from "../../integrations/misc/extract-text" export interface TerminalProcessEvents { line: [line: string] @@ -16,6 +15,13 @@ export interface TerminalProcessEvents { const PROCESS_HOT_TIMEOUT_NORMAL = 2_000 const PROCESS_HOT_TIMEOUT_COMPILING = 15_000 +// how long to wait for command output before timing out +const COMMAND_OUTPUT_TIMEOUT = 5_000 + +// VSCode shell integration sequences +const SEQUENCE_START = "\x1b]633;" // OSC 633 +const SEQUENCE_END = "\x07" // BEL + export class TerminalProcess extends EventEmitter { waitForShellIntegration: boolean = true private isListening: boolean = true @@ -24,131 +30,132 @@ export class TerminalProcess extends EventEmitter { private lastRetrievedIndex: number = 0 isHot: boolean = false private hotTimer: NodeJS.Timeout | null = null - private totalBytes: number = 0 - private contextLimit: number = 64_000 // minimum context (Deepseek) + private contextLimit: number = 0 // Will be set by run() based on API's context window private lastCommand: string = "" - // constructor() { - // super() - - async run(terminal: vscode.Terminal, command: string, contextLimit?: number) { - if (contextLimit) { - this.contextLimit = contextLimit - } + async run(terminal: vscode.Terminal, command: string, contextLimit: number) { + this.contextLimit = contextLimit this.lastCommand = command if (terminal.shellIntegration && terminal.shellIntegration.executeCommand) { const execution = terminal.shellIntegration.executeCommand(command) const stream = execution.read() - // todo: need to handle errors - let isFirstChunk = true - let didOutputNonCommand = false let didEmitEmptyLine = false - for await (let data of stream) { - // Add to total bytes before checking size - const dataBytes = Buffer.from(data).length - this.totalBytes += dataBytes + let foundCommandStart = false + let lastChunkTime = Date.now() - // Check total accumulated size against half of context limit - // Use wouldExceedSizeLimit to avoid creating unnecessary buffer - if (wouldExceedSizeLimit(this.totalBytes, this.contextLimit)) { - // Create size estimate only when needed for error details - const sizeEstimate = estimateContentSize(Buffer.alloc(this.totalBytes), this.contextLimit) - this.emit( - "error", - new ContentTooLargeError({ - type: "terminal", - command, - size: sizeEstimate, - }), - ) + for await (const chunk of stream) { + console.log("[DEBUG] Raw chunk length:", chunk.length) + console.log("[DEBUG] Raw chunk:", chunk) + + // Remove control sequences + let data = chunk.replace(/\[\?[0-9]+[a-z]/g, "") + console.log("[DEBUG] After control sequence removal:", data) + + // Find all sequences + let sequences = [] + let pos = 0 + while (true) { + const startPos = data.indexOf(SEQUENCE_START, pos) + if (startPos === -1) break + + const endPos = data.indexOf(SEQUENCE_END, startPos) + if (endPos === -1) break + + const sequence = data.slice(startPos + SEQUENCE_START.length, endPos) + sequences.push({ type: sequence[0], content: sequence.slice(2), start: startPos, end: endPos + 1 }) + pos = endPos + 1 + } + + console.log("[DEBUG] Found sequences:", sequences) + + // Process sequences + for (const seq of sequences) { + switch (seq.type) { + case "C": // Command output start + if (!foundCommandStart) { + foundCommandStart = true + data = data.slice(seq.end) + lastChunkTime = Date.now() + } + break + case "D": // Command output end + if (foundCommandStart) { + data = data.slice(0, seq.start) + } + break + } + } + + // Skip if we haven't found command start + if (!foundCommandStart) { + continue + } + + // Remove ANSI escape codes + data = stripAnsi(data) + console.log("[DEBUG] After ANSI removal:", data) + + // Process lines + if (this.isListening) { + // Add to buffer and process lines + const newBuffer = this.buffer + data + console.log("[DEBUG] New buffer:", newBuffer) + this.buffer = newBuffer + + // Process complete lines + let lineEndIndex: number + while ((lineEndIndex = this.buffer.indexOf("\n")) !== -1) { + let line = this.buffer.slice(0, lineEndIndex).trimEnd() // removes trailing \r + console.log("[DEBUG] Processing line:", line) + + // Clean up line + line = line.replace(/[^\x20-\x7E]/g, "") // Remove non-printable characters + line = line.replace(/^[^a-zA-Z0-9]*/, "") // Remove leading non-alphanumeric characters + line = this.removeLastLineArtifacts(line) + + // Skip command echo and empty lines + if (line && !command.includes(line.trim())) { + console.log("[DEBUG] Emitting line:", line) + this.emit("line", line) + this.fullOutput += line + "\n" + } + + this.buffer = this.buffer.slice(lineEndIndex + 1) + } + } + + // Check for timeout + if (Date.now() - lastChunkTime > COMMAND_OUTPUT_TIMEOUT) { + console.log("[DEBUG] Command output timeout") + break + } + + // Stop if we found command end + if (sequences.some((seq) => seq.type === "D")) { + break + } + + lastChunkTime = Date.now() + } + + // Process any remaining buffer + if (this.buffer && this.isListening) { + const line = this.removeLastLineArtifacts(this.buffer) + if (line && !command.includes(line.trim())) { + console.log("[DEBUG] Emitting final line:", line) + this.emit("line", line) + this.fullOutput += line + "\n" + } + this.buffer = "" + } + + try { + // Skip empty output + if (!this.fullOutput.trim()) { return } - // 1. Process chunk and remove artifacts - if (isFirstChunk) { - /* - The first chunk we get from this stream needs to be processed to be more human readable, ie remove vscode's custom escape sequences and identifiers, removing duplicate first char bug, etc. - */ - - // bug where sometimes the command output makes its way into vscode shell integration metadata - /* - ]633 is a custom sequence number used by VSCode shell integration: - - OSC 633 ; A ST - Mark prompt start - - OSC 633 ; B ST - Mark prompt end - - OSC 633 ; C ST - Mark pre-execution (start of command output) - - OSC 633 ; D [; ] ST - Mark execution finished with optional exit code - - OSC 633 ; E ; [; ] ST - Explicitly set command line with optional nonce - */ - // if you print this data you might see something like "eecho hello worldo hello world;5ba85d14-e92a-40c4-b2fd-71525581eeb0]633;C" but this is actually just a bunch of escape sequences, ignore up to the first ;C - /* ddateb15026-6a64-40db-b21f-2a621a9830f0]633;CTue Sep 17 06:37:04 EDT 2024 % ]633;D;0]633;P;Cwd=/Users/saoud/Repositories/test */ - // Gets output between ]633;C (command start) and ]633;D (command end) - const outputBetweenSequences = this.removeLastLineArtifacts( - data.match(/\]633;C([\s\S]*?)\]633;D/)?.[1] || "", - ).trim() - - // Once we've retrieved any potential output between sequences, we can remove everything up to end of the last sequence - // https://code.visualstudio.com/docs/terminal/shell-integration#_vs-code-custom-sequences-osc-633-st - const vscodeSequenceRegex = /\x1b\]633;.[^\x07]*\x07/g - const lastMatch = [...data.matchAll(vscodeSequenceRegex)].pop() - if (lastMatch && lastMatch.index !== undefined) { - data = data.slice(lastMatch.index + lastMatch[0].length) - } - // Place output back after removing vscode sequences - if (outputBetweenSequences) { - data = outputBetweenSequences + "\n" + data - } - // remove ansi - data = stripAnsi(data) - // Split data by newlines - let lines = data ? data.split("\n") : [] - // Remove non-human readable characters from the first line - if (lines.length > 0) { - lines[0] = lines[0].replace(/[^\x20-\x7E]/g, "") - } - // Check if first two characters are the same, if so remove the first character - if (lines.length > 0 && lines[0].length >= 2 && lines[0][0] === lines[0][1]) { - lines[0] = lines[0].slice(1) - } - // Remove everything up to the first alphanumeric character for first two lines - if (lines.length > 0) { - lines[0] = lines[0].replace(/^[^a-zA-Z0-9]*/, "") - } - if (lines.length > 1) { - lines[1] = lines[1].replace(/^[^a-zA-Z0-9]*/, "") - } - // Join lines back - data = lines.join("\n") - isFirstChunk = false - } else { - data = stripAnsi(data) - } - - // first few chunks could be the command being echoed back, so we must ignore - // note this means that 'echo' commands wont work - if (!didOutputNonCommand) { - const lines = data.split("\n") - for (let i = 0; i < lines.length; i++) { - if (command.includes(lines[i].trim())) { - lines.splice(i, 1) - i-- // Adjust index after removal - } else { - didOutputNonCommand = true - break - } - } - data = lines.join("\n") - } - - // FIXME: right now it seems that data chunks returned to us from the shell integration stream contains random commas, which from what I can tell is not the expected behavior. There has to be a better solution here than just removing all commas. - data = data.replace(/,/g, "") - - // 2. Set isHot depending on the command - // Set to hot to stall API requests until terminal is cool again - this.isHot = true - if (this.hotTimer) { - clearTimeout(this.hotTimer) - } - // these markers indicate the command is some kind of local dev server recompiling the app, which we want to wait for output of before sending request to cline + // Check if output contains compiling markers const compilingMarkers = ["compiling", "building", "bundling", "transpiling", "generating", "starting"] const markerNullifiers = [ "compiled", @@ -165,8 +172,14 @@ export class TerminalProcess extends EventEmitter { "fail", ] const isCompiling = - compilingMarkers.some((marker) => data.toLowerCase().includes(marker.toLowerCase())) && - !markerNullifiers.some((nullifier) => data.toLowerCase().includes(nullifier.toLowerCase())) + compilingMarkers.some((marker) => this.fullOutput.toLowerCase().includes(marker.toLowerCase())) && + !markerNullifiers.some((nullifier) => this.fullOutput.toLowerCase().includes(nullifier.toLowerCase())) + + // Set hot state + this.isHot = true + if (this.hotTimer) { + clearTimeout(this.hotTimer) + } this.hotTimer = setTimeout( () => { this.isHot = false @@ -174,22 +187,14 @@ export class TerminalProcess extends EventEmitter { isCompiling ? PROCESS_HOT_TIMEOUT_COMPILING : PROCESS_HOT_TIMEOUT_NORMAL, ) - // For non-immediately returning commands we want to show loading spinner right away but this wouldnt happen until it emits a line break, so as soon as we get any output we emit "" to let webview know to show spinner - if (!didEmitEmptyLine && !this.fullOutput && data) { - this.emit("line", "") // empty line to indicate start of command output stream - didEmitEmptyLine = true - } - - this.fullOutput += data - if (this.isListening) { - this.emitIfEol(data) - this.lastRetrievedIndex = this.fullOutput.length - this.buffer.length - } + // Check size + await extractTextFromTerminal(this.fullOutput, this.contextLimit, command) + } catch (error) { + this.emit("error", error) + return } - this.emitRemainingBufferIfListening() - - // for now we don't want this delaying requests since we don't send diagnostics automatically anymore (previous: "even though the command is finished, we still want to consider it 'hot' in case so that api request stalls to let diagnostics catch up") + // for now we don't want this delaying requests since we don't send diagnostics automatically anymore if (this.hotTimer) { clearTimeout(this.hotTimer) } @@ -204,66 +209,17 @@ export class TerminalProcess extends EventEmitter { this.emit("completed") this.emit("continue") this.emit("no_shell_integration") - // setTimeout(() => { - // console.log(`Emitting continue after delay for terminal`) - // // can't emit completed since we don't if the command actually completed, it could still be running server - // }, 500) // Adjust this delay as needed - } - } - - // Inspired by https://github.com/sindresorhus/execa/blob/main/lib/transform/split.js - private emitIfEol(chunk: string) { - // Check size before adding to buffer against half of context limit - const newBufferSize = this.buffer.length + chunk.length - if (wouldExceedSizeLimit(newBufferSize, this.contextLimit)) { - // Create size estimate only when needed for error details - const sizeEstimate = estimateContentSize(Buffer.alloc(0, newBufferSize), this.contextLimit) - this.emit( - "error", - new ContentTooLargeError({ - type: "terminal", - command: this.lastCommand, - size: sizeEstimate, - }), - ) - return - } - - this.buffer += chunk - let lineEndIndex: number - while ((lineEndIndex = this.buffer.indexOf("\n")) !== -1) { - let line = this.buffer.slice(0, lineEndIndex).trimEnd() // removes trailing \r - // Remove \r if present (for Windows-style line endings) - // if (line.endsWith("\r")) { - // line = line.slice(0, -1) - // } - this.emit("line", line) - this.buffer = this.buffer.slice(lineEndIndex + 1) - } - } - - private emitRemainingBufferIfListening() { - if (this.buffer && this.isListening) { - const remainingBuffer = this.removeLastLineArtifacts(this.buffer) - if (remainingBuffer) { - this.emit("line", remainingBuffer) - } - this.buffer = "" - this.lastRetrievedIndex = this.fullOutput.length } } continue() { - this.emitRemainingBufferIfListening() this.isListening = false this.removeAllListeners("line") this.emit("continue") } getUnretrievedOutput(): string { - const unretrieved = this.fullOutput.slice(this.lastRetrievedIndex) - this.lastRetrievedIndex = this.fullOutput.length - return this.removeLastLineArtifacts(unretrieved) + return this.fullOutput } // some processing to remove artifacts like '%' at the end of the buffer (it seems that since vsode uses % at the beginning of newlines in terminal, it makes its way into the stream) diff --git a/src/integrations/terminal/get-latest-output.ts b/src/integrations/terminal/get-latest-output.ts index 0c869e7fad..d1c2807432 100644 --- a/src/integrations/terminal/get-latest-output.ts +++ b/src/integrations/terminal/get-latest-output.ts @@ -1,10 +1,12 @@ import * as vscode from "vscode" +import { extractTextFromTerminal } from "../../integrations/misc/extract-text" /** * Gets the contents of the active terminal + * @param contextWindow The context window size in tokens * @returns The terminal contents as a string */ -export async function getLatestTerminalOutput(): Promise { +export async function getLatestTerminalOutput(contextWindow: number): Promise { // Store original clipboard content to restore later const originalClipboard = await vscode.env.clipboard.readText() @@ -37,7 +39,8 @@ export async function getLatestTerminalOutput(): Promise { terminalContents = lines.slice(Math.max(i, 0)).join("\n") } - return terminalContents + // Check size before returning + return await extractTextFromTerminal(terminalContents, contextWindow, "@terminal mention") } finally { // Restore original clipboard content await vscode.env.clipboard.writeText(originalClipboard) diff --git a/src/services/mcp/McpHub.ts b/src/services/mcp/McpHub.ts index c0b5904263..3a4e81430a 100644 --- a/src/services/mcp/McpHub.ts +++ b/src/services/mcp/McpHub.ts @@ -14,6 +14,7 @@ import * as fs from "fs/promises" import * as path from "path" import * as vscode from "vscode" import { z } from "zod" +import { extractTextFromFile } from "../../integrations/misc/extract-text" import { ClineProvider, GlobalFileNames } from "../../core/webview/ClineProvider" import { DEFAULT_MCP_TIMEOUT_SECONDS, @@ -57,8 +58,10 @@ export class McpHub { private fileWatchers: Map = new Map() connections: McpConnection[] = [] isConnecting: boolean = false + private api: { getModel: () => { info: { contextWindow?: number } } } - constructor(provider: ClineProvider) { + constructor(provider: ClineProvider, api: { getModel: () => { info: { contextWindow?: number } } }) { + this.api = api this.providerRef = new WeakRef(provider) this.watchMcpSettingsFile() this.initializeMcpServers() @@ -107,7 +110,8 @@ export class McpHub { this.disposables.push( vscode.workspace.onDidSaveTextDocument(async (document) => { if (arePathsEqual(document.uri.fsPath, settingsPath)) { - const content = await fs.readFile(settingsPath, "utf-8") + const contextWindow = this.api.getModel().info.contextWindow || 64_000 // minimum context (Deepseek) + const content = await extractTextFromFile(settingsPath, contextWindow) const errorMessage = "Invalid MCP settings format. Please ensure your settings follow the correct JSON format." let config: any @@ -137,7 +141,8 @@ export class McpHub { private async initializeMcpServers(): Promise { try { const settingsPath = await this.getMcpSettingsFilePath() - const content = await fs.readFile(settingsPath, "utf-8") + const contextWindow = this.api.getModel().info.contextWindow || 64_000 // minimum context (Deepseek) + const content = await extractTextFromFile(settingsPath, contextWindow) const config = JSON.parse(content) await this.updateServerConnections(config.mcpServers || {}) } catch (error) { @@ -277,7 +282,8 @@ export class McpHub { // Get autoApprove settings const settingsPath = await this.getMcpSettingsFilePath() - const content = await fs.readFile(settingsPath, "utf-8") + const contextWindow = this.api.getModel().info.contextWindow || 64_000 // minimum context (Deepseek) + const content = await extractTextFromFile(settingsPath, contextWindow) const config = JSON.parse(content) const autoApproveConfig = config.mcpServers[serverName]?.autoApprove || [] @@ -433,7 +439,8 @@ export class McpHub { private async notifyWebviewOfServerChanges(): Promise { // servers should always be sorted in the order they are defined in the settings file const settingsPath = await this.getMcpSettingsFilePath() - const content = await fs.readFile(settingsPath, "utf-8") + const contextWindow = this.api.getModel().info.contextWindow || 64_000 // minimum context (Deepseek) + const content = await extractTextFromFile(settingsPath, contextWindow) const config = JSON.parse(content) const serverOrder = Object.keys(config.mcpServers || {}) await this.providerRef.deref()?.postMessageToWebview({ @@ -468,7 +475,8 @@ export class McpHub { console.error("Settings file not accessible:", error) throw new Error("Settings file not accessible") } - const content = await fs.readFile(settingsPath, "utf-8") + const contextWindow = this.api.getModel().info.contextWindow || 64_000 // minimum context (Deepseek) + const content = await extractTextFromFile(settingsPath, contextWindow) const config = JSON.parse(content) // Validate the config structure @@ -591,7 +599,8 @@ export class McpHub { async toggleToolAutoApprove(serverName: string, toolName: string, shouldAllow: boolean): Promise { try { const settingsPath = await this.getMcpSettingsFilePath() - const content = await fs.readFile(settingsPath, "utf-8") + const contextWindow = this.api.getModel().info.contextWindow || 64_000 // minimum context (Deepseek) + const content = await extractTextFromFile(settingsPath, contextWindow) const config = JSON.parse(content) // Initialize autoApprove if it doesn't exist @@ -628,7 +637,8 @@ export class McpHub { public async deleteServer(serverName: string) { try { const settingsPath = await this.getMcpSettingsFilePath() - const content = await fs.readFile(settingsPath, "utf-8") + const contextWindow = this.api.getModel().info.contextWindow || 64_000 // minimum context (Deepseek) + const content = await extractTextFromFile(settingsPath, contextWindow) const config = JSON.parse(content) if (!config.mcpServers || typeof config.mcpServers !== "object") { config.mcpServers = {} @@ -661,7 +671,8 @@ export class McpHub { } const settingsPath = await this.getMcpSettingsFilePath() - const content = await fs.readFile(settingsPath, "utf-8") + const contextWindow = this.api.getModel().info.contextWindow || 64_000 // minimum context (Deepseek) + const content = await extractTextFromFile(settingsPath, contextWindow) const config = JSON.parse(content) if (!config.mcpServers?.[serverName]) { diff --git a/src/utils/content-size.ts b/src/utils/content-size.ts index fb9bfdcd21..bcccf793ca 100644 --- a/src/utils/content-size.ts +++ b/src/utils/content-size.ts @@ -1,7 +1,7 @@ import { stat } from "fs/promises" -// Rough approximation: 1 token ≈ 4 characters for English text -const CHARS_PER_TOKEN = 4 +// Rough approximation: 1 token ≈ 2 characters for English text +const CHARS_PER_TOKEN = 2 export interface SizeEstimate { bytes: number @@ -21,9 +21,10 @@ export function estimateTokens(bytes: number): number { * Checks if the given byte count would exceed the size limit * More efficient than creating a buffer just to check size */ -export function wouldExceedSizeLimit(byteCount: number, contextLimit: number): boolean { +export function wouldExceedSizeLimit(byteCount: number, maxAllowedSize: number): boolean { const estimatedTokenCount = estimateTokens(byteCount) - return estimatedTokenCount >= getMaxAllowedSize(contextLimit) + console.log(`[WOULD_EXCEED_SIZE_CHECK] Estimated Token Count: ${estimatedTokenCount} tokens`) + return estimatedTokenCount >= maxAllowedSize } /** diff --git a/src/utils/fs.ts b/src/utils/fs.ts index 9f7af84e4a..8474adb9aa 100644 --- a/src/utils/fs.ts +++ b/src/utils/fs.ts @@ -1,5 +1,6 @@ import fs from "fs/promises" import * as path from "path" +import { extractTextFromFile } from "../integrations/misc/extract-text" /** * Asynchronously creates all non-existing subdirectories for a given file path @@ -32,10 +33,30 @@ export async function createDirectoriesForFile(filePath: string): Promise { + return await extractTextFromFile(filePath, contextWindow) +} + +/** + * Safely reads and parses a JSON configuration file + * @param filePath Path to the JSON configuration file + * @param contextWindow Context window limit in tokens + * @returns The parsed configuration object + */ +export async function readJsonConfigFile(filePath: string, contextWindow: number): Promise { + const content = await readConfigFile(filePath, contextWindow) + return JSON.parse(content) as T +} + +/** + * Helper function to check if a path exists + * @param filePath The path to check + * @returns A promise that resolves to true if the path exists, false otherwise */ export async function fileExistsAtPath(filePath: string): Promise { try { From 6f21cb98cc4ac830073ba2646233458393185c31 Mon Sep 17 00:00:00 2001 From: ocasta181 Date: Mon, 10 Mar 2025 20:03:33 -0700 Subject: [PATCH 13/19] back out non-file related checks --- src/core/Cline.ts | 68 ++--- src/core/mentions/index.ts | 3 +- src/integrations/misc/extract-text.ts | 30 +- src/integrations/terminal/TerminalManager.ts | 22 +- src/integrations/terminal/TerminalProcess.ts | 267 +++++++++--------- .../terminal/get-latest-output.ts | 7 +- src/services/mcp/McpHub.ts | 29 +- src/utils/content-size.ts | 2 +- 8 files changed, 192 insertions(+), 236 deletions(-) diff --git a/src/core/Cline.ts b/src/core/Cline.ts index ff874d3610..b427a24749 100644 --- a/src/core/Cline.ts +++ b/src/core/Cline.ts @@ -1140,9 +1140,9 @@ export class Cline { const contextWindow = this.api.getModel().info.contextWindow || 64_000 // minimum context (Deepseek) const maxAllowedSize = getMaxAllowedSize(contextWindow) - const terminalInfo = await this.terminalManager.getOrCreateTerminal(cwd, contextWindow) + const terminalInfo = await this.terminalManager.getOrCreateTerminal(cwd) terminalInfo.terminal.show() // weird visual bug when creating new terminals (even manually) where there's an empty space at the top. - const process = this.terminalManager.runCommand(terminalInfo, command, maxAllowedSize) + const process = this.terminalManager.runCommand(terminalInfo, command) let userFeedback: { text?: string; images?: string[] } | undefined let didContinue = false @@ -1188,7 +1188,7 @@ export class Cline { // the correct order of messages (although the webview is smart about // grouping command_output messages despite any gaps anyways) await delay(50) - // AKF TODO -> investigate failed results + result = result.trim() if (userFeedback) { @@ -2963,16 +2963,6 @@ export class Cline { throw new Error("Cline instance aborted") } - // Log file content being passed to model - userContent.forEach((block) => { - if (block.type === "text" && block.text) { - // Look for file content markers - if (block.text.includes("= 3) { if (this.autoApprovalSettings.enabled && this.autoApprovalSettings.enableNotifications) { showSystemNotification({ @@ -3442,44 +3432,34 @@ export class Cline { if (busyTerminals.length > 0) { // terminals are cool, let's retrieve their output terminalDetails += "\n\n# Actively Running Terminals" - const contextWindow = this.api.getModel().info.contextWindow || 64_000 // minimum context (Deepseek) - - // Get output from all busy terminals - const busyOutputs = await Promise.all( - busyTerminals.map(async (terminal) => { - const output = await this.terminalManager.getUnretrievedOutput(terminal.id, contextWindow) - return { terminal, output } - }), - ) - - // Add output to details - for (const { terminal, output } of busyOutputs) { - terminalDetails += `\n## Original command: \`${terminal.lastCommand}\`` - if (output) { - terminalDetails += `\n### New Output\n${output}` + for (const busyTerminal of busyTerminals) { + terminalDetails += `\n## Original command: \`${busyTerminal.lastCommand}\`` + const newOutput = this.terminalManager.getUnretrievedOutput(busyTerminal.id) + if (newOutput) { + terminalDetails += `\n### New Output\n${newOutput}` + } else { + // details += `\n(Still running, no new output)` // don't want to show this right after running the command } } } // only show inactive terminals if there's output to show if (inactiveTerminals.length > 0) { - const contextWindow = this.api.getModel().info.contextWindow || 64_000 // minimum context (Deepseek) - - // Get output from all inactive terminals - const inactiveOutputs = await Promise.all( - inactiveTerminals.map(async (terminal) => { - const output = await this.terminalManager.getUnretrievedOutput(terminal.id, contextWindow) - return { terminal, output } - }), - ) - - // Filter and add outputs that have content - const outputsWithContent = inactiveOutputs.filter(({ output }) => output) - if (outputsWithContent.length > 0) { + const inactiveTerminalOutputs = new Map() + for (const inactiveTerminal of inactiveTerminals) { + const newOutput = await this.terminalManager.getUnretrievedOutput(inactiveTerminal.id) + if (newOutput) { + inactiveTerminalOutputs.set(inactiveTerminal.id, newOutput) + } + } + if (inactiveTerminalOutputs.size > 0) { terminalDetails += "\n\n# Inactive Terminals" - for (const { terminal, output } of outputsWithContent) { - terminalDetails += `\n## ${terminal.lastCommand}` - terminalDetails += `\n### New Output\n${output}` + for (const [terminalId, newOutput] of inactiveTerminalOutputs) { + const inactiveTerminal = inactiveTerminals.find((t) => t.id === terminalId) + if (inactiveTerminal) { + terminalDetails += `\n## ${inactiveTerminal.lastCommand}` + terminalDetails += `\n### New Output\n${newOutput}` + } } } } diff --git a/src/core/mentions/index.ts b/src/core/mentions/index.ts index c5db0f5359..6bff6b6df5 100644 --- a/src/core/mentions/index.ts +++ b/src/core/mentions/index.ts @@ -117,8 +117,7 @@ export async function parseMentions( } } else if (mention === "terminal") { try { - const contextWindow = api.getModel().info.contextWindow || 64_000 // minimum context (Deepseek) - const terminalOutput = await getLatestTerminalOutput(contextWindow) + const terminalOutput = await getLatestTerminalOutput() parsedText += `\n\n\n${terminalOutput}\n` } catch (error) { parsedText += `\n\n\nError fetching terminal output: ${error.message}\n` diff --git a/src/integrations/misc/extract-text.ts b/src/integrations/misc/extract-text.ts index 5a69cb319b..6b83c5b115 100644 --- a/src/integrations/misc/extract-text.ts +++ b/src/integrations/misc/extract-text.ts @@ -16,20 +16,20 @@ import { ContentTooLargeError } from "../../shared/errors" * @throws ContentTooLargeError if content exceeds size limit */ export async function extractTextFromTerminal(content: string | Buffer, contextWindow: number, command: string): Promise { - console.log(`[TERMINAL_SIZE_CHECK] Checking size for command output: ${command}`) + console.debug(`[TERMINAL_SIZE_CHECK] Checking size for command output: ${command}`) // Convert to string but don't trim yet const rawContent = content.toString() - console.log(`[TERMINAL_SIZE_CHECK] Raw content length: ${rawContent.length}`) + console.debug(`[TERMINAL_SIZE_CHECK] Raw content length: ${rawContent.length}`) // Check size before trimming const sizeEstimate = estimateContentSize(rawContent, contextWindow) - console.log(`[TERMINAL_SIZE_CHECK] Content size: ${sizeEstimate.bytes} bytes`) - console.log(`[TERMINAL_SIZE_CHECK] Estimated tokens: ${sizeEstimate.estimatedTokens}`) - console.log(`[TERMINAL_SIZE_CHECK] Context window: ${contextWindow}`) + console.debug(`[TERMINAL_SIZE_CHECK] Content size: ${sizeEstimate.bytes} bytes`) + console.debug(`[TERMINAL_SIZE_CHECK] Estimated tokens: ${sizeEstimate.estimatedTokens}`) + console.debug(`[TERMINAL_SIZE_CHECK] Context window: ${contextWindow}`) if (sizeEstimate.wouldExceedLimit) { - console.log(`[TERMINAL_SIZE_CHECK] Output exceeds size limit`) + console.debug(`[TERMINAL_SIZE_CHECK] Output exceeds size limit`) throw new ContentTooLargeError({ type: "terminal", command, @@ -39,8 +39,8 @@ export async function extractTextFromTerminal(content: string | Buffer, contextW // Only trim after size check passes const cleanContent = rawContent.trim() - console.log(`[TERMINAL_SIZE_CHECK] Clean content length: ${cleanContent.length}`) - console.log(`[TERMINAL_SIZE_CHECK] Size check passed`) + console.debug(`[TERMINAL_SIZE_CHECK] Clean content length: ${cleanContent.length}`) + console.debug(`[TERMINAL_SIZE_CHECK] Size check passed`) return cleanContent } @@ -51,20 +51,20 @@ export async function extractTextFromFile(filePath: string, contextWindow: numbe throw new Error(`File not found: ${filePath}`) } - console.log(`[FILE_READ_CHECK] Checking size for file: ${filePath}`) + console.debug(`[FILE_READ_CHECK] Checking size for file: ${filePath}`) // Get file stats to check size const stats = await fs.stat(filePath) - console.log(`[FILE_SIZE_CHECK] File size: ${stats.size} bytes`) + console.debug(`[FILE_SIZE_CHECK] File size: ${stats.size} bytes`) // Calculate max allowed size from context window const maxAllowedSize = getMaxAllowedSize(contextWindow) - console.log(`[FILE_SIZE_CHECK] Max allowed size: ${maxAllowedSize} tokens`) + console.debug(`[FILE_SIZE_CHECK] Max allowed size: ${maxAllowedSize} tokens`) // Check if file size would exceed limit before attempting to read // This is more efficient than creating a full SizeEstimate object when we just need a boolean check if (wouldExceedSizeLimit(stats.size, contextWindow)) { - console.log(`[FILE_SIZE_CHECK] File exceeds size limit`) + console.debug(`[FILE_SIZE_CHECK] File exceeds size limit`) // Only create the full size estimate when we need it for the error const sizeEstimate = await estimateFileSize(filePath, maxAllowedSize) throw new ContentTooLargeError({ @@ -73,9 +73,9 @@ export async function extractTextFromFile(filePath: string, contextWindow: numbe size: sizeEstimate, }) } - console.log(`[FILE_SIZE_CHECK] File size check passed`) + console.debug(`[FILE_SIZE_CHECK] File size check passed`) const fileExtension = path.extname(filePath).toLowerCase() - console.log(`[FILE_READ] Reading file: ${filePath}`) + console.debug(`[FILE_READ] Reading file: ${filePath}`) let content: string switch (fileExtension) { case ".pdf": @@ -95,7 +95,7 @@ export async function extractTextFromFile(filePath: string, contextWindow: numbe throw new Error(`Cannot read text for file type: ${fileExtension}`) } } - console.log(`[FILE_READ_COMPLETE] File read complete. Content length: ${content.length} chars`) + console.debug(`[FILE_READ_COMPLETE] File read complete. Content length: ${content.length} chars`) return content } diff --git a/src/integrations/terminal/TerminalManager.ts b/src/integrations/terminal/TerminalManager.ts index cdb15da0c5..eb640b8c9a 100644 --- a/src/integrations/terminal/TerminalManager.ts +++ b/src/integrations/terminal/TerminalManager.ts @@ -3,7 +3,6 @@ import * as vscode from "vscode" import { arePathsEqual } from "../../utils/path" import { mergePromise, TerminalProcess, TerminalProcessResultPromise } from "./TerminalProcess" import { TerminalInfo, TerminalRegistry } from "./TerminalRegistry" -import { extractTextFromTerminal } from "../../integrations/misc/extract-text" /* TerminalManager: @@ -110,7 +109,7 @@ export class TerminalManager { } } - runCommand(terminalInfo: TerminalInfo, command: string, contextLimit: number): TerminalProcessResultPromise { + runCommand(terminalInfo: TerminalInfo, command: string): TerminalProcessResultPromise { terminalInfo.busy = true terminalInfo.lastCommand = command const process = new TerminalProcess() @@ -142,14 +141,14 @@ export class TerminalManager { // if shell integration is already active, run the command immediately if (terminalInfo.terminal.shellIntegration) { process.waitForShellIntegration = false - process.run(terminalInfo.terminal, command, contextLimit) + process.run(terminalInfo.terminal, command) } else { // docs recommend waiting 3s for shell integration to activate pWaitFor(() => terminalInfo.terminal.shellIntegration !== undefined, { timeout: 4000 }).finally(() => { const existingProcess = this.processes.get(terminalInfo.id) if (existingProcess && existingProcess.waitForShellIntegration) { existingProcess.waitForShellIntegration = false - existingProcess.run(terminalInfo.terminal, command, contextLimit) + existingProcess.run(terminalInfo.terminal, command) } }) } @@ -157,7 +156,7 @@ export class TerminalManager { return mergePromise(process, promise) } - async getOrCreateTerminal(cwd: string, contextLimit: number): Promise { + async getOrCreateTerminal(cwd: string): Promise { const terminals = TerminalRegistry.getAllTerminals() // Find available terminal from our pool first (created for this task) @@ -180,7 +179,7 @@ export class TerminalManager { const availableTerminal = terminals.find((t) => !t.busy) if (availableTerminal) { // Navigate back to the desired directory - await this.runCommand(availableTerminal, `cd "${cwd}"`, contextLimit) + await this.runCommand(availableTerminal, `cd "${cwd}"`) this.terminalIds.add(availableTerminal.id) return availableTerminal } @@ -198,19 +197,12 @@ export class TerminalManager { .map((t) => ({ id: t.id, lastCommand: t.lastCommand })) } - async getUnretrievedOutput(terminalId: number, contextLimit: number): Promise { + getUnretrievedOutput(terminalId: number): string { if (!this.terminalIds.has(terminalId)) { return "" } const process = this.processes.get(terminalId) - if (!process) { - return "" - } - const output = process.getUnretrievedOutput() - if (!output) { - return "" - } - return await extractTextFromTerminal(output, contextLimit, `Terminal ${terminalId} output`) + return process ? process.getUnretrievedOutput() : "" } isProcessHot(terminalId: number): boolean { diff --git a/src/integrations/terminal/TerminalProcess.ts b/src/integrations/terminal/TerminalProcess.ts index a21bc12c02..8c6c352a67 100644 --- a/src/integrations/terminal/TerminalProcess.ts +++ b/src/integrations/terminal/TerminalProcess.ts @@ -1,7 +1,6 @@ import { EventEmitter } from "events" import { stripAnsi } from "./ansiUtils" import * as vscode from "vscode" -import { extractTextFromTerminal } from "../../integrations/misc/extract-text" export interface TerminalProcessEvents { line: [line: string] @@ -15,13 +14,6 @@ export interface TerminalProcessEvents { const PROCESS_HOT_TIMEOUT_NORMAL = 2_000 const PROCESS_HOT_TIMEOUT_COMPILING = 15_000 -// how long to wait for command output before timing out -const COMMAND_OUTPUT_TIMEOUT = 5_000 - -// VSCode shell integration sequences -const SEQUENCE_START = "\x1b]633;" // OSC 633 -const SEQUENCE_END = "\x07" // BEL - export class TerminalProcess extends EventEmitter { waitForShellIntegration: boolean = true private isListening: boolean = true @@ -30,132 +22,104 @@ export class TerminalProcess extends EventEmitter { private lastRetrievedIndex: number = 0 isHot: boolean = false private hotTimer: NodeJS.Timeout | null = null - private contextLimit: number = 0 // Will be set by run() based on API's context window - private lastCommand: string = "" - async run(terminal: vscode.Terminal, command: string, contextLimit: number) { - this.contextLimit = contextLimit - this.lastCommand = command + // constructor() { + // super() + + async run(terminal: vscode.Terminal, command: string) { if (terminal.shellIntegration && terminal.shellIntegration.executeCommand) { const execution = terminal.shellIntegration.executeCommand(command) const stream = execution.read() + // todo: need to handle errors + let isFirstChunk = true + let didOutputNonCommand = false let didEmitEmptyLine = false - let foundCommandStart = false - let lastChunkTime = Date.now() + for await (let data of stream) { + // 1. Process chunk and remove artifacts + if (isFirstChunk) { + /* + The first chunk we get from this stream needs to be processed to be more human readable, ie remove vscode's custom escape sequences and identifiers, removing duplicate first char bug, etc. + */ - for await (const chunk of stream) { - console.log("[DEBUG] Raw chunk length:", chunk.length) - console.log("[DEBUG] Raw chunk:", chunk) + // bug where sometimes the command output makes its way into vscode shell integration metadata + /* + ]633 is a custom sequence number used by VSCode shell integration: + - OSC 633 ; A ST - Mark prompt start + - OSC 633 ; B ST - Mark prompt end + - OSC 633 ; C ST - Mark pre-execution (start of command output) + - OSC 633 ; D [; ] ST - Mark execution finished with optional exit code + - OSC 633 ; E ; [; ] ST - Explicitly set command line with optional nonce + */ + // if you print this data you might see something like "eecho hello worldo hello world;5ba85d14-e92a-40c4-b2fd-71525581eeb0]633;C" but this is actually just a bunch of escape sequences, ignore up to the first ;C + /* ddateb15026-6a64-40db-b21f-2a621a9830f0]633;CTue Sep 17 06:37:04 EDT 2024 % ]633;D;0]633;P;Cwd=/Users/saoud/Repositories/test */ + // Gets output between ]633;C (command start) and ]633;D (command end) + const outputBetweenSequences = this.removeLastLineArtifacts( + data.match(/\]633;C([\s\S]*?)\]633;D/)?.[1] || "", + ).trim() - // Remove control sequences - let data = chunk.replace(/\[\?[0-9]+[a-z]/g, "") - console.log("[DEBUG] After control sequence removal:", data) - - // Find all sequences - let sequences = [] - let pos = 0 - while (true) { - const startPos = data.indexOf(SEQUENCE_START, pos) - if (startPos === -1) break - - const endPos = data.indexOf(SEQUENCE_END, startPos) - if (endPos === -1) break - - const sequence = data.slice(startPos + SEQUENCE_START.length, endPos) - sequences.push({ type: sequence[0], content: sequence.slice(2), start: startPos, end: endPos + 1 }) - pos = endPos + 1 - } - - console.log("[DEBUG] Found sequences:", sequences) - - // Process sequences - for (const seq of sequences) { - switch (seq.type) { - case "C": // Command output start - if (!foundCommandStart) { - foundCommandStart = true - data = data.slice(seq.end) - lastChunkTime = Date.now() - } - break - case "D": // Command output end - if (foundCommandStart) { - data = data.slice(0, seq.start) - } - break + // Once we've retrieved any potential output between sequences, we can remove everything up to end of the last sequence + // https://code.visualstudio.com/docs/terminal/shell-integration#_vs-code-custom-sequences-osc-633-st + const vscodeSequenceRegex = /\x1b\]633;.[^\x07]*\x07/g + const lastMatch = [...data.matchAll(vscodeSequenceRegex)].pop() + if (lastMatch && lastMatch.index !== undefined) { + data = data.slice(lastMatch.index + lastMatch[0].length) } + // Place output back after removing vscode sequences + if (outputBetweenSequences) { + data = outputBetweenSequences + "\n" + data + } + // remove ansi + data = stripAnsi(data) + // Split data by newlines + let lines = data ? data.split("\n") : [] + // Remove non-human readable characters from the first line + if (lines.length > 0) { + lines[0] = lines[0].replace(/[^\x20-\x7E]/g, "") + } + // Check if first two characters are the same, if so remove the first character + if (lines.length > 0 && lines[0].length >= 2 && lines[0][0] === lines[0][1]) { + lines[0] = lines[0].slice(1) + } + // Remove everything up to the first alphanumeric character for first two lines + if (lines.length > 0) { + lines[0] = lines[0].replace(/^[^a-zA-Z0-9]*/, "") + } + if (lines.length > 1) { + lines[1] = lines[1].replace(/^[^a-zA-Z0-9]*/, "") + } + // Join lines back + data = lines.join("\n") + isFirstChunk = false + } else { + data = stripAnsi(data) } - // Skip if we haven't found command start - if (!foundCommandStart) { - continue - } - - // Remove ANSI escape codes - data = stripAnsi(data) - console.log("[DEBUG] After ANSI removal:", data) - - // Process lines - if (this.isListening) { - // Add to buffer and process lines - const newBuffer = this.buffer + data - console.log("[DEBUG] New buffer:", newBuffer) - this.buffer = newBuffer - - // Process complete lines - let lineEndIndex: number - while ((lineEndIndex = this.buffer.indexOf("\n")) !== -1) { - let line = this.buffer.slice(0, lineEndIndex).trimEnd() // removes trailing \r - console.log("[DEBUG] Processing line:", line) - - // Clean up line - line = line.replace(/[^\x20-\x7E]/g, "") // Remove non-printable characters - line = line.replace(/^[^a-zA-Z0-9]*/, "") // Remove leading non-alphanumeric characters - line = this.removeLastLineArtifacts(line) - - // Skip command echo and empty lines - if (line && !command.includes(line.trim())) { - console.log("[DEBUG] Emitting line:", line) - this.emit("line", line) - this.fullOutput += line + "\n" + // first few chunks could be the command being echoed back, so we must ignore + // note this means that 'echo' commands wont work + if (!didOutputNonCommand) { + const lines = data.split("\n") + for (let i = 0; i < lines.length; i++) { + if (command.includes(lines[i].trim())) { + lines.splice(i, 1) + i-- // Adjust index after removal + } else { + didOutputNonCommand = true + break } - - this.buffer = this.buffer.slice(lineEndIndex + 1) } + data = lines.join("\n") } - // Check for timeout - if (Date.now() - lastChunkTime > COMMAND_OUTPUT_TIMEOUT) { - console.log("[DEBUG] Command output timeout") - break + // FIXME: right now it seems that data chunks returned to us from the shell integration stream contains random commas, which from what I can tell is not the expected behavior. There has to be a better solution here than just removing all commas. + data = data.replace(/,/g, "") + + // 2. Set isHot depending on the command + // Set to hot to stall API requests until terminal is cool again + this.isHot = true + if (this.hotTimer) { + clearTimeout(this.hotTimer) } - - // Stop if we found command end - if (sequences.some((seq) => seq.type === "D")) { - break - } - - lastChunkTime = Date.now() - } - - // Process any remaining buffer - if (this.buffer && this.isListening) { - const line = this.removeLastLineArtifacts(this.buffer) - if (line && !command.includes(line.trim())) { - console.log("[DEBUG] Emitting final line:", line) - this.emit("line", line) - this.fullOutput += line + "\n" - } - this.buffer = "" - } - - try { - // Skip empty output - if (!this.fullOutput.trim()) { - return - } - - // Check if output contains compiling markers + // these markers indicate the command is some kind of local dev server recompiling the app, which we want to wait for output of before sending request to cline const compilingMarkers = ["compiling", "building", "bundling", "transpiling", "generating", "starting"] const markerNullifiers = [ "compiled", @@ -172,14 +136,8 @@ export class TerminalProcess extends EventEmitter { "fail", ] const isCompiling = - compilingMarkers.some((marker) => this.fullOutput.toLowerCase().includes(marker.toLowerCase())) && - !markerNullifiers.some((nullifier) => this.fullOutput.toLowerCase().includes(nullifier.toLowerCase())) - - // Set hot state - this.isHot = true - if (this.hotTimer) { - clearTimeout(this.hotTimer) - } + compilingMarkers.some((marker) => data.toLowerCase().includes(marker.toLowerCase())) && + !markerNullifiers.some((nullifier) => data.toLowerCase().includes(nullifier.toLowerCase())) this.hotTimer = setTimeout( () => { this.isHot = false @@ -187,14 +145,22 @@ export class TerminalProcess extends EventEmitter { isCompiling ? PROCESS_HOT_TIMEOUT_COMPILING : PROCESS_HOT_TIMEOUT_NORMAL, ) - // Check size - await extractTextFromTerminal(this.fullOutput, this.contextLimit, command) - } catch (error) { - this.emit("error", error) - return + // For non-immediately returning commands we want to show loading spinner right away but this wouldnt happen until it emits a line break, so as soon as we get any output we emit "" to let webview know to show spinner + if (!didEmitEmptyLine && !this.fullOutput && data) { + this.emit("line", "") // empty line to indicate start of command output stream + didEmitEmptyLine = true + } + + this.fullOutput += data + if (this.isListening) { + this.emitIfEol(data) + this.lastRetrievedIndex = this.fullOutput.length - this.buffer.length + } } - // for now we don't want this delaying requests since we don't send diagnostics automatically anymore + this.emitRemainingBufferIfListening() + + // for now we don't want this delaying requests since we don't send diagnostics automatically anymore (previous: "even though the command is finished, we still want to consider it 'hot' in case so that api request stalls to let diagnostics catch up") if (this.hotTimer) { clearTimeout(this.hotTimer) } @@ -209,17 +175,50 @@ export class TerminalProcess extends EventEmitter { this.emit("completed") this.emit("continue") this.emit("no_shell_integration") + // setTimeout(() => { + // console.log(`Emitting continue after delay for terminal`) + // // can't emit completed since we don't if the command actually completed, it could still be running server + // }, 500) // Adjust this delay as needed + } + } + + // Inspired by https://github.com/sindresorhus/execa/blob/main/lib/transform/split.js + private emitIfEol(chunk: string) { + this.buffer += chunk + let lineEndIndex: number + while ((lineEndIndex = this.buffer.indexOf("\n")) !== -1) { + let line = this.buffer.slice(0, lineEndIndex).trimEnd() // removes trailing \r + // Remove \r if present (for Windows-style line endings) + // if (line.endsWith("\r")) { + // line = line.slice(0, -1) + // } + this.emit("line", line) + this.buffer = this.buffer.slice(lineEndIndex + 1) + } + } + + private emitRemainingBufferIfListening() { + if (this.buffer && this.isListening) { + const remainingBuffer = this.removeLastLineArtifacts(this.buffer) + if (remainingBuffer) { + this.emit("line", remainingBuffer) + } + this.buffer = "" + this.lastRetrievedIndex = this.fullOutput.length } } continue() { + this.emitRemainingBufferIfListening() this.isListening = false this.removeAllListeners("line") this.emit("continue") } getUnretrievedOutput(): string { - return this.fullOutput + const unretrieved = this.fullOutput.slice(this.lastRetrievedIndex) + this.lastRetrievedIndex = this.fullOutput.length + return this.removeLastLineArtifacts(unretrieved) } // some processing to remove artifacts like '%' at the end of the buffer (it seems that since vsode uses % at the beginning of newlines in terminal, it makes its way into the stream) diff --git a/src/integrations/terminal/get-latest-output.ts b/src/integrations/terminal/get-latest-output.ts index d1c2807432..0c869e7fad 100644 --- a/src/integrations/terminal/get-latest-output.ts +++ b/src/integrations/terminal/get-latest-output.ts @@ -1,12 +1,10 @@ import * as vscode from "vscode" -import { extractTextFromTerminal } from "../../integrations/misc/extract-text" /** * Gets the contents of the active terminal - * @param contextWindow The context window size in tokens * @returns The terminal contents as a string */ -export async function getLatestTerminalOutput(contextWindow: number): Promise { +export async function getLatestTerminalOutput(): Promise { // Store original clipboard content to restore later const originalClipboard = await vscode.env.clipboard.readText() @@ -39,8 +37,7 @@ export async function getLatestTerminalOutput(contextWindow: number): Promise = new Map() connections: McpConnection[] = [] isConnecting: boolean = false - private api: { getModel: () => { info: { contextWindow?: number } } } - constructor(provider: ClineProvider, api: { getModel: () => { info: { contextWindow?: number } } }) { - this.api = api + constructor(provider: ClineProvider) { this.providerRef = new WeakRef(provider) this.watchMcpSettingsFile() this.initializeMcpServers() @@ -110,8 +107,7 @@ export class McpHub { this.disposables.push( vscode.workspace.onDidSaveTextDocument(async (document) => { if (arePathsEqual(document.uri.fsPath, settingsPath)) { - const contextWindow = this.api.getModel().info.contextWindow || 64_000 // minimum context (Deepseek) - const content = await extractTextFromFile(settingsPath, contextWindow) + const content = await fs.readFile(settingsPath, "utf-8") const errorMessage = "Invalid MCP settings format. Please ensure your settings follow the correct JSON format." let config: any @@ -141,8 +137,7 @@ export class McpHub { private async initializeMcpServers(): Promise { try { const settingsPath = await this.getMcpSettingsFilePath() - const contextWindow = this.api.getModel().info.contextWindow || 64_000 // minimum context (Deepseek) - const content = await extractTextFromFile(settingsPath, contextWindow) + const content = await fs.readFile(settingsPath, "utf-8") const config = JSON.parse(content) await this.updateServerConnections(config.mcpServers || {}) } catch (error) { @@ -282,8 +277,7 @@ export class McpHub { // Get autoApprove settings const settingsPath = await this.getMcpSettingsFilePath() - const contextWindow = this.api.getModel().info.contextWindow || 64_000 // minimum context (Deepseek) - const content = await extractTextFromFile(settingsPath, contextWindow) + const content = await fs.readFile(settingsPath, "utf-8") const config = JSON.parse(content) const autoApproveConfig = config.mcpServers[serverName]?.autoApprove || [] @@ -439,8 +433,7 @@ export class McpHub { private async notifyWebviewOfServerChanges(): Promise { // servers should always be sorted in the order they are defined in the settings file const settingsPath = await this.getMcpSettingsFilePath() - const contextWindow = this.api.getModel().info.contextWindow || 64_000 // minimum context (Deepseek) - const content = await extractTextFromFile(settingsPath, contextWindow) + const content = await fs.readFile(settingsPath, "utf-8") const config = JSON.parse(content) const serverOrder = Object.keys(config.mcpServers || {}) await this.providerRef.deref()?.postMessageToWebview({ @@ -475,8 +468,7 @@ export class McpHub { console.error("Settings file not accessible:", error) throw new Error("Settings file not accessible") } - const contextWindow = this.api.getModel().info.contextWindow || 64_000 // minimum context (Deepseek) - const content = await extractTextFromFile(settingsPath, contextWindow) + const content = await fs.readFile(settingsPath, "utf-8") const config = JSON.parse(content) // Validate the config structure @@ -599,8 +591,7 @@ export class McpHub { async toggleToolAutoApprove(serverName: string, toolName: string, shouldAllow: boolean): Promise { try { const settingsPath = await this.getMcpSettingsFilePath() - const contextWindow = this.api.getModel().info.contextWindow || 64_000 // minimum context (Deepseek) - const content = await extractTextFromFile(settingsPath, contextWindow) + const content = await fs.readFile(settingsPath, "utf-8") const config = JSON.parse(content) // Initialize autoApprove if it doesn't exist @@ -637,8 +628,7 @@ export class McpHub { public async deleteServer(serverName: string) { try { const settingsPath = await this.getMcpSettingsFilePath() - const contextWindow = this.api.getModel().info.contextWindow || 64_000 // minimum context (Deepseek) - const content = await extractTextFromFile(settingsPath, contextWindow) + const content = await fs.readFile(settingsPath, "utf-8") const config = JSON.parse(content) if (!config.mcpServers || typeof config.mcpServers !== "object") { config.mcpServers = {} @@ -671,8 +661,7 @@ export class McpHub { } const settingsPath = await this.getMcpSettingsFilePath() - const contextWindow = this.api.getModel().info.contextWindow || 64_000 // minimum context (Deepseek) - const content = await extractTextFromFile(settingsPath, contextWindow) + const content = await fs.readFile(settingsPath, "utf-8") const config = JSON.parse(content) if (!config.mcpServers?.[serverName]) { diff --git a/src/utils/content-size.ts b/src/utils/content-size.ts index bcccf793ca..d74f216ae7 100644 --- a/src/utils/content-size.ts +++ b/src/utils/content-size.ts @@ -23,7 +23,7 @@ export function estimateTokens(bytes: number): number { */ export function wouldExceedSizeLimit(byteCount: number, maxAllowedSize: number): boolean { const estimatedTokenCount = estimateTokens(byteCount) - console.log(`[WOULD_EXCEED_SIZE_CHECK] Estimated Token Count: ${estimatedTokenCount} tokens`) + console.debug(`[WOULD_EXCEED_SIZE_CHECK] Estimated Token Count: ${estimatedTokenCount} tokens`) return estimatedTokenCount >= maxAllowedSize } From 45ddaf99e1a70cf723b324b08a83c155d56c5ccf Mon Sep 17 00:00:00 2001 From: ocasta181 Date: Mon, 10 Mar 2025 20:07:12 -0700 Subject: [PATCH 14/19] fix tests --- src/utils/content-size.test.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/utils/content-size.test.ts b/src/utils/content-size.test.ts index 2dc6fccb14..848bdd3aa4 100644 --- a/src/utils/content-size.test.ts +++ b/src/utils/content-size.test.ts @@ -9,18 +9,18 @@ const CONTEXT_LIMIT = 1000 // Context limit of 1000 tokens means max allowed siz describe("content-size", () => { describe("estimateTokens", () => { it("estimates tokens based on byte count", () => { - expect(estimateTokens(100)).to.equal(25) // 100 bytes / 4 chars per token = 25 tokens - expect(estimateTokens(7)).to.equal(2) // Should round up for partial tokens + expect(estimateTokens(100)).to.equal(50) // 100 bytes / 2 chars per token = 50 tokens + expect(estimateTokens(7)).to.equal(4) // Should round up for partial tokens }) }) describe("wouldExceedSizeLimit", () => { it("checks if byte count would exceed max allowed size", () => { // For deepseek (64k - 27k = 37k tokens) - expect(wouldExceedSizeLimit(100, 37_000)).to.equal(false) // 25 tokens < 37k tokens - expect(wouldExceedSizeLimit(148000, 37_000)).to.equal(true) // 37k tokens = 37k tokens + expect(wouldExceedSizeLimit(100, 37_000)).to.equal(false) // 50 tokens < 37k tokens + expect(wouldExceedSizeLimit(148000, 37_000)).to.equal(true) // 74k tokens > 37k tokens // For standard models (128k - 30k = 98k tokens) - expect(wouldExceedSizeLimit(392000, 98_000)).to.equal(true) // 98k tokens = 98k tokens + expect(wouldExceedSizeLimit(392000, 98_000)).to.equal(true) // 196k tokens > 98k tokens }) }) @@ -30,7 +30,7 @@ describe("content-size", () => { const result = estimateContentSize(content, CONTEXT_LIMIT) expect(result.bytes).to.equal(11) - expect(result.estimatedTokens).to.equal(3) + expect(result.estimatedTokens).to.equal(6) expect(result.wouldExceedLimit).to.equal(false) }) @@ -39,13 +39,13 @@ describe("content-size", () => { const result = estimateContentSize(content, CONTEXT_LIMIT) expect(result.bytes).to.equal(11) - expect(result.estimatedTokens).to.equal(3) + expect(result.estimatedTokens).to.equal(6) expect(result.wouldExceedLimit).to.equal(false) }) it("detects when content would exceed max allowed size", () => { // Create content that would exceed max allowed size for deepseek (64k - 27k tokens) - const largeContent = "x".repeat(148000) // 37k tokens > (64k - 27k) tokens + const largeContent = "x".repeat(148000) // 74k tokens > (64k - 27k) tokens const result = estimateContentSize(largeContent, 64_000) expect(result.wouldExceedLimit).to.equal(true) @@ -68,7 +68,7 @@ describe("content-size", () => { const result = await estimateFileSize(tempFilePath, CONTEXT_LIMIT) expect(result.bytes).to.equal(11) - expect(result.estimatedTokens).to.equal(3) + expect(result.estimatedTokens).to.equal(6) expect(result.wouldExceedLimit).to.equal(false) }) From 11a32d9a3acf6ac7edc741cad968dc4f21902fce Mon Sep 17 00:00:00 2001 From: ocasta181 Date: Mon, 10 Mar 2025 20:08:00 -0700 Subject: [PATCH 15/19] update changelog --- .changeset/seven-flowers-lay.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/seven-flowers-lay.md b/.changeset/seven-flowers-lay.md index bd1f90704b..b051dfff08 100644 --- a/.changeset/seven-flowers-lay.md +++ b/.changeset/seven-flowers-lay.md @@ -2,4 +2,4 @@ "claude-dev": patch --- -Fix a bug where cline crashes when reading large data from files or command outputs +Fix a bug where cline crashes when reading large data from files From 0f0512363f474d173752cdd2b68469294e87ece0 Mon Sep 17 00:00:00 2001 From: ocasta181 Date: Mon, 10 Mar 2025 20:52:53 -0700 Subject: [PATCH 16/19] read config in try/catch --- src/utils/fs.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/utils/fs.ts b/src/utils/fs.ts index 32d1833fa2..a388abba7f 100644 --- a/src/utils/fs.ts +++ b/src/utils/fs.ts @@ -49,8 +49,15 @@ export async function readConfigFile(filePath: string, contextWindow: number): P * @returns The parsed configuration object */ export async function readJsonConfigFile(filePath: string, contextWindow: number): Promise { - const content = await readConfigFile(filePath, contextWindow) - return JSON.parse(content) as T + try { + const content = await readConfigFile(filePath, contextWindow) + return JSON.parse(content) as T + } catch (error) { + if (error instanceof SyntaxError) { + throw new Error(`Invalid JSON in config file ${filePath}: ${error.message}`) + } + throw new Error(`Failed to read config file ${filePath}: ${error instanceof Error ? error.message : String(error)}`) + } } /** From 0afa4c4fd83ecb835ec4db8a287358686e314200 Mon Sep 17 00:00:00 2001 From: Dennis Bartlett Date: Mon, 10 Mar 2025 20:58:18 -0700 Subject: [PATCH 17/19] Update src/utils/fs.ts Co-authored-by: ellipsis-dev[bot] <65095814+ellipsis-dev[bot]@users.noreply.github.com> --- src/utils/fs.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/fs.ts b/src/utils/fs.ts index a388abba7f..77b3e53ae5 100644 --- a/src/utils/fs.ts +++ b/src/utils/fs.ts @@ -54,7 +54,7 @@ export async function readJsonConfigFile(filePath: string, contextWindow: num return JSON.parse(content) as T } catch (error) { if (error instanceof SyntaxError) { - throw new Error(`Invalid JSON in config file ${filePath}: ${error.message}`) + throw new Error(`Invalid JSON in config file ${filePath}: ${error.message}`, { cause: error }) } throw new Error(`Failed to read config file ${filePath}: ${error instanceof Error ? error.message : String(error)}`) } From e11ad166e663e9c0437289ce4f26ef21e7029b3b Mon Sep 17 00:00:00 2001 From: ocasta181 Date: Tue, 11 Mar 2025 15:01:38 -0700 Subject: [PATCH 18/19] pass less data between functions --- src/core/Cline.ts | 8 +++----- src/core/mentions/index.ts | 12 +++--------- src/integrations/misc/extract-text.ts | 5 ++++- 3 files changed, 10 insertions(+), 15 deletions(-) diff --git a/src/core/Cline.ts b/src/core/Cline.ts index acb50ce072..1783d92cdb 100644 --- a/src/core/Cline.ts +++ b/src/core/Cline.ts @@ -1161,9 +1161,6 @@ export class Cline { // Tools async executeCommandTool(command: string): Promise<[boolean, ToolResponse]> { - const contextWindow = this.api.getModel().info.contextWindow || 64_000 // minimum context (Deepseek) - const maxAllowedSize = getMaxAllowedSize(contextWindow) - const terminalInfo = await this.terminalManager.getOrCreateTerminal(cwd) terminalInfo.terminal.show() // weird visual bug when creating new terminals (even manually) where there's an empty space at the top. const process = this.terminalManager.runCommand(terminalInfo, command) @@ -2003,7 +2000,7 @@ export class Cline { telemetryService.captureToolUsage(this.taskId, block.name, false, true) } // Get context window and used context from API model - const contextWindow = this.api.getModel().info.contextWindow || 64_000 // minimum context (Deepseek) + const contextWindow = this.api.getModel().info.contextWindow // Pass the raw context window size - extractTextFromFile will calculate the appropriate limit const content = await extractTextFromFile(absolutePath, contextWindow) @@ -3379,9 +3376,10 @@ export class Cline { block.text.includes("") || block.text.includes("") ) { + let contextWindow = this.api.getModel().info.contextWindow return { ...block, - text: await parseMentions(block.text, cwd, this.urlContentFetcher, this.api), + text: await parseMentions(block.text, cwd, this.urlContentFetcher, contextWindow), } } } diff --git a/src/core/mentions/index.ts b/src/core/mentions/index.ts index 6bff6b6df5..14ff22d923 100644 --- a/src/core/mentions/index.ts +++ b/src/core/mentions/index.ts @@ -42,7 +42,7 @@ export async function parseMentions( text: string, cwd: string, urlContentFetcher: UrlContentFetcher, - api: { getModel: () => { info: { contextWindow?: number } } }, + contextWindow?: number, ): Promise { const mentions: Set = new Set() let parsedText = text.replace(mentionRegexGlobal, (match, mention) => { @@ -95,7 +95,7 @@ export async function parseMentions( } else if (mention.startsWith("/")) { const mentionPath = mention.slice(1) try { - const content = await getFileOrFolderContent(mentionPath, cwd, api) + const content = await getFileOrFolderContent(mentionPath, cwd, contextWindow) if (mention.endsWith("/")) { parsedText += `\n\n\n${content}\n` } else { @@ -150,11 +150,7 @@ export async function parseMentions( return parsedText } -async function getFileOrFolderContent( - mentionPath: string, - cwd: string, - api: { getModel: () => { info: { contextWindow?: number } } }, -): Promise { +async function getFileOrFolderContent(mentionPath: string, cwd: string, contextWindow?: number): Promise { const absPath = path.resolve(cwd, mentionPath) try { @@ -165,7 +161,6 @@ async function getFileOrFolderContent( if (isBinary) { return "(Binary file, unable to display content)" } - const contextWindow = api.getModel().info.contextWindow || 64_000 // minimum context (Deepseek) const content = await extractTextFromFile(absPath, contextWindow) return content } else if (stats.isDirectory()) { @@ -187,7 +182,6 @@ async function getFileOrFolderContent( if (isBinary) { return undefined } - const contextWindow = api.getModel().info.contextWindow || 64_000 // minimum context (Deepseek) const content = await extractTextFromFile(absoluteFilePath, contextWindow) return `\n${content}\n` } catch (error) { diff --git a/src/integrations/misc/extract-text.ts b/src/integrations/misc/extract-text.ts index 6b83c5b115..3c122b7249 100644 --- a/src/integrations/misc/extract-text.ts +++ b/src/integrations/misc/extract-text.ts @@ -44,7 +44,10 @@ export async function extractTextFromTerminal(content: string | Buffer, contextW return cleanContent } -export async function extractTextFromFile(filePath: string, contextWindow: number): Promise { +export async function extractTextFromFile( + filePath: string, + contextWindow: number = 64_000 /* minimum context (Deepseek) */, +): Promise { try { await fs.access(filePath) } catch (error) { From 48ea04f2eaaa97c95d36d9f076da64e4d97bf5bc Mon Sep 17 00:00:00 2001 From: ocasta181 Date: Tue, 11 Mar 2025 15:06:24 -0700 Subject: [PATCH 19/19] add test description --- src/utils/content-size.test.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/utils/content-size.test.ts b/src/utils/content-size.test.ts index 848bdd3aa4..67f47a5a52 100644 --- a/src/utils/content-size.test.ts +++ b/src/utils/content-size.test.ts @@ -72,6 +72,11 @@ describe("content-size", () => { expect(result.wouldExceedLimit).to.equal(false) }) + /** + * Verifies that estimateFileSize properly throws an error when given a + * non-existent file path, which is important for error handling in the application. + * The test expects fs.stat() inside estimateFileSize to fail and throw. + */ it("throws error for non-existent file", async () => { const nonExistentPath = path.join(os.tmpdir(), "non-existent.txt") try {