Compare commits

...
Author SHA1 Message Date
0xtoshii 785fdfe64e small 2025-05-27 16:50:15 -07:00
Toshii 51e4bbd23b Merge branch 'main' into to/more-file-type-uploads 2025-05-27 16:42:49 -07:00
0xtoshii add8d23036 remove binary check 2025-05-26 17:56:01 -07:00
0xtoshii 8130e8565b nit 2025-05-26 17:20:30 -07:00
0xtoshii b88d1db1fc changeset 2025-05-26 16:59:16 -07:00
0xtoshii 14d6ca7a1c merge 2025-05-26 16:43:48 -07:00
0xtoshii 7947ca28dc grpc 2025-05-26 16:36:47 -07:00
0xtoshii eb6d0255a3 header ui 2025-05-25 21:16:42 -07:00
0xtoshii 2f350acb41 task base 2025-05-25 21:04:44 -07:00
0xtoshii ba8d9eea5b ui fixes 2025-05-24 14:34:37 -07:00
0xtoshii a3ec1f307a file passing update 2025-05-23 18:52:16 -07:00
0xtoshii 095e53102d base 3 2025-05-23 17:49:26 -07:00
0xtoshii 55796bf735 keep old file 2025-05-23 17:11:16 -07:00
0xtoshii a8b276760c pre file parsing 2025-05-23 17:04:24 -07:00
0xtoshii 1683c6e144 more files 2025-05-23 00:02:37 -07:00
0xtoshii 0b8e374119 ui component 2025-05-22 23:45:17 -07:00
0xtoshii c0d08d9214 thumbnails base 2025-05-22 23:15:53 -07:00
0xtoshii b0de2e8bc1 chat view support 2025-05-22 20:11:42 -07:00
0xtoshii 5865ab77b8 update messaging 2025-05-22 17:31:26 -07:00
0xtoshii 965816d322 select files 2025-05-22 16:09:25 -07:00
0xtoshii c0e158853d grpc 2025-05-22 15:59:31 -07:00
0xtoshii a93c7981d8 process files 2025-05-22 15:01:34 -07:00
2cab8635c7 Changeset version bump (#3740)
* changeset version bump

* Updating CHANGELOG.md format

* Update CHANGELOG.md and package.json for version 3.17.1

---------

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: github-actions <github-actions@github.com>
Co-authored-by: Cline Evaluation <cline@example.com>
2025-05-23 01:37:38 +05:30
24 changed files with 673 additions and 181 deletions
+5
View File
@@ -0,0 +1,5 @@
---
"claude-dev": minor
---
add functionality for uploading any file types
+5
View File
@@ -58,3 +58,8 @@ message Boolean {
message StringArray {
repeated string values = 1;
}
message StringArrays {
repeated string values1 = 1;
repeated string values2 = 2;
}
+3
View File
@@ -31,6 +31,9 @@ service FileService {
// Select images from the file system and return as data URLs
rpc selectImages(EmptyRequest) returns (StringArray);
// Select images and other files from the file system and returns as data URLs & paths respectively
rpc selectFiles(EmptyRequest) returns (StringArrays);
// Convert URIs to workspace-relative paths
rpc getRelativePaths(RelativePathsRequest) returns (RelativePaths);
+1
View File
@@ -37,6 +37,7 @@ message ChatSettings {
message ChatContent {
optional string message = 1;
repeated string images = 2;
repeated string files = 3;
}
// Message for auto approval settings
+2
View File
@@ -40,6 +40,7 @@ message NewTaskRequest {
Metadata metadata = 1;
string text = 2;
repeated string images = 3;
repeated string files = 4;
}
// Request message for toggling task favorite status
@@ -104,4 +105,5 @@ message AskResponseRequest {
string response_type = 2;
string text = 3;
repeated string images = 4;
repeated string files = 5;
}
+21
View File
@@ -0,0 +1,21 @@
import { Controller } from ".."
import { BooleanRequest, StringArrays } from "@shared/proto/common"
import { selectFiles as selectFilesIntegration } from "@integrations/misc/process-files"
import { FileMethodHandler } from "./index"
/**
* Prompts the user to select images from the file system and returns them as data URLs
* @param controller The controller instance
* @param request Boolean request, with the value defining whether this model supports images
* @returns Two arrays of image data URLs and other file paths
*/
export const selectFiles: FileMethodHandler = async (controller: Controller, request: BooleanRequest): Promise<StringArrays> => {
try {
const { images, files } = await selectFilesIntegration(request.value)
return StringArrays.create({ values1: images, values2: files })
} catch (error) {
console.error("Error selecting images & files:", error)
// Return empty array on error
return StringArrays.create({ values1: [], values2: [] })
}
}
+7 -5
View File
@@ -141,7 +141,7 @@ export class Controller {
await updateGlobalState(this.context, "userInfo", info)
}
async initTask(task?: string, images?: string[], historyItem?: HistoryItem) {
async initTask(task?: string, images?: string[], files?: string[], historyItem?: HistoryItem) {
await this.clearTask() // ensures that an existing task doesn't exist before starting a new one, although this shouldn't be possible since user must clear task before starting a new one
const {
apiConfiguration,
@@ -188,6 +188,7 @@ export class Controller {
customInstructions,
task,
images,
files,
historyItem,
)
}
@@ -195,7 +196,7 @@ export class Controller {
async reinitExistingTaskFromId(taskId: string) {
const history = await this.getTaskWithId(taskId)
if (history) {
await this.initTask(undefined, undefined, history.historyItem)
await this.initTask(undefined, undefined, undefined, history.historyItem)
}
}
@@ -279,7 +280,7 @@ export class Controller {
// Could also do this in extension .ts
//this.postMessageToWebview({ type: "text", text: `Extension: ${Date.now()}` })
// initializing new instance of Cline will make sure that any agentically running promises in old instance don't affect our new task. this essentially creates a fresh slate for the new task
await this.initTask(message.text, message.images)
await this.initTask(message.text, message.images, message.files)
break
case "apiConfiguration":
if (message.apiConfiguration) {
@@ -618,6 +619,7 @@ export class Controller {
"messageResponse",
chatContent?.message || "PLAN_MODE_TOGGLE_RESPONSE",
chatContent?.images || [],
chatContent?.files || [],
)
} else {
this.cancelTask()
@@ -649,7 +651,7 @@ export class Controller {
// 'abandoned' will prevent this cline instance from affecting future cline instance gui. this may happen if its hanging on a streaming request
this.task.abandoned = true
}
await this.initTask(undefined, undefined, historyItem) // clears task again, so we need to abortTask manually above
await this.initTask(undefined, undefined, undefined, historyItem) // clears task again, so we need to abortTask manually above
// await this.postStateToWebview() // new Cline instance will post state when it's ready. having this here sent an empty messages array to webview leading to virtuoso having to reload the entire list
}
}
@@ -1040,7 +1042,7 @@ export class Controller {
if (id !== this.task?.taskId) {
// non-current task
const { historyItem } = await this.getTaskWithId(id)
await this.initTask(undefined, undefined, historyItem) // clears existing task
await this.initTask(undefined, undefined, undefined, historyItem) // clears existing task
}
await this.postMessageToWebview({
type: "action",
+1 -1
View File
@@ -35,7 +35,7 @@ export async function askResponse(controller: Controller, request: AskResponseRe
}
// Call the task's handler for webview responses
await controller.task.handleWebviewAskResponse(responseType, request.text, request.images)
await controller.task.handleWebviewAskResponse(responseType, request.text, request.images, request.files)
return Empty.create()
} catch (error) {
+1 -1
View File
@@ -9,6 +9,6 @@ import { NewTaskRequest } from "../../../shared/proto/task"
* @returns Empty response
*/
export async function newTask(controller: Controller, request: NewTaskRequest): Promise<Empty> {
await controller.initTask(request.text, request.images)
await controller.initTask(request.text, request.images, request.files)
return Empty.create()
}
+2 -2
View File
@@ -19,7 +19,7 @@ export async function showTaskWithId(controller: Controller, request: StringRequ
// We need to initialize the task before returning data
if (historyItem) {
// Always initialize the task with the history item
await controller.initTask(undefined, undefined, historyItem)
await controller.initTask(undefined, undefined, undefined, historyItem)
// Send UI update to show the chat view
await controller.postMessageToWebview({
@@ -46,7 +46,7 @@ export async function showTaskWithId(controller: Controller, request: StringRequ
const { historyItem: fetchedItem } = await controller.getTaskWithId(id)
// Initialize the task with the fetched item
await controller.initTask(undefined, undefined, fetchedItem)
await controller.initTask(undefined, undefined, undefined, fetchedItem)
// Send UI update to show the chat view
await controller.postMessageToWebview({
+23 -7
View File
@@ -41,15 +41,31 @@ Otherwise, if you have not completed the task and do not need additional informa
invalidMcpToolArgumentError: (serverName: string, toolName: string) =>
`Invalid JSON argument used with ${serverName} for ${toolName}. Please retry with a properly formatted JSON argument.`,
toolResult: (text: string, images?: string[]): string | Array<Anthropic.TextBlockParam | Anthropic.ImageBlockParam> => {
if (images && images.length > 0) {
const textBlock: Anthropic.TextBlockParam = { type: "text", text }
const imageBlocks: Anthropic.ImageBlockParam[] = formatImagesIntoBlocks(images)
// Placing images after text leads to better results
return [textBlock, ...imageBlocks]
} else {
toolResult: (
text: string,
images?: string[],
fileString?: string,
): string | Array<Anthropic.TextBlockParam | Anthropic.ImageBlockParam> => {
let toolResultOutput = []
if (!(images && images.length > 0) && !fileString) {
return text
}
const textBlock: Anthropic.TextBlockParam = { type: "text", text }
toolResultOutput.push(textBlock)
if (images && images.length > 0) {
const imageBlocks: Anthropic.ImageBlockParam[] = formatImagesIntoBlocks(images)
toolResultOutput.push(...imageBlocks)
}
if (fileString) {
const fileBlock: Anthropic.TextBlockParam = { type: "text", text: fileString }
toolResultOutput.push(fileBlock)
}
return toolResultOutput
},
imageBlocks: (images?: string[]): Anthropic.ImageBlockParam[] => {
+249 -91
View File
@@ -25,7 +25,7 @@ import { BrowserSession } from "@services/browser/BrowserSession"
import { UrlContentFetcher } from "@services/browser/UrlContentFetcher"
import { listFiles } from "@services/glob/list-files"
import { regexSearchFiles } from "@services/ripgrep"
import { telemetryService } from "@/services/posthog/telemetry/TelemetryService"
import { telemetryService } from "@services/posthog/telemetry/TelemetryService"
import { parseSourceCodeForDefinitionsTopLevel } from "@services/tree-sitter"
import { ApiConfiguration } from "@shared/api"
import { findLast, findLastIndex, parsePartialArrayString } from "@shared/array"
@@ -101,6 +101,8 @@ import { parseSlashCommands } from "@core/slash-commands"
import WorkspaceTracker from "@integrations/workspace/WorkspaceTracker"
import { McpHub } from "@services/mcp/McpHub"
import { isInTestMode } from "../../services/test/TestMode"
import { processFilesIntoText } from "@integrations/misc/extract-text"
import { featureFlagsService } from "@services/posthog/feature-flags/FeatureFlagsService"
import { StreamingJsonReplacer, ChangeLocation } from "@core/assistant-message/diff-json"
export const cwd =
@@ -141,6 +143,7 @@ export class Task {
private askResponse?: ClineAskResponse
private askResponseText?: string
private askResponseImages?: string[]
private askResponseFiles?: string[]
private lastMessageTs?: number
private consecutiveAutoApprovedRequestsCount: number = 0
private consecutiveMistakeCount: number = 0
@@ -192,6 +195,7 @@ export class Task {
customInstructions?: string,
task?: string,
images?: string[],
files?: string[],
historyItem?: HistoryItem,
) {
this.context = context
@@ -221,7 +225,7 @@ export class Task {
this.taskId = historyItem.id
this.taskIsFavorited = historyItem.isFavorited
this.conversationHistoryDeletedRange = historyItem.conversationHistoryDeletedRange
} else if (task || images) {
} else if (task || images || files) {
this.taskId = Date.now().toString()
} else {
throw new Error("Either historyItem or task/images must be provided")
@@ -281,8 +285,8 @@ export class Task {
// Continue with task initialization
if (historyItem) {
this.resumeTaskFromHistory()
} else if (task || images) {
this.startTask(task, images)
} else if (task || images || files) {
this.startTask(task, images, files)
}
// initialize telemetry
@@ -710,6 +714,7 @@ export class Task {
response: ClineAskResponse
text?: string
images?: string[]
files?: string[]
}> {
// If this Cline instance was aborted by the provider, then the only thing keeping us alive is a promise still running in the background, in which case we don't want to send its result to the webview as it is attached to a new instance of Cline now. So we can safely ignore the result of any active promises, and this class will be deallocated. (Although we set Cline = undefined in provider, that simply removes the reference to this instance, but the instance is still alive until this promise resolves or rejects.)
if (this.abort) {
@@ -757,6 +762,7 @@ export class Task {
this.askResponse = undefined
this.askResponseText = undefined
this.askResponseImages = undefined
this.askResponseFiles = undefined
/*
Bug for the history books:
@@ -780,6 +786,7 @@ export class Task {
this.askResponse = undefined
this.askResponseText = undefined
this.askResponseImages = undefined
this.askResponseFiles = undefined
askTs = Date.now()
this.lastMessageTs = askTs
await this.addToClineMessages({
@@ -797,6 +804,7 @@ export class Task {
this.askResponse = undefined
this.askResponseText = undefined
this.askResponseImages = undefined
this.askResponseFiles = undefined
askTs = Date.now()
this.lastMessageTs = askTs
await this.addToClineMessages({
@@ -816,20 +824,23 @@ export class Task {
response: this.askResponse!,
text: this.askResponseText,
images: this.askResponseImages,
files: this.askResponseFiles,
}
this.askResponse = undefined
this.askResponseText = undefined
this.askResponseImages = undefined
this.askResponseFiles = undefined
return result
}
async handleWebviewAskResponse(askResponse: ClineAskResponse, text?: string, images?: string[]) {
async handleWebviewAskResponse(askResponse: ClineAskResponse, text?: string, images?: string[], files?: string[]) {
this.askResponse = askResponse
this.askResponseText = text
this.askResponseImages = images
this.askResponseFiles = files
}
async say(type: ClineSay, text?: string, images?: string[], partial?: boolean): Promise<undefined> {
async say(type: ClineSay, text?: string, images?: string[], files?: string[], partial?: boolean): Promise<undefined> {
if (this.abort) {
throw new Error("Cline instance aborted")
}
@@ -843,11 +854,9 @@ export class Task {
// existing partial message, so update it
lastMessage.text = text
lastMessage.images = images
lastMessage.files = files
lastMessage.partial = partial
await this.postMessageToWebview({
type: "partialMessage",
partialMessage: lastMessage,
})
await this.postMessageToWebview({ type: "partialMessage", partialMessage: lastMessage })
} else {
// this is a new partial message, so add it with partial state
const sayTs = Date.now()
@@ -858,6 +867,7 @@ export class Task {
say: type,
text,
images,
files,
partial,
})
await this.postStateToWebview()
@@ -870,15 +880,13 @@ export class Task {
// lastMessage.ts = sayTs
lastMessage.text = text
lastMessage.images = images
lastMessage.files = files // Ensure files is updated
lastMessage.partial = false
// instead of streaming partialMessage events, we do a save and post like normal to persist to disk
await this.saveClineMessagesAndUpdateHistory()
// await this.postStateToWebview()
await this.postMessageToWebview({
type: "partialMessage",
partialMessage: lastMessage,
}) // more performant than an entire postStateToWebview
await this.postMessageToWebview({ type: "partialMessage", partialMessage: lastMessage }) // more performant than an entire postStateToWebview
} else {
// this is a new partial=false message, so add it like normal
const sayTs = Date.now()
@@ -889,6 +897,7 @@ export class Task {
say: type,
text,
images,
files,
})
await this.postStateToWebview()
}
@@ -903,6 +912,7 @@ export class Task {
say: type,
text,
images,
files,
})
await this.postStateToWebview()
}
@@ -929,7 +939,7 @@ export class Task {
// Task lifecycle
private async startTask(task?: string, images?: string[]): Promise<void> {
private async startTask(task?: string, images?: string[], files?: string[]): Promise<void> {
try {
await this.clineIgnoreController.initialize()
} catch (error) {
@@ -943,18 +953,31 @@ export class Task {
await this.postStateToWebview()
await this.say("text", task, images)
await this.say("text", task, images, files)
this.isInitialized = true
let imageBlocks: Anthropic.ImageBlockParam[] = formatResponse.imageBlocks(images)
await this.initiateTaskLoop([
let userContent: UserContent = [
{
type: "text",
text: `<task>\n${task}\n</task>`,
},
...imageBlocks,
])
]
if (files && files.length > 0) {
const fileContentString = await processFilesIntoText(files)
if (fileContentString) {
userContent.push({
type: "text",
text: fileContentString,
})
}
}
await this.initiateTaskLoop(userContent)
}
private async resumeTaskFromHistory() {
@@ -1019,14 +1042,16 @@ export class Task {
this.isInitialized = true
const { response, text, images } = await this.ask(askType) // calls poststatetowebview
const { response, text, images, files } = await this.ask(askType) // calls poststatetowebview
let responseText: string | undefined
let responseImages: string[] | undefined
let responseFiles: string[] | undefined
if (response === "messageResponse") {
await this.say("user_feedback", text, images)
await this.say("user_feedback", text, images, files)
await this.saveCheckpoint()
responseText = text
responseImages = images
responseFiles = files
}
// need to make sure that the api conversation history can be resumed by the api, even if it goes out of sync with cline messages
@@ -1107,6 +1132,16 @@ export class Task {
newUserContent.push(...formatResponse.imageBlocks(responseImages))
}
if (responseFiles && responseFiles.length > 0) {
const fileContentString = await processFilesIntoText(responseFiles)
if (fileContentString) {
newUserContent.push({
type: "text",
text: fileContentString,
})
}
}
await this.overwriteApiConversationHistory(modifiedApiConversationHistory)
await this.initiateTaskLoop(newUserContent)
}
@@ -1328,7 +1363,7 @@ export class Task {
terminalInfo.terminal.show() // weird visual bug when creating new terminals (even manually) where there's an empty space at the top.
const process = this.terminalManager.runCommand(terminalInfo, command)
let userFeedback: { text?: string; images?: string[] } | undefined
let userFeedback: { text?: string; images?: string[]; files?: string[] } | undefined
let didContinue = false
// Chunked terminal output buffering
@@ -1354,11 +1389,11 @@ export class Task {
outputBufferSize = 0
chunkEnroute = true
try {
const { response, text, images } = await this.ask("command_output", chunk)
const { response, text, images, files } = await this.ask("command_output", chunk)
if (response === "yesButtonClicked") {
// proceed while running
} else {
userFeedback = { text, images }
userFeedback = { text, images, files }
}
didContinue = true
process.continue()
@@ -1427,8 +1462,14 @@ export class Task {
result = result.trim()
if (userFeedback) {
await this.say("user_feedback", userFeedback.text, userFeedback.images)
await this.say("user_feedback", userFeedback.text, userFeedback.images, userFeedback.files)
await this.saveCheckpoint()
let fileContentString = ""
if (userFeedback.files && userFeedback.files.length > 0) {
fileContentString = await processFilesIntoText(userFeedback.files)
}
return [
true,
formatResponse.toolResult(
@@ -1436,6 +1477,7 @@ export class Task {
result.length > 0 ? `\nHere's the output so far:\n${result}` : ""
}\n\nThe user provided the following feedback:\n<feedback>\n${userFeedback.text}\n</feedback>`,
userFeedback.images,
fileContentString,
),
]
}
@@ -1901,7 +1943,7 @@ export class Task {
}
}
await this.say("text", content, undefined, block.partial)
await this.say("text", content, undefined, undefined, block.partial)
break
}
case "tool_use":
@@ -1992,13 +2034,14 @@ export class Task {
}
// The user can approve, reject, or provide feedback (rejection). However the user may also send a message along with an approval, in which case we add a separate user message with this feedback.
const pushAdditionalToolFeedback = (feedback?: string, images?: string[]) => {
if (!feedback && !images) {
const pushAdditionalToolFeedback = (feedback?: string, images?: string[], fileContentString?: string) => {
if (!feedback && (!images || images.length === 0) && !fileContentString) {
return
}
const content = formatResponse.toolResult(
`The user provided the following feedback:\n<feedback>\n${feedback}\n</feedback>`,
images,
fileContentString,
)
if (typeof content === "string") {
this.userMessageContent.push({
@@ -2011,22 +2054,32 @@ export class Task {
}
const askApproval = async (type: ClineAsk, partialMessage?: string) => {
const { response, text, images } = await this.ask(type, partialMessage, false)
const { response, text, images, files } = await this.ask(type, partialMessage, false)
if (response !== "yesButtonClicked") {
// User pressed reject button or responded with a message, which we treat as a rejection
pushToolResult(formatResponse.toolDenied())
if (text || images?.length) {
pushAdditionalToolFeedback(text, images)
await this.say("user_feedback", text, images)
if (text || (images && images.length > 0) || (files && files.length > 0)) {
let fileContentString = ""
if (files && files.length > 0) {
fileContentString = await processFilesIntoText(files)
}
pushAdditionalToolFeedback(text, images, fileContentString)
await this.say("user_feedback", text, images, files)
await this.saveCheckpoint()
}
this.didRejectTool = true // Prevent further tool uses in this message
return false
} else {
// User hit the approve button, and may have provided feedback
if (text || images?.length) {
pushAdditionalToolFeedback(text, images)
await this.say("user_feedback", text, images)
if (text || (images && images.length > 0) || (files && files.length > 0)) {
let fileContentString = ""
if (files && files.length > 0) {
fileContentString = await processFilesIntoText(files)
}
pushAdditionalToolFeedback(text, images, fileContentString)
await this.say("user_feedback", text, images, files)
await this.saveCheckpoint()
}
return true
@@ -2231,7 +2284,7 @@ export class Task {
if (this.shouldAutoApproveToolWithPath(block.name, relPath)) {
this.removeLastPartialMessageIfExistsWithType("ask", "tool") // in case the user changes auto-approval settings mid stream
await this.say("tool", partialMessage, undefined, block.partial)
await this.say("tool", partialMessage, undefined, undefined, block.partial)
} else {
this.removeLastPartialMessageIfExistsWithType("say", "tool")
await this.ask("tool", partialMessage, block.partial).catch(() => {})
@@ -2303,7 +2356,7 @@ export class Task {
} satisfies ClineSayTool)
if (this.shouldAutoApproveToolWithPath(block.name, relPath)) {
this.removeLastPartialMessageIfExistsWithType("ask", "tool")
await this.say("tool", completeMessage, undefined, false)
await this.say("tool", completeMessage, undefined, undefined, false)
this.consecutiveAutoApprovedRequestsCount++
telemetryService.captureToolUsage(this.taskId, block.name, true, true)
@@ -2318,7 +2371,12 @@ export class Task {
// Need a more customized tool response for file edits to highlight the fact that the file was not updated (particularly important for deepseek)
let didApprove = true
const { response, text, images } = await this.ask("tool", completeMessage, false)
const {
response,
text,
images,
files: askFiles,
} = await this.ask("tool", completeMessage, false)
if (response !== "yesButtonClicked") {
// User either sent a message or pressed reject button
// TODO: add similar context for other tool denial responses, to emphasize ie that a command was not run
@@ -2326,9 +2384,14 @@ export class Task {
? "The file was not updated, and maintains its original contents."
: "The file was not created."
pushToolResult(`The user denied this operation. ${fileDeniedNote}`)
if (text || images?.length) {
pushAdditionalToolFeedback(text, images)
await this.say("user_feedback", text, images)
if (text || (images && images.length > 0) || (askFiles && askFiles.length > 0)) {
let fileContentString = ""
if (askFiles && askFiles.length > 0) {
fileContentString = await processFilesIntoText(askFiles)
}
pushAdditionalToolFeedback(text, images, fileContentString)
await this.say("user_feedback", text, images, askFiles)
await this.saveCheckpoint()
}
this.didRejectTool = true
@@ -2336,9 +2399,14 @@ export class Task {
telemetryService.captureToolUsage(this.taskId, block.name, false, false)
} else {
// User hit the approve button, and may have provided feedback
if (text || images?.length) {
pushAdditionalToolFeedback(text, images)
await this.say("user_feedback", text, images)
if (text || (images && images.length > 0) || (askFiles && askFiles.length > 0)) {
let fileContentString = ""
if (askFiles && askFiles.length > 0) {
fileContentString = await processFilesIntoText(askFiles)
}
pushAdditionalToolFeedback(text, images, fileContentString)
await this.say("user_feedback", text, images, askFiles)
await this.saveCheckpoint()
}
telemetryService.captureToolUsage(this.taskId, block.name, false, true)
@@ -2426,7 +2494,7 @@ export class Task {
} satisfies ClineSayTool)
if (this.shouldAutoApproveToolWithPath(block.name, block.params.path)) {
this.removeLastPartialMessageIfExistsWithType("ask", "tool")
await this.say("tool", partialMessage, undefined, block.partial)
await this.say("tool", partialMessage, undefined, undefined, block.partial)
} else {
this.removeLastPartialMessageIfExistsWithType("say", "tool")
await this.ask("tool", partialMessage, block.partial).catch(() => {})
@@ -2457,7 +2525,7 @@ export class Task {
} satisfies ClineSayTool)
if (this.shouldAutoApproveToolWithPath(block.name, block.params.path)) {
this.removeLastPartialMessageIfExistsWithType("ask", "tool")
await this.say("tool", completeMessage, undefined, false) // need to be sending partialValue bool, since undefined has its own purpose in that the message is treated neither as a partial or completion of a partial, but as a single complete message
await this.say("tool", completeMessage, undefined, undefined, false) // need to be sending partialValue bool, since undefined has its own purpose in that the message is treated neither as a partial or completion of a partial, but as a single complete message
this.consecutiveAutoApprovedRequestsCount++
telemetryService.captureToolUsage(this.taskId, block.name, true, true)
} else {
@@ -2506,7 +2574,7 @@ export class Task {
} satisfies ClineSayTool)
if (this.shouldAutoApproveToolWithPath(block.name, block.params.path)) {
this.removeLastPartialMessageIfExistsWithType("ask", "tool")
await this.say("tool", partialMessage, undefined, block.partial)
await this.say("tool", partialMessage, undefined, undefined, block.partial)
} else {
this.removeLastPartialMessageIfExistsWithType("say", "tool")
await this.ask("tool", partialMessage, block.partial).catch(() => {})
@@ -2538,7 +2606,7 @@ export class Task {
} satisfies ClineSayTool)
if (this.shouldAutoApproveToolWithPath(block.name, block.params.path)) {
this.removeLastPartialMessageIfExistsWithType("ask", "tool")
await this.say("tool", completeMessage, undefined, false)
await this.say("tool", completeMessage, undefined, undefined, false)
this.consecutiveAutoApprovedRequestsCount++
telemetryService.captureToolUsage(this.taskId, block.name, true, true)
} else {
@@ -2579,7 +2647,7 @@ export class Task {
} satisfies ClineSayTool)
if (this.shouldAutoApproveToolWithPath(block.name, block.params.path)) {
this.removeLastPartialMessageIfExistsWithType("ask", "tool")
await this.say("tool", partialMessage, undefined, block.partial)
await this.say("tool", partialMessage, undefined, undefined, block.partial)
} else {
this.removeLastPartialMessageIfExistsWithType("say", "tool")
await this.ask("tool", partialMessage, block.partial).catch(() => {})
@@ -2608,7 +2676,7 @@ export class Task {
} satisfies ClineSayTool)
if (this.shouldAutoApproveToolWithPath(block.name, block.params.path)) {
this.removeLastPartialMessageIfExistsWithType("ask", "tool")
await this.say("tool", completeMessage, undefined, false)
await this.say("tool", completeMessage, undefined, undefined, false)
this.consecutiveAutoApprovedRequestsCount++
telemetryService.captureToolUsage(this.taskId, block.name, true, true)
} else {
@@ -2653,7 +2721,7 @@ export class Task {
} satisfies ClineSayTool)
if (this.shouldAutoApproveToolWithPath(block.name, block.params.path)) {
this.removeLastPartialMessageIfExistsWithType("ask", "tool")
await this.say("tool", partialMessage, undefined, block.partial)
await this.say("tool", partialMessage, undefined, undefined, block.partial)
} else {
this.removeLastPartialMessageIfExistsWithType("say", "tool")
await this.ask("tool", partialMessage, block.partial).catch(() => {})
@@ -2690,7 +2758,7 @@ export class Task {
} satisfies ClineSayTool)
if (this.shouldAutoApproveToolWithPath(block.name, block.params.path)) {
this.removeLastPartialMessageIfExistsWithType("ask", "tool")
await this.say("tool", completeMessage, undefined, false)
await this.say("tool", completeMessage, undefined, undefined, false)
this.consecutiveAutoApprovedRequestsCount++
telemetryService.captureToolUsage(this.taskId, block.name, true, true)
} else {
@@ -2742,6 +2810,7 @@ export class Task {
"browser_action_launch",
removeClosingTag("url", url),
undefined,
undefined,
block.partial,
)
} else {
@@ -2761,6 +2830,7 @@ export class Task {
text: removeClosingTag("text", text),
} satisfies ClineSayBrowserAction),
undefined,
undefined,
block.partial,
)
}
@@ -2779,7 +2849,7 @@ export class Task {
if (this.shouldAutoApproveTool(block.name)) {
this.removeLastPartialMessageIfExistsWithType("ask", "browser_action_launch")
await this.say("browser_action_launch", url, undefined, false)
await this.say("browser_action_launch", url, undefined, undefined, false)
this.consecutiveAutoApprovedRequestsCount++
} else {
showNotificationForApprovalIfAutoApprovalEnabled(
@@ -2836,6 +2906,7 @@ export class Task {
text,
} satisfies ClineSayBrowserAction),
undefined,
undefined,
false,
)
switch (action) {
@@ -2959,7 +3030,7 @@ export class Task {
(requiresApprovalPerLLM && autoApproveSafe && autoApproveAll)
) {
this.removeLastPartialMessageIfExistsWithType("ask", "command")
await this.say("command", command, undefined, false)
await this.say("command", command, undefined, undefined, false)
this.consecutiveAutoApprovedRequestsCount++
didAutoApprove = true
} else {
@@ -3028,7 +3099,7 @@ export class Task {
if (this.shouldAutoApproveTool(block.name)) {
this.removeLastPartialMessageIfExistsWithType("ask", "use_mcp_server")
await this.say("use_mcp_server", partialMessage, undefined, block.partial)
await this.say("use_mcp_server", partialMessage, undefined, undefined, block.partial)
} else {
this.removeLastPartialMessageIfExistsWithType("say", "use_mcp_server")
await this.ask("use_mcp_server", partialMessage, block.partial).catch(() => {})
@@ -3087,7 +3158,7 @@ export class Task {
if (this.shouldAutoApproveTool(block.name) && isToolAutoApproved) {
this.removeLastPartialMessageIfExistsWithType("ask", "use_mcp_server")
await this.say("use_mcp_server", completeMessage, undefined, false)
await this.say("use_mcp_server", completeMessage, undefined, undefined, false)
this.consecutiveAutoApprovedRequestsCount++
} else {
showNotificationForApprovalIfAutoApprovalEnabled(
@@ -3165,7 +3236,7 @@ export class Task {
if (this.shouldAutoApproveTool(block.name)) {
this.removeLastPartialMessageIfExistsWithType("ask", "use_mcp_server")
await this.say("use_mcp_server", partialMessage, undefined, block.partial)
await this.say("use_mcp_server", partialMessage, undefined, undefined, block.partial)
} else {
this.removeLastPartialMessageIfExistsWithType("say", "use_mcp_server")
await this.ask("use_mcp_server", partialMessage, block.partial).catch(() => {})
@@ -3194,7 +3265,7 @@ export class Task {
if (this.shouldAutoApproveTool(block.name)) {
this.removeLastPartialMessageIfExistsWithType("ask", "use_mcp_server")
await this.say("use_mcp_server", completeMessage, undefined, false)
await this.say("use_mcp_server", completeMessage, undefined, undefined, false)
this.consecutiveAutoApprovedRequestsCount++
} else {
showNotificationForApprovalIfAutoApprovalEnabled(
@@ -3262,7 +3333,11 @@ export class Task {
// Store the number of options for telemetry
const options = parsePartialArrayString(optionsRaw || "[]")
const { text, images } = await this.ask("followup", JSON.stringify(sharedMessage), false)
const {
text,
images,
files: followupFiles,
} = await this.ask("followup", JSON.stringify(sharedMessage), false)
// Check if options contains the text response
if (optionsRaw && text && parsePartialArrayString(optionsRaw).includes(text)) {
@@ -3280,10 +3355,17 @@ export class Task {
} else {
// Option not selected, send user feedback
telemetryService.captureOptionsIgnored(this.taskId, options.length, "act")
await this.say("user_feedback", text ?? "", images)
await this.say("user_feedback", text ?? "", images, followupFiles)
}
pushToolResult(formatResponse.toolResult(`<answer>\n${text}\n</answer>`, images))
let fileContentString = ""
if (followupFiles && followupFiles.length > 0) {
fileContentString = await processFilesIntoText(followupFiles)
}
pushToolResult(
formatResponse.toolResult(`<answer>\n${text}\n</answer>`, images, fileContentString),
)
await this.saveCheckpoint()
break
}
@@ -3315,15 +3397,21 @@ export class Task {
})
}
const { text, images } = await this.ask("new_task", context, false)
const { text, images, files: newTaskFiles } = await this.ask("new_task", context, false)
// If the user provided a response, treat it as feedback
if (text || images?.length) {
await this.say("user_feedback", text ?? "", images)
if (text || (images && images.length > 0) || (newTaskFiles && newTaskFiles.length > 0)) {
let fileContentString = ""
if (newTaskFiles && newTaskFiles.length > 0) {
fileContentString = await processFilesIntoText(newTaskFiles)
}
await this.say("user_feedback", text ?? "", images, newTaskFiles)
pushToolResult(
formatResponse.toolResult(
`The user provided feedback instead of creating a new task:\n<feedback>\n${text}\n</feedback>`,
images,
fileContentString,
),
)
} else {
@@ -3363,15 +3451,21 @@ export class Task {
})
}
const { text, images } = await this.ask("condense", context, false)
const { text, images, files: condenseFiles } = await this.ask("condense", context, false)
// If the user provided a response, treat it as feedback
if (text || images?.length) {
await this.say("user_feedback", text ?? "", images)
if (text || (images && images.length > 0) || (condenseFiles && condenseFiles.length > 0)) {
let fileContentString = ""
if (condenseFiles && condenseFiles.length > 0) {
fileContentString = await processFilesIntoText(condenseFiles)
}
await this.say("user_feedback", text ?? "", images, condenseFiles)
pushToolResult(
formatResponse.toolResult(
`The user provided feedback on the condensed conversation summary:\n<feedback>\n${text}\n</feedback>`,
images,
fileContentString,
),
)
} else {
@@ -3486,15 +3580,21 @@ export class Task {
cline_version: clineVersion,
})
const { text, images } = await this.ask("report_bug", bugReportData, false)
const { text, images, files: reportBugFiles } = await this.ask("report_bug", bugReportData, false)
// If the user provided a response, treat it as feedback
if (text || images?.length) {
await this.say("user_feedback", text ?? "", images)
if (text || (images && images.length > 0) || (reportBugFiles && reportBugFiles.length > 0)) {
let fileContentString = ""
if (reportBugFiles && reportBugFiles.length > 0) {
fileContentString = await processFilesIntoText(reportBugFiles)
}
await this.say("user_feedback", text ?? "", images, reportBugFiles)
pushToolResult(
formatResponse.toolResult(
`The user did not submit the bug, and provided feedback on the Github issue generated instead:\n<feedback>\n${text}\n</feedback>`,
images,
fileContentString,
),
)
} else {
@@ -3563,7 +3663,11 @@ export class Task {
const options = parsePartialArrayString(optionsRaw || "[]")
this.isAwaitingPlanResponse = true
let { text, images } = await this.ask("plan_mode_respond", JSON.stringify(sharedMessage), false)
let {
text,
images,
files: planResponseFiles,
} = await this.ask("plan_mode_respond", JSON.stringify(sharedMessage), false)
this.isAwaitingPlanResponse = false
// webview invoke sendMessage will send this marker in order to put webview into the proper state (responding to an ask) and as a flag to extension that the user switched to ACT mode.
@@ -3586,13 +3690,22 @@ export class Task {
}
} else {
// Option not selected, send user feedback
if (text || images?.length) {
if (
text ||
(images && images.length > 0) ||
(planResponseFiles && planResponseFiles.length > 0)
) {
telemetryService.captureOptionsIgnored(this.taskId, options.length, "plan")
await this.say("user_feedback", text ?? "", images)
await this.say("user_feedback", text ?? "", images, planResponseFiles)
await this.saveCheckpoint()
}
}
let fileContentString = ""
if (planResponseFiles && planResponseFiles.length > 0) {
fileContentString = await processFilesIntoText(planResponseFiles)
}
if (this.didRespondToPlanAskBySwitchingMode) {
pushToolResult(
formatResponse.toolResult(
@@ -3601,11 +3714,18 @@ export class Task {
? `\n\nThe user also provided the following message when switching to ACT MODE:\n<user_message>\n${text}\n</user_message>`
: ""),
images,
fileContentString,
),
)
} else {
// if we didn't switch to ACT MODE, then we can just send the user_feedback message
pushToolResult(formatResponse.toolResult(`<user_message>\n${text}\n</user_message>`, images))
pushToolResult(
formatResponse.toolResult(
`<user_message>\n${text}\n</user_message>`,
images,
fileContentString,
),
)
}
//
@@ -3623,7 +3743,7 @@ export class Task {
// shouldn't happen
break
} else {
await this.say("load_mcp_documentation", "", undefined, false)
await this.say("load_mcp_documentation", "", undefined, undefined, false)
pushToolResult(await loadMcpDocumentation(this.mcpHub))
break
}
@@ -3688,7 +3808,13 @@ export class Task {
} else {
// last message is completion_result
// we have command string, which means we have the result as well, so finish it (doesn't have to exist yet)
await this.say("completion_result", removeClosingTag("result", result), undefined, false)
await this.say(
"completion_result",
removeClosingTag("result", result),
undefined,
undefined,
false,
)
await this.saveCheckpoint(true)
await addNewChangesFlagToLastCompletionResultMessage()
await this.ask("command", removeClosingTag("command", command), block.partial).catch(
@@ -3701,6 +3827,7 @@ export class Task {
"completion_result",
removeClosingTag("result", result),
undefined,
undefined,
block.partial,
)
}
@@ -3724,7 +3851,7 @@ export class Task {
if (command) {
if (lastMessage && lastMessage.ask !== "command") {
// haven't sent a command message yet so first send completion_result then command
await this.say("completion_result", result, undefined, false)
await this.say("completion_result", result, undefined, undefined, false)
await this.saveCheckpoint(true)
await addNewChangesFlagToLastCompletionResultMessage()
telemetryService.captureTaskCompleted(this.taskId)
@@ -3749,19 +3876,24 @@ export class Task {
// user didn't reject, but the command may have output
commandResult = execCommandResult
} else {
await this.say("completion_result", result, undefined, false)
await this.say("completion_result", result, undefined, undefined, false)
await this.saveCheckpoint(true)
await addNewChangesFlagToLastCompletionResultMessage()
telemetryService.captureTaskCompleted(this.taskId)
}
// we already sent completion_result says, an empty string asks relinquishes control over button and field
const { response, text, images } = await this.ask("completion_result", "", false)
const {
response,
text,
images,
files: completionFiles,
} = await this.ask("completion_result", "", false)
if (response === "yesButtonClicked") {
pushToolResult("") // signals to recursive loop to stop (for now this never happens since yesButtonClicked will trigger a new task)
break
}
await this.say("user_feedback", text ?? "", images)
await this.say("user_feedback", text ?? "", images, completionFiles)
await this.saveCheckpoint()
const toolResults: (Anthropic.TextBlockParam | Anthropic.ImageBlockParam)[] = []
@@ -3786,6 +3918,18 @@ export class Task {
})
this.userMessageContent.push(...toolResults)
let fileContentString = ""
if (completionFiles && completionFiles.length > 0) {
fileContentString = await processFilesIntoText(completionFiles)
}
if (fileContentString) {
this.userMessageContent.push({
type: "text",
text: fileContentString,
})
}
//
break
}
@@ -3850,22 +3994,36 @@ export class Task {
message: "Cline is having trouble. Would you like to continue the task?",
})
}
const { response, text, images } = await this.ask(
const { response, text, images, files } = await this.ask(
"mistake_limit_reached",
this.api.getModel().id.includes("claude")
? `This may indicate a failure in his thought process or inability to use a tool properly, which can be mitigated with some user guidance (e.g. "Try breaking down the task into smaller steps").`
: "Cline uses complex prompts and iterative task execution that may be challenging for less capable models. For best results, it's recommended to use Claude 3.7 Sonnet for its advanced agentic coding capabilities.",
)
if (response === "messageResponse") {
userContent.push(
...[
{
type: "text",
text: formatResponse.tooManyMistakes(text),
} as Anthropic.Messages.TextBlockParam,
...formatResponse.imageBlocks(images),
],
)
// This userContent is for the *next* API call.
const feedbackUserContent: UserContent = []
feedbackUserContent.push({
type: "text",
text: formatResponse.tooManyMistakes(text),
})
if (images && images.length > 0) {
feedbackUserContent.push(...formatResponse.imageBlocks(images))
}
let fileContentString = ""
if (files && files.length > 0) {
fileContentString = await processFilesIntoText(files)
}
if (fileContentString) {
feedbackUserContent.push({
type: "text",
text: fileContentString,
})
}
userContent = feedbackUserContent
}
this.consecutiveMistakeCount = 0
}
@@ -4087,13 +4245,13 @@ export class Task {
reasoningMessage += chunk.reasoning
// fixes bug where cancelling task > aborts task > for loop may be in middle of streaming reasoning > say function throws error before we get a chance to properly clean up and cancel the task.
if (!this.abort) {
await this.say("reasoning", reasoningMessage, undefined, true)
await this.say("reasoning", reasoningMessage, undefined, undefined, true)
}
break
case "text":
if (reasoningMessage && assistantMessage.length === 0) {
// complete reasoning message
await this.say("reasoning", reasoningMessage, undefined, false)
await this.say("reasoning", reasoningMessage, undefined, undefined, false)
}
assistantMessage += chunk.text
// parse raw assistant message into content blocks
+31
View File
@@ -75,3 +75,34 @@ async function extractTextFromIPYNB(filePath: string): Promise<string> {
return extractedText
}
/**
* Helper function used to load file(s) and format them into a string
*/
export async function processFilesIntoText(files: string[]): Promise<string> {
const fileContentsPromises = files.map(async (filePath) => {
try {
// Check if file exists and is binary
//const isBinary = await isBinaryFile(filePath).catch(() => false)
//if (isBinary) {
// return `<file_content path="${filePath.toPosix()}">\n(Binary file, unable to display content)\n</file_content>`
//}
const content = await extractTextFromFile(filePath)
return `<file_content path="${filePath.toPosix()}">\n${content}\n</file_content>`
} catch (error) {
console.error(`Error processing file ${filePath}:`, error)
return `<file_content path="${filePath.toPosix()}">\nError fetching content: ${error.message}\n</file_content>`
}
})
const fileContents = await Promise.all(fileContentsPromises)
const validFileContents = fileContents.filter((content) => content !== null).join("\n\n")
if (validFileContents) {
return `Files attached by the user:\n\n${validFileContents}`
}
// returns empty string if no files were loaded properly
return ""
}
+109
View File
@@ -0,0 +1,109 @@
import * as vscode from "vscode"
import fs from "fs/promises"
import * as path from "path"
import sizeOf from "image-size"
/**
* Supports processing of images and other file types
* For models which don't support images, will not allow them to be selected
*/
export async function selectFiles(imagesAllowed: boolean): Promise<{ images: string[]; files: string[] }> {
const IMAGE_EXTENSIONS = ["png", "jpg", "jpeg", "webp"] // supported by anthropic and openrouter
const OTHER_FILE_EXTENSIONS = ["xml", "json", "txt", "log", "md", "docx", "ipynb", "pdf"]
const options: vscode.OpenDialogOptions = {
canSelectMany: true,
openLabel: "Select",
filters: {
Files: imagesAllowed ? [...IMAGE_EXTENSIONS, ...OTHER_FILE_EXTENSIONS] : OTHER_FILE_EXTENSIONS,
},
}
const fileUris = await vscode.window.showOpenDialog(options)
if (!fileUris || fileUris.length === 0) {
return { images: [], files: [] }
}
const processFilesPromises = fileUris.map(async (uri) => {
const filePath = uri.fsPath
const fileExtension = path.extname(filePath).toLowerCase().substring(1)
//const fileName = path.basename(filePath)
const isImage = IMAGE_EXTENSIONS.includes(fileExtension)
if (isImage) {
let buffer: Buffer
try {
// Read the file into a buffer first
buffer = await fs.readFile(filePath)
// Convert Node.js Buffer to Uint8Array
const uint8Array = new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength)
const dimensions = sizeOf(uint8Array) // Get dimensions from Uint8Array
if (dimensions.width! > 7500 || dimensions.height! > 7500) {
console.warn(`Image dimensions exceed 7500px, skipping: ${filePath}`)
vscode.window.showErrorMessage(
`Image too large: ${path.basename(filePath)} was skipped (dimensions exceed 7500px).`,
)
return null
}
} catch (error) {
console.error(`Error reading file or getting dimensions for ${filePath}:`, error)
vscode.window.showErrorMessage(`Could not read dimensions for ${path.basename(filePath)}, skipping.`)
return null
}
// If dimensions are valid, proceed to convert the existing buffer to base64
const base64 = buffer.toString("base64")
const mimeType = getMimeType(filePath)
return { type: "image", data: `data:${mimeType};base64,${base64}` }
} else {
// for standard models we will check the size of the file to ensure its not too large
try {
const stats = await fs.stat(filePath)
if (stats.size > 20 * 1000 * 1024) {
console.warn(`File too large, skipping: ${filePath}`)
vscode.window.showErrorMessage(`File too large: ${path.basename(filePath)} was skipped (size exceeds 20MB).`)
return null
}
} catch (error) {
console.error(`Error checking file size for ${filePath}:`, error)
vscode.window.showErrorMessage(`Could not check file size for ${path.basename(filePath)}, skipping.`)
return null
}
return { type: "file", data: filePath }
}
})
const dataUrlsWithNulls = await Promise.all(processFilesPromises)
const dataUrlsWithoutNulls = dataUrlsWithNulls.filter((item) => item !== null)
const images: string[] = []
const files: string[] = []
for (const item of dataUrlsWithoutNulls) {
if (item.type === "image") {
images.push(item.data)
} else {
files.push(item.data)
}
}
return { images, files }
}
function getMimeType(filePath: string): string {
const ext = path.extname(filePath).toLowerCase()
switch (ext) {
case ".png":
return "image/png"
case ".jpeg":
case ".jpg":
return "image/jpeg"
case ".webp":
return "image/webp"
default:
throw new Error(`Unsupported file type: ${ext}`)
}
}
+1
View File
@@ -1,4 +1,5 @@
export interface ChatContent {
message?: string
images?: string[]
files?: string[]
}
+2
View File
@@ -51,6 +51,7 @@ export interface ExtensionMessage {
| "focusChatInput"
state?: ExtensionState
images?: string[]
files?: string[]
ollamaModels?: string[]
lmStudioModels?: string[]
vsCodeLmModels?: { vendor?: string; family?: string; version?: string; id?: string }[]
@@ -147,6 +148,7 @@ export interface ClineMessage {
text?: string
reasoning?: string
images?: string[]
files?: string[]
partial?: boolean
lastCheckpointHash?: string
isCheckpointCheckedOut?: boolean
+1
View File
@@ -34,6 +34,7 @@ export interface WebviewMessage {
disabled?: boolean
apiConfiguration?: ApiConfiguration
images?: string[]
files?: string[]
bool?: boolean
number?: number
browserSettings?: BrowserSettings
@@ -35,6 +35,7 @@ export function convertChatContentToProtoChatContent(chatContent?: ChatContent):
return {
message: chatContent.message,
images: chatContent.images || [],
files: chatContent.files || [],
}
}
@@ -49,5 +50,6 @@ export function convertProtoChatContentToChatContent(protoChatContent?: ProtoCha
return {
message: protoChatContent.message,
images: protoChatContent.images || [],
files: protoChatContent.files || [],
}
}
+2 -1
View File
@@ -111,7 +111,7 @@ interface ChatRowProps {
isLast: boolean
onHeightChange: (isTaller: boolean) => void
inputValue?: string
sendMessageFromChatRow?: (text: string, images: string[]) => void
sendMessageFromChatRow?: (text: string, images: string[], files: string[]) => void
onSetQuote: (text: string) => void
}
@@ -1046,6 +1046,7 @@ export const ChatRowContent = ({
<UserMessage
text={message.text}
images={message.images}
files={message.files}
messageTs={message.ts}
sendMessageFromChatRow={sendMessageFromChatRow}
/>
+48 -20
View File
@@ -33,7 +33,7 @@ import { CODE_BLOCK_BG_COLOR } from "@/components/common/CodeBlock"
import Thumbnails from "@/components/common/Thumbnails"
import Tooltip from "@/components/common/Tooltip"
import ApiOptions, { normalizeApiConfiguration } from "@/components/settings/ApiOptions"
import { MAX_IMAGES_PER_MESSAGE } from "@/components/chat/ChatView"
import { MAX_IMAGES_AND_FILES_PER_MESSAGE } from "@/components/chat/ChatView"
import ContextMenu from "@/components/chat/ContextMenu"
import SlashCommandMenu from "@/components/chat/SlashCommandMenu"
import { ChatSettings } from "@shared/ChatSettings"
@@ -65,11 +65,13 @@ interface ChatTextAreaProps {
setInputValue: (value: string) => void
sendingDisabled: boolean
placeholderText: string
selectedFiles: string[]
selectedImages: string[]
setSelectedImages: React.Dispatch<React.SetStateAction<string[]>>
setSelectedFiles: React.Dispatch<React.SetStateAction<string[]>>
onSend: () => void
onSelectImages: () => void
shouldDisableImages: boolean
onSelectFilesAndImages: () => void
shouldDisableFilesAndImages: boolean
onHeightChange?: (height: number) => void
onFocusChange?: (isFocused: boolean) => void
}
@@ -250,11 +252,13 @@ const ChatTextArea = forwardRef<HTMLTextAreaElement, ChatTextAreaProps>(
setInputValue,
sendingDisabled,
placeholderText,
selectedFiles,
selectedImages,
setSelectedImages,
setSelectedFiles,
onSend,
onSelectImages,
shouldDisableImages,
onSelectFilesAndImages,
shouldDisableFilesAndImages,
onHeightChange,
onFocusChange,
},
@@ -836,7 +840,7 @@ const ChatTextArea = forwardRef<HTMLTextAreaElement, ChatTextAreaProps>(
const [type, subtype] = item.type.split("/")
return type === "image" && acceptedTypes.includes(subtype)
})
if (!shouldDisableImages && imageItems.length > 0) {
if (!shouldDisableFilesAndImages && imageItems.length > 0) {
e.preventDefault()
const imagePromises = imageItems.map((item) => {
return new Promise<string | null>((resolve) => {
@@ -873,13 +877,28 @@ const ChatTextArea = forwardRef<HTMLTextAreaElement, ChatTextAreaProps>(
const dataUrls = imageDataArray.filter((dataUrl): dataUrl is string => dataUrl !== null)
//.map((dataUrl) => dataUrl.split(",")[1]) // strip the mime type prefix, sharp doesn't need it
if (dataUrls.length > 0) {
setSelectedImages((prevImages) => [...prevImages, ...dataUrls].slice(0, MAX_IMAGES_PER_MESSAGE))
const filesAndImagesLength = selectedImages.length + selectedFiles.length
const availableSlots = MAX_IMAGES_AND_FILES_PER_MESSAGE - filesAndImagesLength
if (availableSlots > 0) {
const imagesToAdd = Math.min(dataUrls.length, availableSlots)
setSelectedImages((prevImages) => [...prevImages, ...dataUrls.slice(0, imagesToAdd)])
}
} else {
console.warn("No valid images were processed")
}
}
},
[shouldDisableImages, setSelectedImages, cursorPosition, setInputValue, inputValue, showDimensionErrorMessage],
[
shouldDisableFilesAndImages,
setSelectedImages,
selectedImages,
selectedFiles,
cursorPosition,
setInputValue,
inputValue,
showDimensionErrorMessage,
],
)
const handleThumbnailsHeightChange = useCallback((height: number) => {
@@ -988,6 +1007,7 @@ const ChatTextArea = forwardRef<HTMLTextAreaElement, ChatTextAreaProps>(
chatContent: {
message: inputValue.trim() ? inputValue : undefined,
images: selectedImages.length > 0 ? selectedImages : undefined,
files: selectedFiles.length > 0 ? selectedFiles : undefined,
},
})
// Focus the textarea after mode toggle with slight delay
@@ -995,7 +1015,7 @@ const ChatTextArea = forwardRef<HTMLTextAreaElement, ChatTextAreaProps>(
textAreaRef.current?.focus()
}, 100)
}, changeModeDelay)
}, [chatSettings.mode, showModelSelector, submitApiConfig, inputValue, selectedImages])
}, [chatSettings.mode, showModelSelector, submitApiConfig, inputValue, selectedImages, selectedFiles])
useShortcut("Meta+Shift+a", onModeToggle, { disableTextInputs: false }) // important that we don't disable the text input here
@@ -1272,7 +1292,7 @@ const ChatTextArea = forwardRef<HTMLTextAreaElement, ChatTextAreaProps>(
return type === "image" && acceptedTypes.includes(subtype)
})
if (shouldDisableImages || imageFiles.length === 0) {
if (shouldDisableFilesAndImages || imageFiles.length === 0) {
return
}
@@ -1280,7 +1300,13 @@ const ChatTextArea = forwardRef<HTMLTextAreaElement, ChatTextAreaProps>(
const dataUrls = imageDataArray.filter((dataUrl): dataUrl is string => dataUrl !== null)
if (dataUrls.length > 0) {
setSelectedImages((prevImages) => [...prevImages, ...dataUrls].slice(0, MAX_IMAGES_PER_MESSAGE))
const filesAndImagesLength = selectedImages.length + selectedFiles.length
const availableSlots = MAX_IMAGES_AND_FILES_PER_MESSAGE - filesAndImagesLength
if (availableSlots > 0) {
const imagesToAdd = Math.min(dataUrls.length, availableSlots)
setSelectedImages((prevImages) => [...prevImages, ...dataUrls.slice(0, imagesToAdd)])
}
} else {
console.warn("No valid images were processed")
}
@@ -1400,7 +1426,7 @@ const ChatTextArea = forwardRef<HTMLTextAreaElement, ChatTextAreaProps>(
fontWeight: "bold",
fontSize: "12px",
}}>
Only image files are supported
Files other than images are currently disabled
</span>
</div>
)}
@@ -1544,10 +1570,12 @@ const ChatTextArea = forwardRef<HTMLTextAreaElement, ChatTextAreaProps>(
}}
onScroll={() => updateHighlights()}
/>
{selectedImages.length > 0 && (
{(selectedImages.length > 0 || selectedFiles.length > 0) && (
<Thumbnails
images={selectedImages}
files={selectedFiles}
setImages={setSelectedImages}
setFiles={setSelectedFiles}
onHeightChange={handleThumbnailsHeightChange}
style={{
position: "absolute",
@@ -1637,21 +1665,21 @@ const ChatTextArea = forwardRef<HTMLTextAreaElement, ChatTextAreaProps>(
</VSCodeButton>
</Tooltip>
<Tooltip tipText="Add Images">
<Tooltip tipText="Add Files & Images">
<VSCodeButton
data-testid="images-button"
data-testid="files-button"
appearance="icon"
aria-label="Add Images"
disabled={shouldDisableImages}
aria-label="Add Files & Images"
disabled={shouldDisableFilesAndImages}
onClick={() => {
if (!shouldDisableImages) {
onSelectImages()
if (!shouldDisableFilesAndImages) {
onSelectFilesAndImages()
}
}}
style={{ padding: "0px 0px", height: "20px" }}>
<ButtonContainer>
<span
className="codicon codicon-device-camera flex items-center"
className="codicon codicon-add flex items-center"
style={{ fontSize: "14px", marginBottom: -3 }}
/>
</ButtonContainer>
+54 -38
View File
@@ -88,7 +88,8 @@ async function convertHtmlToMarkdown(html: string) {
return cleanupMarkdownEscapes(md)
}
export const MAX_IMAGES_PER_MESSAGE = 20 // Anthropic limits to 20 images
// Anthropic limits to 20 images, which we use to constrain both images & files for simplicity
export const MAX_IMAGES_AND_FILES_PER_MESSAGE = 20
const QUICK_WINS_HISTORY_THRESHOLD = 300
const ChatView = ({ isHidden, showAnnouncement, hideAnnouncement, showHistoryView }: ChatViewProps) => {
@@ -120,6 +121,7 @@ const ChatView = ({ isHidden, showAnnouncement, hideAnnouncement, showHistoryVie
const textAreaRef = useRef<HTMLTextAreaElement>(null)
const [sendingDisabled, setSendingDisabled] = useState(false)
const [selectedImages, setSelectedImages] = useState<string[]>([])
const [selectedFiles, setSelectedFiles] = useState<string[]>([])
// we need to hold on to the ask because useEffect > lastMessage will always let us know when an ask comes in and handle it, but by the time handleMessage is called, the last message might not be the ask anymore (it could be a say that followed)
const [clineAsk, setClineAsk] = useState<ClineAsk | undefined>(undefined)
@@ -370,6 +372,7 @@ const ChatView = ({ isHidden, showAnnouncement, hideAnnouncement, showHistoryVie
setInputValue("")
setSendingDisabled(true)
setSelectedImages([])
setSelectedFiles([])
setClineAsk(undefined)
setEnableButtons(false)
}
@@ -442,9 +445,9 @@ const ChatView = ({ isHidden, showAnnouncement, hideAnnouncement, showHistoryVie
}, [modifiedMessages, clineAsk, enableButtons, primaryButtonText])
const handleSendMessage = useCallback(
async (text: string, images: string[]) => {
async (text: string, images: string[], files: string[]) => {
let messageToSend = text.trim()
const hasContent = messageToSend || images.length > 0
const hasContent = messageToSend || images.length > 0 || files.length > 0
// Prepend the active quote if it exists
if (activeQuote && hasContent) {
@@ -457,7 +460,7 @@ const ChatView = ({ isHidden, showAnnouncement, hideAnnouncement, showHistoryVie
if (hasContent) {
console.log("[ChatView] handleSendMessage - Sending message:", messageToSend)
if (messages.length === 0) {
await TaskServiceClient.newTask({ text: messageToSend, images })
await TaskServiceClient.newTask({ text: messageToSend, images, files })
} else if (clineAsk) {
switch (clineAsk) {
case "followup":
@@ -472,24 +475,13 @@ const ChatView = ({ isHidden, showAnnouncement, hideAnnouncement, showHistoryVie
case "resume_completed_task":
case "mistake_limit_reached":
case "new_task": // user can provide feedback or reject the new task suggestion
await TaskServiceClient.askResponse({
responseType: "messageResponse",
text: messageToSend,
images,
})
break
case "condense":
await TaskServiceClient.askResponse({
responseType: "messageResponse",
text: messageToSend,
images,
})
break
case "report_bug":
await TaskServiceClient.askResponse({
responseType: "messageResponse",
text: messageToSend,
images,
files,
})
break
// there is no other case that a textfield should be enabled
@@ -499,6 +491,7 @@ const ChatView = ({ isHidden, showAnnouncement, hideAnnouncement, showHistoryVie
setActiveQuote(null) // Clear quote when sending message
setSendingDisabled(true)
setSelectedImages([])
setSelectedFiles([])
setClineAsk(undefined)
setEnableButtons(false)
// setPrimaryButtonText(undefined)
@@ -518,7 +511,7 @@ const ChatView = ({ isHidden, showAnnouncement, hideAnnouncement, showHistoryVie
This logic depends on the useEffect[messages] above to set clineAsk, after which buttons are shown and we then send an askResponse to the extension.
*/
const handlePrimaryButtonClick = useCallback(
async (text?: string, images?: string[]) => {
async (text?: string, images?: string[], files?: string[]) => {
const trimmedInput = text?.trim()
switch (clineAsk) {
case "api_req_failed":
@@ -530,11 +523,12 @@ const ChatView = ({ isHidden, showAnnouncement, hideAnnouncement, showHistoryVie
case "resume_task":
case "mistake_limit_reached":
case "auto_approval_max_req_reached":
if (trimmedInput || (images && images.length > 0)) {
if (trimmedInput || (images && images.length > 0) || (files && files.length > 0)) {
await TaskServiceClient.askResponse({
responseType: "yesButtonClicked",
text: trimmedInput,
images: images,
files: files,
})
} else {
await TaskServiceClient.askResponse({
@@ -545,6 +539,7 @@ const ChatView = ({ isHidden, showAnnouncement, hideAnnouncement, showHistoryVie
setInputValue("")
setActiveQuote(null) // Clear quote when using primary button
setSelectedImages([])
setSelectedFiles([])
break
case "completion_result":
case "resume_completed_task":
@@ -556,6 +551,7 @@ const ChatView = ({ isHidden, showAnnouncement, hideAnnouncement, showHistoryVie
await TaskServiceClient.newTask({
text: lastMessage?.text,
images: [],
files: [],
})
break
case "condense":
@@ -576,7 +572,7 @@ const ChatView = ({ isHidden, showAnnouncement, hideAnnouncement, showHistoryVie
)
const handleSecondaryButtonClick = useCallback(
async (text?: string, images?: string[]) => {
async (text?: string, images?: string[], files?: string[]) => {
const trimmedInput = text?.trim()
if (isStreaming) {
await TaskServiceClient.cancelTask({})
@@ -594,11 +590,12 @@ const ChatView = ({ isHidden, showAnnouncement, hideAnnouncement, showHistoryVie
case "tool":
case "browser_action_launch":
case "use_mcp_server":
if (trimmedInput || (images && images.length > 0)) {
if (trimmedInput || (images && images.length > 0) || (files && files.length > 0)) {
await TaskServiceClient.askResponse({
responseType: "noButtonClicked",
text: trimmedInput,
images: images,
files: files,
})
} else {
// responds to the API with a "This operation failed" and lets it try again
@@ -610,6 +607,7 @@ const ChatView = ({ isHidden, showAnnouncement, hideAnnouncement, showHistoryVie
setInputValue("")
setActiveQuote(null) // Clear quote when using secondary button
setSelectedImages([])
setSelectedFiles([])
break
}
setSendingDisabled(true)
@@ -634,18 +632,40 @@ const ChatView = ({ isHidden, showAnnouncement, hideAnnouncement, showHistoryVie
return normalizeApiConfiguration(apiConfiguration)
}, [apiConfiguration])
const selectImages = useCallback(async () => {
const selectFilesAndImages = useCallback(async () => {
try {
const response = await FileServiceClient.selectImages({})
if (response && response.values && response.values.length > 0) {
setSelectedImages((prevImages) => [...prevImages, ...response.values].slice(0, MAX_IMAGES_PER_MESSAGE))
const response = await FileServiceClient.selectFiles({
value: selectedModelInfo.supportsImages,
})
if (
response &&
response.values1 &&
response.values2 &&
(response.values1.length > 0 || response.values2.length > 0)
) {
const currentTotal = selectedImages.length + selectedFiles.length
const availableSlots = MAX_IMAGES_AND_FILES_PER_MESSAGE - currentTotal
if (availableSlots > 0) {
// Prioritize images first
const imagesToAdd = Math.min(response.values1.length, availableSlots)
if (imagesToAdd > 0) {
setSelectedImages((prevImages) => [...prevImages, ...response.values1.slice(0, imagesToAdd)])
}
// Use remaining slots for files
const remainingSlots = availableSlots - imagesToAdd
if (remainingSlots > 0) {
setSelectedFiles((prevFiles) => [...prevFiles, ...response.values2.slice(0, remainingSlots)])
}
}
}
} catch (error) {
console.error("Error selecting images:", error)
console.error("Error selecting images & files:", error)
}
}, [])
}, [selectedModelInfo.supportsImages])
const shouldDisableImages = !selectedModelInfo.supportsImages || selectedImages.length >= MAX_IMAGES_PER_MESSAGE
const shouldDisableFilesAndImages = selectedImages.length + selectedFiles.length >= MAX_IMAGES_AND_FILES_PER_MESSAGE
const handleMessage = useCallback(
(e: MessageEvent) => {
@@ -667,12 +687,6 @@ const ChatView = ({ isHidden, showAnnouncement, hideAnnouncement, showHistoryVie
break
}
break
case "selectedImages":
const newImages = message.images ?? []
if (newImages.length > 0) {
setSelectedImages((prevImages) => [...prevImages, ...newImages].slice(0, MAX_IMAGES_PER_MESSAGE))
}
break
case "addToInput":
setInputValue((prevValue) => {
const newText = message.text ?? ""
@@ -1125,7 +1139,7 @@ const ChatView = ({ isHidden, showAnnouncement, hideAnnouncement, showHistoryVie
flex: secondaryButtonText ? 1 : 2,
marginRight: secondaryButtonText ? "6px" : "0",
}}
onClick={() => handlePrimaryButtonClick(inputValue, selectedImages)}>
onClick={() => handlePrimaryButtonClick(inputValue, selectedImages, selectedFiles)}>
{primaryButtonText}
</VSCodeButton>
)}
@@ -1137,7 +1151,7 @@ const ChatView = ({ isHidden, showAnnouncement, hideAnnouncement, showHistoryVie
flex: isStreaming ? 2 : 1,
marginLeft: isStreaming ? 0 : "6px",
}}
onClick={() => handleSecondaryButtonClick(inputValue, selectedImages)}>
onClick={() => handleSecondaryButtonClick(inputValue, selectedImages, selectedFiles)}>
{isStreaming ? "Cancel" : secondaryButtonText}
</VSCodeButton>
)}
@@ -1167,9 +1181,11 @@ const ChatView = ({ isHidden, showAnnouncement, hideAnnouncement, showHistoryVie
placeholderText={placeholderText}
selectedImages={selectedImages}
setSelectedImages={setSelectedImages}
onSend={() => handleSendMessage(inputValue, selectedImages)}
onSelectImages={selectImages}
shouldDisableImages={shouldDisableImages}
setSelectedFiles={setSelectedFiles}
selectedFiles={selectedFiles}
onSend={() => handleSendMessage(inputValue, selectedImages, selectedFiles)}
onSelectFilesAndImages={selectFilesAndImages}
shouldDisableFilesAndImages={shouldDisableFilesAndImages}
onHeightChange={() => {
if (isAtBottom) {
scrollToBottomAuto()
@@ -352,7 +352,9 @@ const TaskHeader: React.FC<TaskHeaderProps> = ({
See less
</div>
)}
{task.images && task.images.length > 0 && <Thumbnails images={task.images} />}
{((task.images && task.images.length > 0) || (task.files && task.files.length > 0)) && (
<Thumbnails images={task.images ?? []} files={task.files ?? []} />
)}
<div
style={{
@@ -8,12 +8,13 @@ import { ClineCheckpointRestore } from "@shared/WebviewMessage"
interface UserMessageProps {
text?: string
files?: string[]
images?: string[]
messageTs?: number // Timestamp for the message, needed for checkpoint restore
sendMessageFromChatRow?: (text: string, images: string[]) => void
sendMessageFromChatRow?: (text: string, images: string[], files: string[]) => void
}
const UserMessage: React.FC<UserMessageProps> = ({ text, images, messageTs, sendMessageFromChatRow }) => {
const UserMessage: React.FC<UserMessageProps> = ({ text, images, files, messageTs, sendMessageFromChatRow }) => {
const [isEditing, setIsEditing] = useState(false)
const [editedText, setEditedText] = useState(text || "")
const textAreaRef = useRef<HTMLTextAreaElement>(null)
@@ -52,7 +53,7 @@ const UserMessage: React.FC<UserMessageProps> = ({ text, images, messageTs, send
})
setTimeout(() => {
sendMessageFromChatRow?.(editedText, images || [])
sendMessageFromChatRow?.(editedText, images || [], files || [])
}, delay)
} catch (err) {
console.error("Checkpoint restore error:", err)
@@ -145,7 +146,9 @@ const UserMessage: React.FC<UserMessageProps> = ({ text, images, messageTs, send
{highlightText(editedText || text)}
</span>
)}
{images && images.length > 0 && <Thumbnails images={images} style={{ marginTop: "8px" }} />}
{((images && images.length > 0) || (files && files.length > 0)) && (
<Thumbnails images={images ?? []} files={files ?? []} style={{ marginTop: "8px" }} />
)}
</div>
)
}
+93 -10
View File
@@ -5,13 +5,15 @@ import { vscode } from "@/utils/vscode"
interface ThumbnailsProps {
images: string[]
files: string[]
style?: React.CSSProperties
setImages?: React.Dispatch<React.SetStateAction<string[]>>
setFiles?: React.Dispatch<React.SetStateAction<string[]>>
onHeightChange?: (height: number) => void
}
const Thumbnails = ({ images, style, setImages, onHeightChange }: ThumbnailsProps) => {
const [hoveredIndex, setHoveredIndex] = useState<number | null>(null)
const Thumbnails = ({ images, files, style, setImages, setFiles, onHeightChange }: ThumbnailsProps) => {
const [hoveredIndex, setHoveredIndex] = useState<string | null>(null)
const containerRef = useRef<HTMLDivElement>(null)
const { width } = useWindowSize()
@@ -25,18 +27,27 @@ const Thumbnails = ({ images, style, setImages, onHeightChange }: ThumbnailsProp
onHeightChange?.(height)
}
setHoveredIndex(null)
}, [images, width, onHeightChange])
}, [images, files, width, onHeightChange])
const handleDelete = (index: number) => {
const handleDeleteImages = (index: number) => {
setImages?.((prevImages) => prevImages.filter((_, i) => i !== index))
}
const isDeletable = setImages !== undefined
const handleDeleteFiles = (index: number) => {
setFiles?.((prevFiles) => prevFiles.filter((_, i) => i !== index))
}
const isDeletableImages = setImages !== undefined
const isDeletableFiles = setFiles !== undefined
const handleImageClick = (image: string) => {
FileServiceClient.openImage({ value: image }).catch((err) => console.error("Failed to open image:", err))
}
const handleFileClick = (filePath: string) => {
FileServiceClient.openFile({ value: filePath }).catch((err) => console.error("Failed to open file:", err))
}
return (
<div
ref={containerRef}
@@ -49,13 +60,13 @@ const Thumbnails = ({ images, style, setImages, onHeightChange }: ThumbnailsProp
}}>
{images.map((image, index) => (
<div
key={index}
key={`image-${index}`}
style={{ position: "relative" }}
onMouseEnter={() => setHoveredIndex(index)}
onMouseEnter={() => setHoveredIndex(`image-${index}`)}
onMouseLeave={() => setHoveredIndex(null)}>
<img
src={image}
alt={`Thumbnail ${index + 1}`}
alt={`Thumbnail image-${index + 1}`}
style={{
width: 34,
height: 34,
@@ -65,9 +76,9 @@ const Thumbnails = ({ images, style, setImages, onHeightChange }: ThumbnailsProp
}}
onClick={() => handleImageClick(image)}
/>
{isDeletable && hoveredIndex === index && (
{isDeletableImages && hoveredIndex === `image-${index}` && (
<div
onClick={() => handleDelete(index)}
onClick={() => handleDeleteImages(index)}
style={{
position: "absolute",
top: -4,
@@ -92,6 +103,78 @@ const Thumbnails = ({ images, style, setImages, onHeightChange }: ThumbnailsProp
)}
</div>
))}
{files.map((filePath, index) => {
const fileName = filePath.split(/[\\/]/).pop() || filePath
return (
<div
key={`file-${index}`}
style={{ position: "relative" }}
onMouseEnter={() => setHoveredIndex(`file-${index}`)}
onMouseLeave={() => setHoveredIndex(null)}>
<div
style={{
width: 34,
height: 34,
borderRadius: 4,
cursor: "pointer",
backgroundColor: "var(--vscode-editor-background)",
border: "1px solid var(--vscode-input-border)",
display: "flex",
flexDirection: "column",
alignItems: "center",
justifyContent: "center",
}}
onClick={() => handleFileClick(filePath)}>
<span
className="codicon codicon-file"
style={{
fontSize: 16,
color: "var(--vscode-foreground)",
}}></span>
<span
style={{
fontSize: 7,
marginTop: 1,
overflow: "hidden",
textOverflow: "ellipsis",
maxWidth: "90%",
whiteSpace: "nowrap",
textAlign: "center",
}}
title={fileName}>
{fileName}
</span>
</div>
{isDeletableFiles && hoveredIndex === `file-${index}` && (
<div
onClick={() => handleDeleteFiles(index)}
style={{
position: "absolute",
top: -4,
right: -4,
width: 13,
height: 13,
borderRadius: "50%",
backgroundColor: "var(--vscode-badge-background)",
display: "flex",
justifyContent: "center",
alignItems: "center",
cursor: "pointer",
}}>
<span
className="codicon codicon-close"
style={{
color: "var(--vscode-foreground)",
fontSize: 10,
fontWeight: "bold",
}}></span>
</div>
)}
</div>
)
})}
</div>
)
}