dim check change

Co-authored-by: Ding Fei <fding@feysh.com>

src/core/task/ToolExecutor.ts

@@ -32,6 +32,7 @@ import {
 	COMPLETION_RESULT_CHANGES_FLAG,
 } from "@shared/ExtensionMessage"
 import { ClineAskResponse } from "@shared/WebviewMessage"
+import { extractFileContent, FileContentResult } from "@integrations/misc/extract-file-content"
 import { COMMAND_REQ_APP_STRING } from "@shared/combineCommandSequences"
 import { fileExistsAtPath } from "@utils/fs"
 import { isClaude4ModelFamily, isGemini2dot5ModelFamily } from "@utils/model-utils"
@@ -839,12 +840,18 @@ export class ToolExecutor {
 							telemetryService.captureToolUsage(this.taskId, block.name, this.api.getModel().id, false, true)
 						}
 						// now execute the tool like normal
-						const content = await extractTextFromFile(absolutePath)
+						const supportsImages = this.api.getModel().info.supportsImages ?? false
+						const result = await extractFileContent(absolutePath, supportsImages)
 
 						// Track file read operation
 						await this.fileContextTracker.trackFileContext(relPath, "read_tool")
 
-						this.pushToolResult(content, block)
+						this.pushToolResult(result.text, block)
+
+						if (result.imageBlock) {
+							this.taskState.userMessageContent.push(result.imageBlock)
+						}
+
 						await this.saveCheckpoint()
 						break
 					}

src/integrations/misc/extract-file-content.ts

@@ -0,0 +1,53 @@
+import * as path from "path"
+import fs from "fs/promises"
+import { Anthropic } from "@anthropic-ai/sdk"
+import { callTextExtractionFunctions } from "./extract-text"
+import { extractImageContent } from "./extract-images"
+
+export type FileContentResult = {
+	text: string
+	imageBlock?: Anthropic.ImageBlockParam
+}
+
+/**
+ * Extract content from a file, handling both text and images
+ * Extra logic for handling images based on whether the model supports images
+ */
+export async function extractFileContent(absolutePath: string, modelSupportsImages: boolean): Promise<FileContentResult> {
+	// Check if file exists first
+	try {
+		await fs.access(absolutePath)
+	} catch (error) {
+		throw new Error(`File not found: ${absolutePath}`)
+	}
+
+	const fileExtension = path.extname(absolutePath).toLowerCase()
+	const imageExtensions = [".png", ".jpg", ".jpeg", ".webp"]
+	const isImage = imageExtensions.includes(fileExtension)
+
+	if (isImage && modelSupportsImages) {
+		const imageResult = await extractImageContent(absolutePath)
+
+		if (imageResult.success) {
+			return {
+				text: "Successfully read image",
+				imageBlock: imageResult.imageBlock,
+			}
+		} else {
+			throw new Error(imageResult.error)
+		}
+	} else if (isImage && !modelSupportsImages) {
+		throw new Error(`Current model does not support image input`)
+	} else {
+		// Handle text files using existing extraction functions
+		try {
+			const textContent = await callTextExtractionFunctions(absolutePath)
+			return {
+				text: textContent,
+			}
+		} catch (error) {
+			const errorMessage = error instanceof Error ? error.message : "Unknown error"
+			throw new Error(`Error reading file: ${errorMessage}`)
+		}
+	}
+}

src/integrations/misc/extract-images.ts

@@ -0,0 +1,54 @@
+import fs from "fs/promises"
+import * as path from "path"
+import sizeOf from "image-size"
+import { Anthropic } from "@anthropic-ai/sdk"
+import { getMimeType } from "./process-files"
+
+/**
+ * Extract image content without VSCode dependencies
+ * Returns success/error result to avoid throwing exceptions
+ */
+export async function extractImageContent(
+	filePath: string,
+): Promise<{ success: true; imageBlock: Anthropic.ImageBlockParam } | { success: false; error: string }> {
+	try {
+		// Read the file into a buffer
+		const buffer = await fs.readFile(filePath)
+
+		// Convert Node.js Buffer to Uint8Array for image-size
+		const uint8Array = new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength)
+
+		// Get dimensions from Uint8Array
+		const dimensions = sizeOf(uint8Array)
+
+		if (!dimensions.width || !dimensions.height) {
+			return { success: false, error: "Could not determine image dimensions, so image could not be read" }
+		}
+
+		if (dimensions.width > 7500 || dimensions.height > 7500) {
+			return {
+				success: false,
+				error: "Image dimensions exceed 7500px by 7500px, so image could not be read",
+			}
+		}
+
+		// Convert buffer to base64
+		const base64 = buffer.toString("base64")
+		const mimeType = getMimeType(filePath) as "image/jpeg" | "image/png" | "image/webp"
+
+		// Create the image block in Anthropic format
+		const imageBlock: Anthropic.ImageBlockParam = {
+			type: "image",
+			source: {
+				type: "base64",
+				media_type: mimeType,
+				data: base64,
+			},
+		}
+
+		return { success: true, imageBlock }
+	} catch (error) {
+		const errorMessage = error instanceof Error ? error.message : "Unknown error"
+		return { success: false, error: `Error reading image: ${errorMessage}` }
+	}
+}

src/integrations/misc/extract-text.ts

@@ -31,7 +31,16 @@ export async function extractTextFromFile(filePath: string): Promise<string> {
 	} catch (error) {
 		throw new Error(`File not found: ${filePath}`)
 	}
+
+	return callTextExtractionFunctions(filePath)
+}
+
+/**
+ * Expects the fs.access call to have already been performed prior to calling
+ */
+export async function callTextExtractionFunctions(filePath: string): Promise<string> {
 	const fileExtension = path.extname(filePath).toLowerCase()
+
 	switch (fileExtension) {
 		case ".pdf":
 			return extractTextFromPDF(filePath)

src/integrations/misc/process-files.ts

@@ -44,7 +44,7 @@ export async function selectFiles(imagesAllowed: boolean): Promise<{ images: str
 				// Convert Node.js Buffer to Uint8Array
 				const uint8Array = new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength)
 				const dimensions = sizeOf(uint8Array) // Get dimensions from Uint8Array
-				if (dimensions.width! > 7500 || dimensions.height! > 7500) {
+				if (dimensions.width! > 7680 || dimensions.height! > 7680) {
 					console.warn(`Image dimensions exceed 7500px, skipping: ${filePath}`)
 					HostProvider.window.showMessage({
 						type: ShowMessageType.ERROR,
@@ -107,7 +107,7 @@ export async function selectFiles(imagesAllowed: boolean): Promise<{ images: str
 	return { images, files }
 }
 
-function getMimeType(filePath: string): string {
+export function getMimeType(filePath: string): string {
 	const ext = path.extname(filePath).toLowerCase()
 	switch (ext) {
 		case ".png":

webview-ui/src/components/chat/ChatRow.tsx

@@ -385,6 +385,13 @@ export const ChatRowContent = memo(
 			return null
 		}, [message.ask, message.say, message.text])
 
+		// Helper function to check if file is an image
+		const isImageFile = (filePath: string): boolean => {
+			const imageExtensions = [".png", ".jpg", ".jpeg", ".webp"]
+			const extension = filePath.toLowerCase().split(".").pop()
+			return extension ? imageExtensions.includes(`.${extension}`) : false
+		}
+
 		if (tool) {
 			const colorMap = {
 				red: "var(--vscode-errorForeground)",
@@ -440,10 +447,11 @@ export const ChatRowContent = memo(
 						</>
 					)
 				case "readFile":
+					const isImage = isImageFile(tool.path || "")
 					return (
 						<>
 							<div style={headerStyle}>
-								{toolIcon("file-code")}
+								{toolIcon(isImage ? "file-media" : "file-code")}
 								{tool.operationIsLocatedInWorkspace === false &&
 									toolIcon("sign-out", "yellow", -90, "This file is outside of your workspace")}
 								<span style={{ fontWeight: "bold" }}>
@@ -464,17 +472,21 @@ export const ChatRowContent = memo(
 										display: "flex",
 										alignItems: "center",
 										padding: "9px 10px",
-										cursor: "pointer",
-										userSelect: "none",
-										WebkitUserSelect: "none",
-										MozUserSelect: "none",
-										msUserSelect: "none",
+										cursor: isImage ? "default" : "pointer",
+										userSelect: isImage ? "text" : "none",
+										WebkitUserSelect: isImage ? "text" : "none",
+										MozUserSelect: isImage ? "text" : "none",
+										msUserSelect: isImage ? "text" : "none",
 									}}
-									onClick={() => {
-										FileServiceClient.openFile(StringRequest.create({ value: tool.content })).catch((err) =>
-											console.error("Failed to open file:", err),
-										)
-									}}>
+									onClick={
+										isImage
+											? undefined
+											: () => {
+													FileServiceClient.openFile(
+														StringRequest.create({ value: tool.content }),
+													).catch((err) => console.error("Failed to open file:", err))
+												}
+									}>
 									{tool.path?.startsWith(".") && <span>.</span>}
 									<span
 										className="ph-no-capture"
@@ -489,12 +501,14 @@ export const ChatRowContent = memo(
 										{cleanPathPrefix(tool.path ?? "") + "\u200E"}
 									</span>
 									<div style={{ flexGrow: 1 }}></div>
-									<span
-										className={`codicon codicon-link-external`}
-										style={{
-											fontSize: 13.5,
-											margin: "1px 0",
-										}}></span>
+									{!isImage && (
+										<span
+											className={`codicon codicon-link-external`}
+											style={{
+												fontSize: 13.5,
+												margin: "1px 0",
+											}}></span>
+									)}
 								</div>
 							</div>
 						</>