diff --git a/site/src/components/ai-elements/tool.stories.tsx b/site/src/components/ai-elements/tool.stories.tsx index 85e581cbc6..d92288198b 100644 --- a/site/src/components/ai-elements/tool.stories.tsx +++ b/site/src/components/ai-elements/tool.stories.tsx @@ -643,3 +643,112 @@ export const EditFilesError: Story = { result: { error: "File not found" }, }, }; + +// --------------------------------------------------------------------------- +// Computer tool stories +// --------------------------------------------------------------------------- + +import { DESKTOP_SCREENSHOT_BASE64 } from "./tool/__fixtures__/desktopScreenshot"; + +export const ComputerScreenshot: Story = { + args: { + name: "computer", + status: "completed", + result: { + data: DESKTOP_SCREENSHOT_BASE64, + text: "", + mime_type: "image/jpeg", + }, + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + expect(canvas.getByText("Screenshot")).toBeInTheDocument(); + const img = canvas.getByRole("img", { + name: "Screenshot from computer tool", + }); + expect(img).toBeInTheDocument(); + expect(img.getAttribute("src")).toContain("data:image/jpeg;base64,"); + // Image should be wrapped in a link that opens in a new tab. + const link = img.closest("a"); + expect(link).toHaveAttribute("target", "_blank"); + }, +}; + +export const ComputerRunning: Story = { + args: { + name: "computer", + status: "running", + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + expect(canvas.getByText("Taking screenshot…")).toBeInTheDocument(); + expect(canvasElement.querySelector(".animate-spin")).not.toBeNull(); + }, +}; + +export const ComputerTextFallback: Story = { + args: { + name: "computer", + status: "completed", + result: { + data: "", + text: "Screen resolution: 1920x1080\nActive window: Terminal", + mime_type: "image/png", + }, + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + // Text-only results are collapsed by default (no image). + const toggle = canvas.getByRole("button", { name: /Screenshot/ }); + expect(toggle).toBeInTheDocument(); + expect(canvas.queryByRole("img")).toBeNull(); + + await userEvent.click(toggle); + expect( + canvas.getByText(/Screen resolution: 1920x1080/), + ).toBeInTheDocument(); + }, +}; + +export const ComputerError: Story = { + args: { + name: "computer", + status: "error", + isError: true, + result: { + data: "", + text: "", + mime_type: "image/png", + }, + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + expect(canvas.getByText("Screenshot")).toBeInTheDocument(); + // Icon and label should have the destructive color class. + const label = canvas.getByText("Screenshot"); + expect(label.className).toContain("text-content-destructive"); + }, +}; + +export const ComputerArrayResult: Story = { + args: { + name: "computer", + status: "completed", + result: [ + { + type: "image", + data: DESKTOP_SCREENSHOT_BASE64, + mime_type: "image/jpeg", + }, + { type: "text", text: "Clicked on button" }, + ], + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + const img = canvas.getByRole("img", { + name: "Screenshot from computer tool", + }); + expect(img).toBeInTheDocument(); + expect(img.getAttribute("src")).toContain("data:image/jpeg;base64,"); + }, +}; diff --git a/site/src/components/ai-elements/tool/ComputerTool.tsx b/site/src/components/ai-elements/tool/ComputerTool.tsx new file mode 100644 index 0000000000..7a4f1432bf --- /dev/null +++ b/site/src/components/ai-elements/tool/ComputerTool.tsx @@ -0,0 +1,85 @@ +import { + Tooltip, + TooltipContent, + TooltipTrigger, +} from "components/Tooltip/Tooltip"; +import { CircleAlertIcon, LoaderIcon } from "lucide-react"; +import type React from "react"; +import { cn } from "utils/cn"; +import { ToolCollapsible } from "./ToolCollapsible"; +import type { ToolStatus } from "./utils"; + +/** + * Renders screenshots returned by Anthropic's computer use tool. + * When the result contains base64 image data, the actual image is + * displayed instead of raw JSON. The image is clickable and opens + * in a new tab at full resolution. + */ +export const ComputerTool: React.FC<{ + imageData: string; + mimeType: string; + text: string; + status: ToolStatus; + isError: boolean; + errorMessage?: string; +}> = ({ imageData, mimeType, text, status, isError, errorMessage }) => { + const isRunning = status === "running"; + const hasImage = imageData.length > 0; + const hasText = text.length > 0; + const hasContent = hasImage || hasText; + + return ( + + + {isRunning ? "Taking screenshot…" : "Screenshot"} + + {isError && ( + + + + + + {errorMessage || "Failed to take screenshot"} + + + )} + {isRunning && ( + + )} + + } + > + {hasImage ? ( +
+ + Screenshot from computer tool + +
+ ) : hasText ? ( +
+
+						{text}
+					
+
+ ) : null} +
+ ); +}; diff --git a/site/src/components/ai-elements/tool/Tool.tsx b/site/src/components/ai-elements/tool/Tool.tsx index a4da9f5de1..36dbc5e6ed 100644 --- a/site/src/components/ai-elements/tool/Tool.tsx +++ b/site/src/components/ai-elements/tool/Tool.tsx @@ -5,6 +5,7 @@ import type { ComponentPropsWithRef, FC } from "react"; import { memo } from "react"; import { cn } from "utils/cn"; import { ChatSummarizedTool } from "./ChatSummarizedTool"; +import { ComputerTool } from "./ComputerTool"; import { CreateWorkspaceTool } from "./CreateWorkspaceTool"; import { EditFilesTool } from "./EditFilesTool"; import { @@ -366,6 +367,53 @@ const ChatSummarizedRenderer: FC = ({ ); }; +const ComputerRenderer: FC = ({ + status, + result, + isError, +}) => { + // The result can be a single object with {data, text, mime_type} + // or an array of content blocks. + let imageData = ""; + let mimeType = "image/png"; + let text = ""; + + if (Array.isArray(result)) { + for (const block of result) { + const blockRec = asRecord(block); + if (blockRec) { + if (blockRec.type === "image" || asString(blockRec.data)) { + imageData = asString(blockRec.data); + mimeType = asString(blockRec.mime_type) || "image/png"; + } + if ( + blockRec.type === "text" || + (!imageData && asString(blockRec.text)) + ) { + text = asString(blockRec.text); + } + } + } + } else { + const rec = asRecord(result); + if (rec) { + imageData = asString(rec.data); + mimeType = asString(rec.mime_type) || "image/png"; + text = asString(rec.text); + } + } + + return ( + + ); +}; + // Generic fallback renderer — only path that needs theme, diff // viewers, and file content helpers. const GenericToolRenderer: FC = ({ @@ -461,6 +509,7 @@ const toolRenderers: Record> = { message_agent: SubagentRenderer, close_agent: SubagentRenderer, chat_summarized: ChatSummarizedRenderer, + computer: ComputerRenderer, }; // --------------------------------------------------------------------------- diff --git a/site/src/components/ai-elements/tool/ToolIcon.tsx b/site/src/components/ai-elements/tool/ToolIcon.tsx index 9628a0c169..943404336b 100644 --- a/site/src/components/ai-elements/tool/ToolIcon.tsx +++ b/site/src/components/ai-elements/tool/ToolIcon.tsx @@ -2,6 +2,7 @@ import { BotIcon, FileIcon, FilePenIcon, + MonitorIcon, PlusCircleIcon, TerminalIcon, WrenchIcon, @@ -30,6 +31,8 @@ export const ToolIcon: React.FC<{ name: string; isError: boolean }> = ({ return ; case "chat_summarized": return ; + case "computer": + return ; default: return ; } diff --git a/site/src/components/ai-elements/tool/ToolLabel.tsx b/site/src/components/ai-elements/tool/ToolLabel.tsx index e0a9b9d865..e98b1703ec 100644 --- a/site/src/components/ai-elements/tool/ToolLabel.tsx +++ b/site/src/components/ai-elements/tool/ToolLabel.tsx @@ -154,6 +154,12 @@ export const ToolLabel: React.FC<{ Summarized ); + case "computer": + return ( + + Screenshot + + ); default: return ( {name} diff --git a/site/src/components/ai-elements/tool/__fixtures__/desktopScreenshot.ts b/site/src/components/ai-elements/tool/__fixtures__/desktopScreenshot.ts new file mode 100644 index 0000000000..9cd90b53b4 --- /dev/null +++ b/site/src/components/ai-elements/tool/__fixtures__/desktopScreenshot.ts @@ -0,0 +1,4 @@ +// Auto-generated 400x225 JPEG screenshot used as a realistic +// test fixture for ComputerTool storybook stories. +export const DESKTOP_SCREENSHOT_BASE64 = + "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDABALDA4MChAODQ4SERATGCgaGBYWGDEjJR0oOjM9PDkzODdASFxOQERXRTc4UG1RV19iZ2hnPk1xeXBkeFxlZ2P/2wBDARESEhgVGC8aGi9jQjhCY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2P/wAARCADhAZADASIAAhEBAxEB/8QAGwABAAIDAQEAAAAAAAAAAAAAAAEDBAYHAgX/xABAEAACAQIDBAcGBAUDAwUAAAAAAQIDEQQSIQUTMZEWQVFSVWHRBiIzdKGyFDJxgQcVIzRzJULwJDWxQ2JyweH/xAAXAQEBAQEAAAAAAAAAAAAAAAAAAQID/8QAGxEBAQEBAQEBAQAAAAAAAAAAAAEREjEhAkH/2gAMAwEAAhEDEQA/ANz0jGH9OU8zt7vV5vyFWUKFGtVlFyVNXsrv/wAF8aDS/O+R7pwUE9bt8Wbv6ZkYeHrKrhY1ZYbLJyytKV1+t11B4mmo5nhKyWmuXrM4Gdq4wViIOeVYStronlEKyk1F4SpGWa3B248bmc2kRm8nyG0yMLfpNqeDrJXdmlfRX1+hO/hnUXhK/FK+XTUzM6XG5G8iNp8RuafdRVFQnUq091KOS1pvhK/YXbyIzxG0yPnzqZJQjklLO7XS0WnWWFzpQvpJpfoRuY998jr1GMqoFu5j33yG5j33yHUMqoFu5j33yG5j33yHUMqoFu5j33yG5j33yHUMqoFu5j33yG5j33yHUMqoFu5j33yG5j33yHUMqoFu5j33yG5j33yHUMqoFu5j33yG5j33yHUMqoFu5j33yG5j33yHUMqoFu5j33yG5j33yHUMqoFu5j33yG5j33yHUMqoFu5j33yG5j33yHUMqoFu5j33yG5j33yHUMqoFu5j33yG5j33yHUMqoFu5j33yG5j33yHUMqoFu5j33yG5j33yHUMqoFu5j33yG5j33yHUMqoFu5j33yG5j33yHUMqiUlCLlJ2S6xmWTOtVa+nWZCoxb0m7/oT+GXffIdQysLDVnXoqo6coXeilxPVOq51KkMko5Ha74S/QzPwy775EfhtNJfQnUXKtnFytabjZ306z11cTkfSPbPiWI5r0HSPbPiWI5r0MctutQi4xs5OWvWUxw8li3WdS6s7K767ft1fU5X0j2z4liOa9B0j2z4liOa9ByOr4mDqUpRjKza4/uULDPdQi6804pr3eGrv/8Ahy/pHtnxLEc16DpHtnxLEc16DB1RQnGhk3spz19+SMX8JiIxahi6nle70Oa9I9s+JYjmvQdI9s+JYjmvQYjpX4XEvN/1c1fhZdZ6eFr5rrFzXll6jmfSPbPiWI5r0HSPbPiWI5r0GDpf4Su4RTxc3JK2a1r631LKNCpTm5TrTqaWtI5h0j2z4liOa9B0j2z4liOa9Bg6tYWOVL2i2zf/ALliOa9C/D+0O2XF22lLR/8AqMuUdOsLM5m/aHbMZQi9o3zy45tF+vYeMR7SbZjLJ/MJOyTvBk+jp9hY5Z0m2y1H/UK2l+tajpNtnM3/ADCtr1aWH0dTsLHK5e0m2ZSb/mNdXd7JqyI6R7Z8SxHNehcHVbCxyrpHtnxLEc16DpHtnxLEc16DB1Wwscq6R7Z8SxHNeg6R7Z8SxHNegwdVsLHKeke2fEsRzXoOke2fEsRzXoMHVrCxynpHtnxLEc16DpHtnxLEc16DB1awscp6R7Z8SxHNeg6R7Z8SxHNegwdWsLHKeke2fEsRzXoOke2fEsRzXoMHVrCxynpHtnxLEc16DpHtnxLEc16DB1awscp6R7Z8SxHNehPSLbNl/qWI5r0GDqthY5T0j2z4liOa9B0j2z4liOa9Bg6tYWOU9I9s+JYjmvQdI9s+JYjmvQYOrWFjlPSPbPiWI5r0HSPbPiWI5r0GDq1hY5T0j2z4liOa9B0j2z4liOa9Bg6tY8VoSnRnCEnCTVlJdRyzpHtnxLEc16DpHtnxLEc16DB1LC0ZUqUYSm5OPW+zqWox+HniaSjTmk0768DlvSPbPiWI5r0HSPbPiWI5r0E2XTHWaEHTowhKWZpWbJpRcItO2sm9PN3OS9I9s+JYjmvQ3D2H2hjsfh8VLE4iVdwqxV6j4Kzvaxmz+q54ADYAAAAAAAAAAAAAJjxL8PBypNrDxq+8ldu37FEeJ7oqLlaUZyXZFhF06bVSC/CKOv5c2r8iucGo1P6CVktU/wAvD/n7nmajq1Cp+bRt/wDNSJ2T0jKOi4sD1Gzw8rU03Fq8r9rLp03u5NYRRVn72a9uJjxS3c7qT1VmuCLZRp5HlpV79V3p1ge6VNypq2EUtF7zkesJTnKk2sJTqLM1mlK3ZoVQjSyK9Ou3bq4DD080pOVKpKPBZe24F8aU3VrL8FTdlG8FL8unUU04Sc6qWHjJr/bf8uj/AOfseowpOVR/h67ikvyy1j+v6lcIwzVE6VVpcEnqtHx/8gWRpy3FN/h4vNa082r4leKg41rOiqby/li7/uelCG6pvc1ru15J6S/QicYLExjuqsYu3uN+8wMcGTKnTdOWSjWUrK0pcEF+FSeeFVS7EwMYHuq6bkt0pJf+56ngKAAAAAAAAEvgiCXwQEAAAAAAAAAAAAAAAAG+fw4/tcd/kh9poZvn8OP7XHf5IfaS+DQwAUAAAAAAAAAAAAAEx4llFxirurOEr6ZV9SuPE90nRSe9hOTv/tlYI9TcXL485Xd22vqeZOLzf1JPRWuuJ7U8Nl1pTvfvdRE54dpZKU4vtcrjR4i0qc1mabtoloy2pVzQaeJqTv1NaMswFbBUsTTliqDnTUZKUfzXbXu/UwxoyIVbU0vxNRaflSIpSp7u061SPvXUYr6lAGjJhUhGc3HEVY6JRklx06zxCaUqn9aau+KWstHxKQNF6qWpQW+mmre71LiTKpF18yr1HZK02teJjgDIq1rx93EVZu3BqxRKTk25O7fWQAAACgAAAAAAABL4Igl8EBAAAAAAAAAAAAAAAABvn8OP7XHf5IfaaGb5/Dj+1x3+SH2kvg0MAFAAAAAAAAAAAAABMeJBMeJAAAAAAAAAAAAAAAAAAAAAAAAAAAACXwRBL4ICAAAAAAAAAAAAAAAADfP4cf2uO/yQ+00M3b+H+Jp4fDYpVG1vK8IRsr65SXwaSACgAAAAAAAAAAAAAmPEgmPEgAAAAAAAAAAAAAAAAAAAAAAAAAAABL4Igl8EBAAAAAAAAAAAAAAAABtXsh8B/OU/tNVNq9kPgP5yn9pL4NVABQAAAAAAAAAAAAATHiQTHiQAAAAAAAAAAAAAAAAAAAAAAAAAAAAl8EQS+CAgAAAAAAAAAAAAAAAA2r2Q+A/nKf2mqm1eyHwH85T+0l8GqgAoAAAAAAAAAAAAAJXE85kSFeKspafoBGZDMj1ml3voecq7foRDMhmQyrt+gyrt+gUzIZkMq7foMq7foAzIZkMq7foMq7foAzIZkMq7foMq7foAzIZkMq7foMq7foAzIZkMq7foMq7foAzIZkMq7foMq7foAzIZkMq7foMq7foAzIZkTG8eEuPkNbWzdd+ARF/1JvdIm7197j5EcFYqgAAAAAAAAAAAAAAABtXsh8B/OU/tNVNq9kPgP5yn9pL4NVABQAAAAAAAAAAAAlWTV1dX1XaBAPuVMVs6tjJTq1Kc6FnuacqDiqOq0ll1el+t66kU6+w1kjLDpxzK7kpZrOcr3s+qOX9xiPiA+5L+T0qeGjUjBylCDk4KTavB3ctbPXK7LXRiOI2QoRoyhCVKNWnKUlTkpSWRKVn1e91dgwfDC14H1q+KwNGM54GlTjX3kcsrSeWOXVq/n2oycZjdlYqVedWLm88lDLBxllusuW2iVs176gfAB9fG4jZywmIpYNJSqZPywkou02+vVPLa/azApVoUMRRq4fPBxtnc7S167acLAY614A+9i8ZsvFbybUXWdZ5ZThJJQz+7olwy3uuJRi8Rs14XFUsJTpxUpU5Us9OWdJJqSv28H2WA+RdXtfUH2YY/BToYLDVordU6V6ksr1n71k1bzWpc8ds2lGdOg2qeaeROD0TnTa4//GQHwAbBKvsKrjp1KqTjJtyy05pO83r2qSj+xThcRsqjKGaP9KWHlCpFQlvHJ2vd8Ldlhg+KCyvl39TduLhmeVwTSt5J6orAAAKAAAAAAAAAAAAAAAAAAAAAAAAG1eyHwH85T+01U2r2Q+A/nKf2kvg1UAFAAAAAAAAAAAACY2zLM7K+rAlwkna2oyT7rLqkKcZwWe2Z6tTzadTCp09V+JS6+ARTu5LTKMkuxl+6o5tcTdLyPFSEIwvGtnlfggK93LukZJWvbQZn2vmLvtYE7ufdZDi48VYKTXBshtviwAACgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAG1eyHwH85T+01U2r2Q+A/nKf2kvg1UAFAAAAAAAAAAAAABK4lkaUpQUk4Wfb1Fa4lsJxjCKdGLcXe76whLDzjBzbp2tfSSEaE5RzLd89eFyM8cjW5jdq2a+qJU4JJfh4vt14gS8PPXWGnH6+hEqE4q7ycyZ1ITTth4xb60zzGaUFGVKMmuDYE0qMqscycV5Nfp6nr8NO6Wam0+tMjeQvf8NC3Zcrm80rqCWlrAWVcPKlDM5U2r9RTfyXIuVSCik8PFtJK9yqfvTclFRu+C6gIv5LkL+S5Cz7BZ9gC/kuQv5LkLPsFn2AL+S5C/kuQs+wWfYAv5LkL+S5Cz7BZ9gC/kuQv5LkLPsFn2AL+S5C/kuQs+wWfYAv5LkL+S5Cz7BZ9gC/kuQ6hZ9g4LXtCoAAAAAAAAAAAAADavZD4D+cp/aaqbV7IfAfzlP7SXwaqACgAAAAAAAAAAABKtdX0V9bAQD7k8JgKuMlBPDww6TdF0q6Uqqul7zlezs2+C4WFPAbIeSMsXNtySzqpFKznKN7NaWST/cYj4YPtvZeAhTw+9xDpzqwhL3qsdc0G+FvdV1HV9pMcDsvJGlOtFS3tNTrKsnli4Juy61muvIYPhg+riKGzsJGdSm3XmqkYxpzmnFJxu27cVfTRoycXhNlVJV5QrU6EYTlGO7mmkk1b3Xdyum9VorDB8EH18bhdn4fCYjcVIzqe5lTqRm177WjXbGzt1GLgKWFe0sJGtVjUoykt7mTio9qb/wDsDCB914LZmJxClCtTo02qbsqijo1LPo72d0tLviVRwWzHCm3iJJqKc1vY6t0nLTTS0lb9xg+OD7UsJsyNOX9bNGOeaUakc79yDUb27XJcOomvsnBwwOLr0a06qpZss1NW0mkla2t073QwfEB9nCUdnSwWFniFRi97FVb1LymnLXg7xSXFNfuWQwGy4re1asVauk6SrqXuZ7NO3lrddQwfCB9bFYLZ8MFWq06zVZTko01UjJJKVku13Wt0fJAAAKAAAAAAAAAAAAAAAAAAAAAAAAG1eyHwH85T+01U2r2Q+A/nKf2kvg1UAFAAAAAAAAAAAACYq8kr2u7XAiz7BbyMmpSlGpC85Rc3a84W/R/oenh6yWtSGuvEIxXmb1u7K2pFn2GS6FfOouUb2zLXzsFQrSV88P3fkBjW8hbyMpYetZveQ1fbxPFWNWm0m76XvECiwLZwk6Sm23fX8uiXDiVAAAFCc8smTNLJe+W+l/0IAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA2r2Q+A/nKf2mqm1eyHwH85T+0l8GqgAoAAAAAAAAAAAAADd+OoAANt2u720AAAAALu1r6dgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABtXsh8B/OU/tNVNy9hcPGvh6zln/AKeJhJZVf/a+PkS+DTQAUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA3v+HH9rjv8kPtAJfB/9k=";