feat(site): rich preview for the spawn_computer_use_agent tool (#23684)

Adds preview cards for the `spawn_computer_use_agent` tool. Spawning
that agent now renders a rich saying "Spawning computer use sub-agent".
The "waiting for" tool now displays an inline desktop preview which can
be clicked to reveal the desktop in the sidebar.


https://github.com/user-attachments/assets/e486ca0e-a569-4142-bb12-db3b707967b8
This commit is contained in:
Hugo Dutka
2026-03-26 21:52:03 +00:00
committed by GitHub
parent 113aaa79a0
commit f87d6e6e82
17 changed files with 876 additions and 151 deletions
@@ -1,7 +1,8 @@
import type { Meta, StoryObj } from "@storybook/react-vite";
import { expect, spyOn, userEvent, waitFor, within } from "storybook/test";
import { expect, fn, spyOn, userEvent, waitFor, within } from "storybook/test";
import { reactRouterParameters } from "storybook-addon-remix-react-router";
import { Tool } from "./tool";
import { DesktopPanelContext } from "./tool/DesktopPanelContext";
const executeCommand = "git fetch origin";
const meta: Meta<typeof Tool> = {
@@ -1094,3 +1095,117 @@ export const MCPToolFailedUnifiedStyle: Story = {
expect(canvasElement.querySelector(".text-content-destructive")).toBeNull();
},
};
// ---------------------------------------------------------------------------
// spawn_computer_use_agent stories
// ---------------------------------------------------------------------------
export const SpawnComputerUseAgentRunning: Story = {
args: {
name: "spawn_computer_use_agent",
status: "running",
args: {
title: "Visual regression check",
prompt:
"Open the browser and check for visual regressions on the dashboard page.",
},
result: {
chat_id: "desktop-child-1",
title: "Visual regression check",
status: "pending",
},
},
play: async ({ canvasElement }) => {
const canvas = within(canvasElement);
expect(canvas.getByText(/Spawning/)).toBeInTheDocument();
expect(canvasElement.querySelector(".animate-spin")).not.toBeNull();
},
};
export const SpawnComputerUseAgentCompleted: Story = {
args: {
name: "spawn_computer_use_agent",
status: "completed",
args: {
title: "Visual regression check",
prompt:
"Open the browser and check for visual regressions on the dashboard page.",
},
result: {
chat_id: "desktop-child-1",
title: "Visual regression check",
status: "completed",
duration_ms: "12400",
},
},
play: async ({ canvasElement }) => {
const canvas = within(canvasElement);
expect(canvas.getByText(/Spawned/)).toBeInTheDocument();
expect(canvas.getByText(/Visual regression check/)).toBeInTheDocument();
expect(canvas.getByText("Worked for 12s")).toBeInTheDocument();
expect(canvas.getByRole("link", { name: "View agent" })).toHaveAttribute(
"href",
"/agents/desktop-child-1",
);
},
};
export const SpawnComputerUseAgentError: Story = {
args: {
name: "spawn_computer_use_agent",
status: "error",
isError: true,
result: {
chat_id: "desktop-child-1",
status: "error",
},
},
play: async ({ canvasElement }) => {
expect(canvasElement.querySelector(".lucide-circle-x")).not.toBeNull();
},
};
// ---------------------------------------------------------------------------
// wait_agent with computer-use subagent stories
// ---------------------------------------------------------------------------
export const WaitAgentComputerUseRunning: Story = {
args: {
name: "wait_agent",
status: "running",
args: {
chat_id: "desktop-child-1",
},
result: {
chat_id: "desktop-child-1",
status: "pending",
},
computerUseSubagentIds: new Set(["desktop-child-1"]),
},
decorators: [
(Story) => (
<DesktopPanelContext.Provider
value={{
desktopChatId: "desktop-child-1",
onOpenDesktop: fn(),
}}
>
<Story />
</DesktopPanelContext.Provider>
),
],
play: async ({ canvasElement }) => {
const canvas = within(canvasElement);
expect(canvas.getByText(/Waiting for/)).toBeInTheDocument();
// Running state shows the spinner icon.
expect(canvasElement.querySelector(".lucide-loader")).not.toBeNull();
// The VNC preview container should mount (the connection will
// stay in "connecting" state without a real WebSocket, which
// is expected — we only verify the container renders).
await waitFor(() => {
expect(
canvas.getByRole("button", { name: "Open desktop tab" }),
).toBeInTheDocument();
});
},
};
@@ -0,0 +1,12 @@
import { createContext, useContext } from "react";
interface DesktopPanelContextValue {
/** The parent chat ID used for the desktop VNC connection. */
desktopChatId?: string;
/** Opens the right sidebar panel and switches to the Desktop tab. */
onOpenDesktop?: () => void;
}
export const DesktopPanelContext = createContext<DesktopPanelContextValue>({});
export const useDesktopPanel = () => useContext(DesktopPanelContext);
@@ -0,0 +1,130 @@
import type { Meta, StoryObj } from "@storybook/react-vite";
import { expect, fn, within } from "storybook/test";
import { InlineDesktopPreview } from "./InlineDesktopPreview";
const meta: Meta<typeof InlineDesktopPreview> = {
title: "components/ai-elements/InlineDesktopPreview",
component: InlineDesktopPreview,
decorators: [
(Story) => (
<div className="max-w-md rounded-lg border border-solid border-border-default bg-surface-primary p-4">
<Story />
</div>
),
],
args: {
chatId: "desktop-chat-1",
onClick: fn(),
},
};
export default meta;
type Story = StoryObj<typeof InlineDesktopPreview>;
// ---------------------------------------------------------------------------
// Idle — hook has not started connecting yet.
// ---------------------------------------------------------------------------
export const Idle: Story = {
args: {
connectionOverride: {
status: "idle",
hasConnected: false,
reconnect: fn(),
attach: fn(),
rfb: null,
remoteClipboardText: null,
},
},
play: async ({ canvasElement }) => {
// The idle state shows a loading spinner.
const canvas = within(canvasElement);
expect(canvas.getByTitle("Loading spinner")).toBeInTheDocument();
},
};
// ---------------------------------------------------------------------------
// Connecting — WebSocket handshake in progress.
// ---------------------------------------------------------------------------
export const Connecting: Story = {
args: {
connectionOverride: {
status: "connecting",
hasConnected: false,
reconnect: fn(),
attach: fn(),
rfb: null,
remoteClipboardText: null,
},
},
play: async ({ canvasElement }) => {
const canvas = within(canvasElement);
expect(canvas.getByTitle("Loading spinner")).toBeInTheDocument();
},
};
// ---------------------------------------------------------------------------
// Connected — VNC canvas attached.
// ---------------------------------------------------------------------------
export const Connected: Story = {
args: {
connectionOverride: {
status: "connected",
hasConnected: true,
reconnect: fn(),
attach: fn(),
rfb: null,
remoteClipboardText: null,
},
},
play: async ({ canvasElement }) => {
// The connected state renders the VNC container with
// pointer-events-none to act as a read-only preview.
expect(canvasElement.querySelector(".pointer-events-none")).not.toBeNull();
},
};
// ---------------------------------------------------------------------------
// Disconnected — connection dropped, auto-reconnecting.
// ---------------------------------------------------------------------------
export const Disconnected: Story = {
args: {
connectionOverride: {
status: "disconnected",
hasConnected: true,
reconnect: fn(),
attach: fn(),
rfb: null,
remoteClipboardText: null,
},
},
play: async ({ canvasElement }) => {
const canvas = within(canvasElement);
expect(canvas.getByText(/Desktop disconnected/)).toBeInTheDocument();
},
};
// ---------------------------------------------------------------------------
// Error — connection failed permanently.
// ---------------------------------------------------------------------------
export const ErrorState: Story = {
args: {
connectionOverride: {
status: "error",
hasConnected: false,
reconnect: fn(),
attach: fn(),
rfb: null,
remoteClipboardText: null,
},
},
play: async ({ canvasElement }) => {
const canvas = within(canvasElement);
expect(
canvas.getByText(/Could not connect to desktop/),
).toBeInTheDocument();
},
};
@@ -0,0 +1,141 @@
import { ExternalLinkIcon } from "lucide-react";
import {
type UseDesktopConnectionResult,
useDesktopConnection,
} from "pages/AgentsPage/hooks/useDesktopConnection";
import type React from "react";
import { useEffect, useRef, useState } from "react";
import { Spinner } from "#/components/Spinner/Spinner";
/** Default aspect ratio used before the remote framebuffer size is known. */
const DEFAULT_ASPECT = "16 / 9";
/**
* Non-interactive inline VNC desktop preview. The noVNC canvas is
* blocked from receiving pointer/keyboard events so it acts as a
* read-only thumbnail. An invisible overlay captures clicks and
* forwards them to `onClick` (e.g. opens the sidebar Desktop tab).
*
* The container's aspect-ratio is derived from the remote desktop's
* framebuffer dimensions so there is no dead space around the
* preview.
*/
export const InlineDesktopPreview: React.FC<{
chatId: string;
onClick?: () => void;
/** Optional override for the desktop connection hook result.
* When provided, the real hook is skipped entirely. Used by
* Storybook stories to inject mock connection states without
* relying on module-level spies. */
connectionOverride?: UseDesktopConnectionResult;
}> = ({ chatId, onClick, connectionOverride }) => {
// Pass undefined chatId when the override is provided so the
// real hook skips its WebSocket connection logic entirely.
const realConnection = useDesktopConnection({
chatId: connectionOverride ? undefined : chatId,
});
const { status, attach } = connectionOverride ?? realConnection;
const [aspectRatio, setAspectRatio] = useState(DEFAULT_ASPECT);
const containerRef = useRef<HTMLElement | null>(null);
// Derive the aspect ratio from the noVNC canvas once connected.
// noVNC renders into a <canvas> whose intrinsic width/height
// attributes match the remote framebuffer dimensions (when
// clipViewport is disabled, which is the case here since
// scaleViewport is enabled). Querying the canvas from the DOM
// avoids accessing noVNC's private _fbWidth/_fbHeight fields.
useEffect(() => {
if (status !== "connected" || !containerRef.current) {
return;
}
let timeoutId: ReturnType<typeof setTimeout> | null = null;
const readDimensions = () => {
const canvas = containerRef.current?.querySelector("canvas");
if (canvas && canvas.width > 0 && canvas.height > 0) {
setAspectRatio(`${canvas.width} / ${canvas.height}`);
return true;
}
return false;
};
if (!readDimensions()) {
// The canvas dimensions may not be set immediately after
// the status transitions to "connected". Retry once after
// a short delay as a fallback.
timeoutId = setTimeout(readDimensions, 500);
}
return () => {
if (timeoutId !== null) {
clearTimeout(timeoutId);
}
};
}, [status]);
const wrapWithOverlay = (children: React.ReactNode) => (
<div className="group relative">
{children}
{/* Transparent overlay — dims the preview on hover and shows
an external-link icon so it's clear clicking opens the
sidebar desktop tab. */}
{onClick && (
<button
type="button"
onClick={onClick}
aria-label="Open desktop tab"
className="absolute inset-0 z-10 flex cursor-pointer items-center justify-center border-0 bg-black/0 p-0 transition-colors group-hover:bg-black/50"
>
<ExternalLinkIcon className="h-6 w-6 text-white opacity-0 drop-shadow-md transition-opacity group-hover:opacity-100" />
</button>
)}
</div>
);
if (status === "idle" || status === "connecting") {
return wrapWithOverlay(
<div
className="flex items-center justify-center text-content-secondary"
style={{ aspectRatio: DEFAULT_ASPECT }}
>
<Spinner loading className="h-5 w-5" />
</div>,
);
}
if (status === "disconnected") {
return wrapWithOverlay(
<div
className="flex items-center justify-center text-xs text-content-secondary"
style={{ aspectRatio }}
>
Desktop disconnected. Reconnecting
</div>,
);
}
if (status === "error") {
return wrapWithOverlay(
<div
className="flex items-center justify-center text-xs text-content-secondary"
style={{ aspectRatio: DEFAULT_ASPECT }}
>
Could not connect to desktop.
</div>,
);
}
// status === "connected" — pointer-events-none on the VNC
// container prevents noVNC from capturing any input.
return wrapWithOverlay(
<div
ref={(el) => {
containerRef.current = el;
if (el) attach(el);
}}
className="pointer-events-none w-full"
style={{ aspectRatio }}
/>,
);
};
@@ -5,6 +5,7 @@ import {
ClockIcon,
ExternalLinkIcon,
LoaderIcon,
MonitorIcon,
} from "lucide-react";
import type React from "react";
import { useState } from "react";
@@ -12,6 +13,8 @@ import { Link } from "react-router";
import { cn } from "utils/cn";
import { ScrollArea } from "#/components/ScrollArea/ScrollArea";
import { Response } from "../response";
import { useDesktopPanel } from "./DesktopPanelContext";
import { InlineDesktopPreview } from "./InlineDesktopPreview";
import {
isSubagentSuccessStatus,
shortDurationMs,
@@ -46,6 +49,12 @@ const SUBAGENT_VERBS: Record<
error: "Failed to terminate ",
timeout: "Timed out terminating ",
},
spawn_computer_use_agent: {
completed: "Spawned ",
running: "Spawning ",
error: "Failed to spawn ",
timeout: "Timed out spawning ",
},
};
/**
@@ -60,8 +69,16 @@ const SubagentStatusIcon: React.FC<{
toolStatus: ToolStatus;
isError: boolean;
isTimeout: boolean;
}> = ({ subagentStatus, toolStatus, isError, isTimeout }) => {
variant?: "default" | "computer-use";
}> = ({
subagentStatus,
toolStatus,
isError,
isTimeout,
variant = "default",
}) => {
const subagentCompleted = isSubagentSuccessStatus(subagentStatus);
const DefaultIcon = variant === "computer-use" ? MonitorIcon : BotIcon;
if (isTimeout && !subagentCompleted) {
return <ClockIcon className="h-4 w-4 shrink-0 text-content-secondary" />;
}
@@ -73,7 +90,7 @@ const SubagentStatusIcon: React.FC<{
<LoaderIcon className="h-4 w-4 shrink-0 animate-spin motion-reduce:animate-none text-content-link" />
);
}
return <BotIcon className="h-4 w-4 shrink-0 text-content-secondary" />;
return <DefaultIcon className="h-4 w-4 shrink-0 text-content-secondary" />;
};
/**
@@ -94,6 +111,9 @@ export const SubagentTool: React.FC<{
toolStatus: ToolStatus;
isError: boolean;
isTimeout?: boolean;
/** Show an inline VNC desktop preview (for computer-use subagents). */
showDesktopPreview?: boolean;
variant?: "default" | "computer-use";
}> = ({
toolName,
title,
@@ -106,8 +126,11 @@ export const SubagentTool: React.FC<{
toolStatus,
isError,
isTimeout = false,
showDesktopPreview,
variant = "default",
}) => {
const [expanded, setExpanded] = useState(false);
const { desktopChatId, onOpenDesktop } = useDesktopPanel();
const hasPrompt = Boolean(prompt?.trim());
const hasMessage = Boolean(message?.trim());
const hasReport = Boolean(report?.trim());
@@ -118,7 +141,7 @@ export const SubagentTool: React.FC<{
<div className="w-full">
<button
type="button"
aria-expanded={expanded}
aria-expanded={hasExpandableContent ? expanded : undefined}
onClick={() => hasExpandableContent && setExpanded((v) => !v)}
className={cn(
"border-0 bg-transparent p-0 m-0 font-[inherit] text-[inherit] text-left",
@@ -131,6 +154,7 @@ export const SubagentTool: React.FC<{
toolStatus={toolStatus}
isError={isError}
isTimeout={isTimeout}
variant={variant}
/>
<span className="min-w-0 flex-1 truncate text-sm text-content-secondary">
{SUBAGENT_VERBS[toolName]?.[
@@ -156,7 +180,7 @@ export const SubagentTool: React.FC<{
</span>
{durationLabel && (
<span className="shrink-0 text-xs text-content-secondary">
Worked for {durationLabel}
{`Worked for ${durationLabel}`}
</span>
)}
{hasExpandableContent && (
@@ -169,6 +193,15 @@ export const SubagentTool: React.FC<{
)}
</button>
{showDesktopPreview && desktopChatId && (
<div className="mt-1.5 overflow-hidden rounded-lg border border-solid border-border-default">
<InlineDesktopPreview
chatId={desktopChatId}
onClick={onOpenDesktop}
/>
</div>
)}
{expanded && hasPrompt && (
<ScrollArea
className="mt-1.5 rounded-md border border-solid border-border-default"
+25 -1
View File
@@ -58,6 +58,11 @@ interface ToolProps extends Omit<ComponentPropsWithRef<"div">, "children"> {
isError?: boolean;
/** Maps sub-agent chat IDs to their titles, built from spawn tool results. */
subagentTitles?: Map<string, string>;
/** Set of chat IDs spawned by `spawn_computer_use_agent`. */
computerUseSubagentIds?: Set<string>;
/** When false, suppresses inline VNC previews while still
* allowing the MonitorIcon variant to render. */
showDesktopPreviews?: boolean;
/** Maps sub-agent chat IDs to real-time status updates from stream events. */
subagentStatusOverrides?: Map<string, string>;
/** MCP server config ID associated with this tool call. */
@@ -75,6 +80,8 @@ type ToolRendererProps = {
result: unknown;
isError: boolean;
subagentTitles?: Map<string, string>;
computerUseSubagentIds?: Set<string>;
showDesktopPreviews?: boolean;
subagentStatusOverrides?: Map<string, string>;
mcpServerConfigId?: string;
mcpServers?: readonly TypesGen.MCPServerConfig[];
@@ -268,6 +275,8 @@ const SubagentRenderer: FC<ToolRendererProps> = ({
result,
isError,
subagentTitles,
computerUseSubagentIds,
showDesktopPreviews = true,
subagentStatusOverrides,
}) => {
const parsedArgs = parseArgs(args);
@@ -293,7 +302,9 @@ const SubagentRenderer: FC<ToolRendererProps> = ({
(rec ? asString(rec.title) : "") ||
(parsedArgs ? asString(parsedArgs.title) : "") ||
(chatId && subagentTitles?.get(chatId)) ||
"Sub-agent";
(name === "spawn_computer_use_agent"
? "Computer use sub-agent"
: "Sub-agent");
const subagentCompleted = isSubagentSuccessStatus(subagentStatus);
const subagentToolStatus = mapSubagentStatusToToolStatus(
subagentStatus,
@@ -313,6 +324,10 @@ const SubagentRenderer: FC<ToolRendererProps> = ({
(resultStr.toLowerCase().includes("timed out") ||
errorStr.toLowerCase().includes("timed out"));
const variant =
name === "spawn_computer_use_agent" || computerUseSubagentIds?.has(chatId)
? "computer-use"
: "default";
return (
<SubagentTool
toolName={name}
@@ -326,6 +341,10 @@ const SubagentRenderer: FC<ToolRendererProps> = ({
toolStatus={subagentToolStatus}
isError={subagentIsError}
isTimeout={isTimeout}
showDesktopPreview={
showDesktopPreviews && computerUseSubagentIds?.has(chatId)
}
variant={variant}
/>
);
};
@@ -603,6 +622,7 @@ const toolRenderers: Record<string, FC<ToolRendererProps>> = {
wait_agent: SubagentRenderer,
message_agent: SubagentRenderer,
close_agent: SubagentRenderer,
spawn_computer_use_agent: SubagentRenderer,
chat_summarized: ChatSummarizedRenderer,
propose_plan: ProposePlanRenderer,
computer: ComputerRenderer,
@@ -621,6 +641,8 @@ export const Tool = memo(
result,
isError = false,
subagentTitles,
computerUseSubagentIds,
showDesktopPreviews,
subagentStatusOverrides,
mcpServerConfigId,
mcpServers,
@@ -649,6 +671,8 @@ export const Tool = memo(
result={result}
isError={isError}
subagentTitles={subagentTitles}
computerUseSubagentIds={computerUseSubagentIds}
showDesktopPreviews={showDesktopPreviews}
subagentStatusOverrides={subagentStatusOverrides}
mcpServerConfigId={mcpServerConfigId}
mcpServers={mcpServers}
@@ -83,6 +83,7 @@ export const ToolIcon: React.FC<{
case "propose_plan":
return <ClipboardListIcon className={base} />;
case "computer":
case "spawn_computer_use_agent":
return <MonitorIcon className={base} />;
default:
return <WrenchIcon className={base} />;
@@ -673,6 +673,92 @@ export const WithSubagentCards: Story = {
},
};
/** spawn_computer_use_agent tool renders with an "Open Desktop" button
* that opens the right sidebar panel and switches to the Desktop tab. */
export const WithComputerUseAgent: Story = {
parameters: {
queries: [
...buildQueries(
{
id: CHAT_ID,
...baseChatFields,
title: "Desktop automation task",
status: "running",
},
{
messages: [
{
id: 1,
chat_id: CHAT_ID,
created_at: "2026-02-18T00:00:01.000Z",
role: "user",
content: [
{
type: "text",
text: "Can you check the browser for visual regressions?",
},
],
},
{
id: 2,
chat_id: CHAT_ID,
created_at: "2026-02-18T00:00:02.000Z",
role: "assistant",
content: [
{
type: "text",
text: "I'll spawn a computer use agent to visually inspect the browser.",
},
{
type: "tool-call",
tool_call_id: "tool-desktop-1",
tool_name: "spawn_computer_use_agent",
args: {
title: "Visual regression check",
prompt:
"Open the browser and check for visual regressions on the dashboard page.",
},
},
{
type: "tool-result",
tool_call_id: "tool-desktop-1",
tool_name: "spawn_computer_use_agent",
result: {
chat_id: "desktop-child-1",
title: "Visual regression check",
status: "completed",
duration_ms: "12400",
},
},
{
type: "text",
text: "The desktop agent has finished its visual inspection. No regressions found. You can click **Open Desktop** above to view the desktop session.",
},
],
},
],
queued_messages: [],
has_more: false,
},
{ diffUrl: undefined },
),
// Enable the desktop feature so the Desktop tab appears in the sidebar.
{
key: ["chat-desktop-enabled"],
data: { enable_desktop: true },
},
],
},
play: async ({ canvasElement }) => {
const canvas = within(canvasElement);
// The tool should show "Spawned ... Visual regression check".
await waitFor(() => {
expect(canvas.getByText(/Visual regression check/)).toBeInTheDocument();
});
},
};
/** Completed reasoning part renders inline. */
export const WithReasoningInline: Story = {
parameters: {
@@ -1188,3 +1274,88 @@ export const FailedSendWithActiveStream: Story = {
).toBeInTheDocument();
},
};
/** wait_agent for a computer-use subagent renders the VNC preview card
* (SubagentTool with computer-use variant) instead of the plain SubagentTool card. */
export const WithWaitAgentComputerUseVNC: Story = {
parameters: {
queries: [
...buildQueries(
{
id: CHAT_ID,
...baseChatFields,
title: "Wait agent computer use",
status: "running",
},
{
messages: [
{
id: 1,
chat_id: CHAT_ID,
created_at: "2026-02-18T00:00:01.000Z",
role: "assistant",
content: [
{
type: "tool-call",
tool_call_id: "tool-spawn-desktop",
tool_name: "spawn_computer_use_agent",
args: {
title: "Visual check",
prompt: "Check the browser.",
},
},
{
type: "tool-result",
tool_call_id: "tool-spawn-desktop",
tool_name: "spawn_computer_use_agent",
result: {
chat_id: "desktop-child-1",
title: "Visual check",
status: "completed",
},
},
],
},
],
queued_messages: [],
has_more: false,
},
{ diffUrl: undefined },
),
{
key: ["chat-desktop-enabled"],
data: { enable_desktop: true },
},
],
// The wait_agent arrives via WebSocket so it renders in
// the streaming/running state (no tool-result yet).
webSocket: {
"/chats/": [
{
event: "message",
data: wrapSSE({
type: "message_part",
chat_id: CHAT_ID,
message_part: {
part: {
type: "tool-call",
tool_call_id: "tool-wait-desktop",
tool_name: "wait_agent",
args_delta: '{"chat_id":"desktop-child-1"}',
},
},
}),
},
],
},
},
play: async ({ canvasElement }) => {
const canvas = within(canvasElement);
// The wait_agent card should show "Waiting for" (running state)
// rendered via SubagentTool with VNC preview.
await waitFor(() => {
expect(canvas.getByText(/Waiting for/)).toBeInTheDocument();
});
},
};
@@ -249,6 +249,8 @@ const BlockList: FC<{
keyPrefix: string;
isStreaming?: boolean;
subagentTitles?: Map<string, string>;
computerUseSubagentIds?: Set<string>;
showDesktopPreviews?: boolean;
subagentStatusOverrides?: Map<string, TypesGen.ChatStatus>;
mcpServers?: readonly TypesGen.MCPServerConfig[];
onImageClick?: (src: string) => void;
@@ -260,6 +262,8 @@ const BlockList: FC<{
keyPrefix,
isStreaming = false,
subagentTitles,
computerUseSubagentIds,
showDesktopPreviews,
subagentStatusOverrides,
mcpServers,
onImageClick,
@@ -353,6 +357,8 @@ const BlockList: FC<{
status={tool.status}
isError={tool.isError}
subagentTitles={subagentTitles}
computerUseSubagentIds={computerUseSubagentIds}
showDesktopPreviews={showDesktopPreviews}
subagentStatusOverrides={
isStreaming ? subagentStatusOverrides : undefined
}
@@ -391,6 +397,8 @@ const BlockList: FC<{
status={tool.status}
isError={tool.isError}
subagentTitles={subagentTitles}
computerUseSubagentIds={computerUseSubagentIds}
showDesktopPreviews={showDesktopPreviews}
subagentStatusOverrides={
isStreaming ? subagentStatusOverrides : undefined
}
@@ -401,7 +409,6 @@ const BlockList: FC<{
</>
);
};
const ChatMessageItem = memo<{
message: TypesGen.ChatMessage;
parsed: ParsedMessageContent;
@@ -420,6 +427,8 @@ const ChatMessageItem = memo<{
urlTransform?: UrlTransform;
mcpServers?: readonly TypesGen.MCPServerConfig[];
subagentTitles?: Map<string, string>;
computerUseSubagentIds?: Set<string>;
showDesktopPreviews?: boolean;
}>(
({
message,
@@ -432,6 +441,8 @@ const ChatMessageItem = memo<{
urlTransform,
mcpServers,
subagentTitles,
computerUseSubagentIds,
showDesktopPreviews,
}) => {
const isUser = message.role === "user";
const isSavingMessage = savingMessageId === message.id;
@@ -622,6 +633,8 @@ const ChatMessageItem = memo<{
tools={parsed.tools}
keyPrefix={String(message.id)}
subagentTitles={subagentTitles}
computerUseSubagentIds={computerUseSubagentIds}
showDesktopPreviews={showDesktopPreviews}
onImageClick={setPreviewImage}
onTextFileClick={setPreviewText}
urlTransform={urlTransform}
@@ -663,6 +676,7 @@ export const StreamingOutput: FC<{
streamState: StreamState | null;
streamTools: readonly MergedTool[];
subagentTitles?: Map<string, string>;
computerUseSubagentIds?: Set<string>;
subagentStatusOverrides?: Map<string, TypesGen.ChatStatus>;
liveStatus: LiveStatusModel;
startingResetKey?: string;
@@ -672,6 +686,7 @@ export const StreamingOutput: FC<{
streamState,
streamTools,
subagentTitles,
computerUseSubagentIds,
subagentStatusOverrides,
liveStatus,
startingResetKey,
@@ -705,6 +720,7 @@ export const StreamingOutput: FC<{
keyPrefix="stream"
isStreaming={isStreaming}
subagentTitles={subagentTitles}
computerUseSubagentIds={computerUseSubagentIds}
subagentStatusOverrides={subagentStatusOverrides}
urlTransform={urlTransform}
mcpServers={mcpServers}
@@ -1019,6 +1035,8 @@ interface ConversationTimelineProps {
savingMessageId?: number | null;
urlTransform?: UrlTransform;
mcpServers?: readonly TypesGen.MCPServerConfig[];
computerUseSubagentIds?: Set<string>;
showDesktopPreviews?: boolean;
}
export const ConversationTimeline: FC<ConversationTimelineProps> = ({
@@ -1028,6 +1046,8 @@ export const ConversationTimeline: FC<ConversationTimelineProps> = ({
savingMessageId,
urlTransform,
mcpServers,
computerUseSubagentIds,
showDesktopPreviews,
}) => {
const subagentTitles = buildSubagentTitles(parsedMessages);
@@ -1074,6 +1094,8 @@ export const ConversationTimeline: FC<ConversationTimelineProps> = ({
isAfterEditingMessage={afterEditingMessageIds.has(message.id)}
mcpServers={mcpServers}
subagentTitles={subagentTitles}
computerUseSubagentIds={computerUseSubagentIds}
showDesktopPreviews={showDesktopPreviews}
/>
),
)}
@@ -36,6 +36,7 @@ interface LiveStreamTailContentProps {
liveStatus: LiveStatusModel;
startingResetKey?: string;
subagentTitles: Map<string, string>;
computerUseSubagentIds?: Set<string>;
subagentStatusOverrides: Map<string, TypesGen.ChatStatus>;
urlTransform?: UrlTransform;
mcpServers?: readonly TypesGen.MCPServerConfig[];
@@ -48,6 +49,7 @@ export const LiveStreamTailContent = ({
liveStatus,
startingResetKey,
subagentTitles,
computerUseSubagentIds,
subagentStatusOverrides,
urlTransform,
mcpServers,
@@ -81,6 +83,7 @@ export const LiveStreamTailContent = ({
liveStatus={liveStatus}
startingResetKey={startingResetKey}
subagentTitles={subagentTitles}
computerUseSubagentIds={computerUseSubagentIds}
subagentStatusOverrides={subagentStatusOverrides}
urlTransform={urlTransform}
mcpServers={mcpServers}
@@ -111,6 +114,7 @@ interface LiveStreamTailProps {
isTranscriptEmpty: boolean;
startingResetKey?: string;
subagentTitles: Map<string, string>;
computerUseSubagentIds?: Set<string>;
urlTransform?: UrlTransform;
mcpServers?: readonly TypesGen.MCPServerConfig[];
}
@@ -121,6 +125,7 @@ export const LiveStreamTail = ({
isTranscriptEmpty,
startingResetKey,
subagentTitles,
computerUseSubagentIds,
urlTransform,
mcpServers,
}: LiveStreamTailProps) => {
@@ -157,6 +162,7 @@ export const LiveStreamTail = ({
liveStatus={liveStatus}
startingResetKey={startingResetKey}
subagentTitles={subagentTitles}
computerUseSubagentIds={computerUseSubagentIds}
subagentStatusOverrides={subagentStatusOverrides}
urlTransform={urlTransform}
mcpServers={mcpServers}
@@ -45,6 +45,13 @@ describe("parseToolResultIsError", () => {
{ status: "completed" },
),
).toBe(false);
expect(
parseToolResultIsError(
"spawn_computer_use_agent",
{ error: "metadata" },
{ status: "completed" },
),
).toBe(false);
});
});
@@ -19,7 +19,10 @@ const appendText = (current: string, next: string): string => {
};
const isSubagentToolName = (name: string): boolean =>
name === "spawn_agent" || name === "wait_agent" || name === "message_agent";
name === "spawn_agent" ||
name === "spawn_computer_use_agent" ||
name === "wait_agent" ||
name === "message_agent";
const isCompletedSubagentResult = (
toolName: string,
@@ -264,7 +267,10 @@ export const buildSubagentTitles = (
const map = new Map<string, string>();
for (const { parsed } of parsedMessages) {
for (const tool of parsed.tools) {
if (tool.name !== "spawn_agent") {
if (
tool.name !== "spawn_agent" &&
tool.name !== "spawn_computer_use_agent"
) {
continue;
}
const rec = asRecord(tool.result);
@@ -280,3 +286,25 @@ export const buildSubagentTitles = (
}
return map;
};
export const buildComputerUseSubagentIds = (
parsedMessages: readonly ParsedMessageEntry[],
): Set<string> => {
const ids = new Set<string>();
for (const { parsed } of parsedMessages) {
for (const tool of parsed.tools) {
if (tool.name !== "spawn_computer_use_agent") {
continue;
}
const rec = asRecord(tool.result);
if (!rec) {
continue;
}
const chatId = asString(rec.chat_id);
if (chatId) {
ids.add(chatId);
}
}
}
return ids;
};
@@ -24,6 +24,7 @@ import { ConversationTimeline } from "./AgentDetail/ConversationTimeline";
import { getLatestContextUsage } from "./AgentDetail/chatHelpers";
import { LiveStreamTail } from "./AgentDetail/LiveStreamTail";
import {
buildComputerUseSubagentIds,
buildSubagentTitles,
parseMessagesWithMergedTools,
} from "./AgentDetail/messageParsing";
@@ -68,11 +69,17 @@ export const AgentDetailTimeline: FC<AgentDetailTimelineProps> = ({
.filter(isChatMessage);
const parsedMessages = parseMessagesWithMergedTools(messages);
const subagentTitles = buildSubagentTitles(parsedMessages);
const computerUseSubagentIds = buildComputerUseSubagentIds(parsedMessages);
const onRenderProfiler = useOnRenderProfiler();
return (
<Profiler id="AgentChat" onRender={onRenderProfiler}>
<div className="mx-auto flex w-full max-w-3xl flex-col gap-3 py-6">
{/* VNC sessions for completed agents may already be
terminated, so inline desktop previews are disabled
via showDesktopPreviews={false} to avoid a perpetual
"disconnected" state. The MonitorIcon variant still
renders correctly. */}
<ConversationTimeline
parsedMessages={parsedMessages}
onEditUserMessage={onEditUserMessage}
@@ -80,6 +87,8 @@ export const AgentDetailTimeline: FC<AgentDetailTimelineProps> = ({
savingMessageId={savingMessageId}
urlTransform={urlTransform}
mcpServers={mcpServers}
computerUseSubagentIds={computerUseSubagentIds}
showDesktopPreviews={false}
/>
<LiveStreamTail
store={store}
@@ -87,6 +96,7 @@ export const AgentDetailTimeline: FC<AgentDetailTimelineProps> = ({
startingResetKey={chatID}
isTranscriptEmpty={parsedMessages.length === 0}
subagentTitles={subagentTitles}
computerUseSubagentIds={computerUseSubagentIds}
urlTransform={urlTransform}
mcpServers={mcpServers}
/>
@@ -6,6 +6,7 @@ import { pageTitle } from "utils/page";
import type * as TypesGen from "#/api/typesGenerated";
import type { ChatDiffStatus, ChatMessagePart } from "#/api/typesGenerated";
import type { ModelSelectorOption } from "#/components/ai-elements";
import { DesktopPanelContext } from "#/components/ai-elements/tool/DesktopPanelContext";
import { Button } from "#/components/Button/Button";
import type { ChatDetailError } from "../utils/usageLimitMessage";
import { AgentChatInput, type ChatMessageInputRef } from "./AgentChatInput";
@@ -187,6 +188,20 @@ export const AgentDetailView: FC<AgentDetailViewProps> = ({
);
const visualExpanded = dragVisualExpanded ?? isRightPanelExpanded;
// State for programmatically switching the sidebar tab (e.g. when
// the user clicks the inline desktop preview card).
const [sidebarTabId, setSidebarTabId] = useState<string | null>(null);
const handleOpenDesktop = () => {
onSetShowSidebarPanel(true);
setSidebarTabId("desktop");
};
const desktopPanelCtx = {
desktopChatId,
onOpenDesktop: desktopChatId ? handleOpenDesktop : undefined,
};
// Compute local diff stats from git watcher unified diffs.
const titleElement = (
@@ -198,155 +213,161 @@ export const AgentDetailView: FC<AgentDetailViewProps> = ({
const shouldShowSidebar = showSidebarPanel;
return (
<div
className={cn(
"relative flex min-h-0 min-w-0 flex-1",
shouldShowSidebar && !visualExpanded && "flex-row",
)}
>
{titleElement}
<DesktopPanelContext value={desktopPanelCtx}>
<div
className={cn(
"relative flex min-h-0 min-w-0 flex-1 flex-col overflow-x-hidden",
visualExpanded && "hidden",
shouldShowSidebar && "max-md:hidden",
"relative flex min-h-0 min-w-0 flex-1",
shouldShowSidebar && !visualExpanded && "flex-row",
)}
>
<div className="relative z-10 shrink-0 overflow-visible">
<AgentDetailTopBar
chatTitle={chatTitle}
parentChat={parentChat}
panel={{
showSidebarPanel,
onToggleSidebar: () => onSetShowSidebarPanel((prev) => !prev),
}}
workspace={{
canOpenEditors,
canOpenWorkspace,
onOpenInEditor: handleOpenInEditor,
onViewWorkspace: handleViewWorkspace,
onOpenTerminal: handleOpenTerminal,
sshCommand,
}}
onArchiveAgent={handleArchiveAgentAction}
onUnarchiveAgent={handleUnarchiveAgentAction}
onArchiveAndDeleteWorkspace={handleArchiveAndDeleteWorkspaceAction}
hasWorkspace={hasWorkspace}
isArchived={isArchived}
diffStatusData={diffStatusData}
isSidebarCollapsed={isSidebarCollapsed}
onToggleSidebarCollapsed={onToggleSidebarCollapsed}
/>
{isArchived && (
<div className="flex shrink-0 items-center gap-2 border-b border-border-default bg-surface-secondary px-4 py-2 text-xs text-content-secondary">
<ArchiveIcon className="h-4 w-4 shrink-0" />
This agent has been archived and is read-only.
</div>
{titleElement}
<div
className={cn(
"relative flex min-h-0 min-w-0 flex-1 flex-col overflow-x-hidden",
visualExpanded && "hidden",
shouldShowSidebar && "max-md:hidden",
)}
<div
aria-hidden
className="pointer-events-none absolute inset-x-0 top-full z-10 h-3 sm:h-6 bg-surface-primary"
style={{
maskImage:
"linear-gradient(to bottom, black 0%, rgba(0,0,0,0.6) 40%, rgba(0,0,0,0.2) 70%, transparent 100%)",
WebkitMaskImage:
"linear-gradient(to bottom, black 0%, rgba(0,0,0,0.6) 40%, rgba(0,0,0,0.2) 70%, transparent 100%)",
}}
/>
</div>
<ScrollAnchoredContainer
scrollContainerRef={scrollContainerRef}
isFetchingMoreMessages={isFetchingMoreMessages}
hasMoreMessages={hasMoreMessages}
onFetchMoreMessages={onFetchMoreMessages}
>
<div className="px-4">
<AgentDetailTimeline
chatID={agentId}
<div className="relative z-10 shrink-0 overflow-visible">
<AgentDetailTopBar
chatTitle={chatTitle}
parentChat={parentChat}
panel={{
showSidebarPanel,
onToggleSidebar: () => onSetShowSidebarPanel((prev) => !prev),
}}
workspace={{
canOpenEditors,
canOpenWorkspace,
onOpenInEditor: handleOpenInEditor,
onViewWorkspace: handleViewWorkspace,
onOpenTerminal: handleOpenTerminal,
sshCommand,
}}
onArchiveAgent={handleArchiveAgentAction}
onUnarchiveAgent={handleUnarchiveAgentAction}
onArchiveAndDeleteWorkspace={
handleArchiveAndDeleteWorkspaceAction
}
hasWorkspace={hasWorkspace}
isArchived={isArchived}
diffStatusData={diffStatusData}
isSidebarCollapsed={isSidebarCollapsed}
onToggleSidebarCollapsed={onToggleSidebarCollapsed}
/>
{isArchived && (
<div className="flex shrink-0 items-center gap-2 border-b border-border-default bg-surface-secondary px-4 py-2 text-xs text-content-secondary">
<ArchiveIcon className="h-4 w-4 shrink-0" />
This agent has been archived and is read-only.
</div>
)}
<div
aria-hidden
className="pointer-events-none absolute inset-x-0 top-full z-10 h-3 sm:h-6 bg-surface-primary"
style={{
maskImage:
"linear-gradient(to bottom, black 0%, rgba(0,0,0,0.6) 40%, rgba(0,0,0,0.2) 70%, transparent 100%)",
WebkitMaskImage:
"linear-gradient(to bottom, black 0%, rgba(0,0,0,0.6) 40%, rgba(0,0,0,0.2) 70%, transparent 100%)",
}}
/>
</div>
<ScrollAnchoredContainer
scrollContainerRef={scrollContainerRef}
isFetchingMoreMessages={isFetchingMoreMessages}
hasMoreMessages={hasMoreMessages}
onFetchMoreMessages={onFetchMoreMessages}
>
<div className="px-4">
<AgentDetailTimeline
chatID={agentId}
store={store}
persistedError={persistedError}
onEditUserMessage={editing.handleEditUserMessage}
editingMessageId={editing.editingMessageId}
savingMessageId={pendingEditMessageId}
urlTransform={urlTransform}
mcpServers={mcpServers}
/>
</div>
</ScrollAnchoredContainer>
<div className="shrink-0 overflow-y-auto px-4 pb-4 md:pb-0 [scrollbar-gutter:stable] [scrollbar-width:thin]">
<AgentDetailInput
store={store}
persistedError={persistedError}
onEditUserMessage={editing.handleEditUserMessage}
editingMessageId={editing.editingMessageId}
savingMessageId={pendingEditMessageId}
urlTransform={urlTransform}
compressionThreshold={compressionThreshold}
onSend={editing.handleSendFromInput}
onDeleteQueuedMessage={handleDeleteQueuedMessage}
onPromoteQueuedMessage={handlePromoteQueuedMessage}
onInterrupt={handleInterrupt}
isInputDisabled={isInputDisabled}
isSendPending={isSubmissionPending}
isInterruptPending={isInterruptPending}
hasModelOptions={hasModelOptions}
selectedModel={effectiveSelectedModel}
onModelChange={setSelectedModel}
modelOptions={modelOptions}
modelSelectorPlaceholder={modelSelectorPlaceholder}
isModelCatalogLoading={isModelCatalogLoading}
inputRef={editing.chatInputRef}
initialValue={editing.editorInitialValue}
onContentChange={editing.handleContentChange}
editingQueuedMessageID={editing.editingQueuedMessageID}
onStartQueueEdit={editing.handleStartQueueEdit}
onCancelQueueEdit={editing.handleCancelQueueEdit}
isEditingHistoryMessage={editing.editingMessageId !== null}
onCancelHistoryEdit={editing.handleCancelHistoryEdit}
editingFileBlocks={editing.editingFileBlocks}
mcpServers={mcpServers}
selectedMCPServerIds={selectedMCPServerIds}
onMCPSelectionChange={onMCPSelectionChange}
onMCPAuthComplete={onMCPAuthComplete}
/>
</div>
</ScrollAnchoredContainer>
<div className="shrink-0 overflow-y-auto px-4 pb-4 md:pb-0 [scrollbar-gutter:stable] [scrollbar-width:thin]">
<AgentDetailInput
store={store}
compressionThreshold={compressionThreshold}
onSend={editing.handleSendFromInput}
onDeleteQueuedMessage={handleDeleteQueuedMessage}
onPromoteQueuedMessage={handlePromoteQueuedMessage}
onInterrupt={handleInterrupt}
isInputDisabled={isInputDisabled}
isSendPending={isSubmissionPending}
isInterruptPending={isInterruptPending}
hasModelOptions={hasModelOptions}
selectedModel={effectiveSelectedModel}
onModelChange={setSelectedModel}
modelOptions={modelOptions}
modelSelectorPlaceholder={modelSelectorPlaceholder}
isModelCatalogLoading={isModelCatalogLoading}
inputRef={editing.chatInputRef}
initialValue={editing.editorInitialValue}
onContentChange={editing.handleContentChange}
editingQueuedMessageID={editing.editingQueuedMessageID}
onStartQueueEdit={editing.handleStartQueueEdit}
onCancelQueueEdit={editing.handleCancelQueueEdit}
isEditingHistoryMessage={editing.editingMessageId !== null}
onCancelHistoryEdit={editing.handleCancelHistoryEdit}
editingFileBlocks={editing.editingFileBlocks}
mcpServers={mcpServers}
selectedMCPServerIds={selectedMCPServerIds}
onMCPSelectionChange={onMCPSelectionChange}
onMCPAuthComplete={onMCPAuthComplete}
/>
</div>
</div>
<RightPanel
isOpen={shouldShowSidebar}
isExpanded={isRightPanelExpanded}
onToggleExpanded={() => setIsRightPanelExpanded((prev) => !prev)}
onClose={() => onSetShowSidebarPanel(false)}
onVisualExpandedChange={setDragVisualExpanded}
isSidebarCollapsed={isSidebarCollapsed}
onToggleSidebarCollapsed={onToggleSidebarCollapsed}
>
<SidebarTabView
tabs={[
{
id: "git",
label: "Git",
content: (
<GitPanel
prTab={
prNumber && agentId
? { prNumber, chatId: agentId }
: undefined
}
repositories={gitWatcher.repositories}
onRefresh={gitWatcher.refresh}
onCommit={handleCommit}
isExpanded={visualExpanded}
remoteDiffStats={diffStatusData}
chatInputRef={editing.chatInputRef}
/>
),
},
]}
onClose={() => onSetShowSidebarPanel(false)}
isExpanded={visualExpanded}
<RightPanel
isOpen={shouldShowSidebar}
isExpanded={isRightPanelExpanded}
onToggleExpanded={() => setIsRightPanelExpanded((prev) => !prev)}
onClose={() => onSetShowSidebarPanel(false)}
onVisualExpandedChange={setDragVisualExpanded}
isSidebarCollapsed={isSidebarCollapsed}
onToggleSidebarCollapsed={onToggleSidebarCollapsed}
chatTitle={chatTitle}
desktopChatId={desktopChatId}
/>
</RightPanel>
</div>
>
<SidebarTabView
activeTabId={sidebarTabId}
onActiveTabChange={setSidebarTabId}
tabs={[
{
id: "git",
label: "Git",
content: (
<GitPanel
prTab={
prNumber && agentId
? { prNumber, chatId: agentId }
: undefined
}
repositories={gitWatcher.repositories}
onRefresh={gitWatcher.refresh}
onCommit={handleCommit}
isExpanded={visualExpanded}
remoteDiffStats={diffStatusData}
chatInputRef={editing.chatInputRef}
/>
),
},
]}
onClose={() => onSetShowSidebarPanel(false)}
isExpanded={visualExpanded}
onToggleExpanded={() => setIsRightPanelExpanded((prev) => !prev)}
isSidebarCollapsed={isSidebarCollapsed}
onToggleSidebarCollapsed={onToggleSidebarCollapsed}
chatTitle={chatTitle}
desktopChatId={desktopChatId}
/>
</RightPanel>
</div>
</DesktopPanelContext>
);
};
@@ -36,6 +36,8 @@ const meta: Meta<typeof SidebarTabView> = {
component: SidebarTabView,
args: {
tabs: [gitTab],
activeTabId: "git",
onActiveTabChange: fn(),
isExpanded: false,
onToggleExpanded: fn(),
},
@@ -42,6 +42,10 @@ interface SidebarTabViewProps {
onClose?: () => void;
/** Desktop chat ID. Omitted if desktop is not available. */
desktopChatId?: string;
/** The currently active tab ID (controlled by the parent). */
activeTabId: string | null;
/** Called when the user switches tabs. */
onActiveTabChange: (tabId: string) => void;
}
/** How far (px) each chevron click scrolls the tab strip. */
@@ -107,12 +111,10 @@ export const SidebarTabView: FC<SidebarTabViewProps> = ({
chatTitle,
onClose,
desktopChatId,
activeTabId,
onActiveTabChange,
}) => {
const tabIdPrefix = useId();
const [activeTabId, setActiveTabId] = useState<string | null>(
tabs.length > 0 ? tabs[0].id : null,
);
// Build the full list of tab IDs including the desktop tab
// so that effectiveTabId validation covers it.
const allTabIds = new Set(tabs.map((t) => t.id));
@@ -227,7 +229,7 @@ export const SidebarTabView: FC<SidebarTabViewProps> = ({
id={`${tabIdPrefix}-tab-${tab.id}`}
role="tab"
aria-selected={isActive}
onClick={() => setActiveTabId(tab.id)}
onClick={() => onActiveTabChange(tab.id)}
variant="outline"
size="lg"
className={cn(
@@ -257,7 +259,7 @@ export const SidebarTabView: FC<SidebarTabViewProps> = ({
id={`${tabIdPrefix}-tab-desktop`}
role="tab"
aria-selected={effectiveTabId === "desktop"}
onClick={() => setActiveTabId("desktop")}
onClick={() => onActiveTabChange("desktop")}
variant="outline"
size="lg"
className={cn(
@@ -15,7 +15,7 @@ type DesktopConnectionStatus =
| "disconnected"
| "error";
interface UseDesktopConnectionResult {
export interface UseDesktopConnectionResult {
/** Current connection status. */
status: DesktopConnectionStatus;
/** Whether the connection has ever been established. */