From 9ac67a5253ed0aa192477eb8f7e3d1c5530efe18 Mon Sep 17 00:00:00 2001 From: Hugo Dutka Date: Thu, 2 Apr 2026 19:52:05 +0200 Subject: [PATCH] feat(site): agents desktop recordings frontend (#23895) This PR modifies the `wait_agent` tool call card to display screen recordings of computer use subagents. The backend logic was added in https://github.com/coder/coder/pull/23894. There's one big inefficiency in the current implementation: to display video thumbnails, the frontend downloads the entire video files from the backend. Our backend does not support HTTP range requests to only fetch the first frame. I'll be fixing that in a later PR. https://github.com/user-attachments/assets/684cea8b-66a9-45f8-96b2-57433da41c1c --- site/.storybook/main.ts | 2 +- site/.storybook/static/tiny-recording.mp4 | Bin 0 -> 1582 bytes .../tools/InlineDesktopPreview.tsx | 7 +- .../tools/RecordingPreview.stories.tsx | 74 ++++++++++++++++ .../ChatElements/tools/RecordingPreview.tsx | 77 ++++++++++++++++ .../ChatElements/tools/SubagentTool.tsx | 82 ++++++++++++++---- .../ChatElements/tools/Tool.stories.tsx | 54 ++++++++++++ .../components/ChatElements/tools/Tool.tsx | 2 + .../ChatElements/tools/previewConstants.ts | 8 ++ .../components/VideoLightbox.stories.tsx | 71 +++++++++++++++ .../AgentsPage/components/VideoLightbox.tsx | 41 +++++++++ 11 files changed, 393 insertions(+), 25 deletions(-) create mode 100644 site/.storybook/static/tiny-recording.mp4 create mode 100644 site/src/pages/AgentsPage/components/ChatElements/tools/RecordingPreview.stories.tsx create mode 100644 site/src/pages/AgentsPage/components/ChatElements/tools/RecordingPreview.tsx create mode 100644 site/src/pages/AgentsPage/components/ChatElements/tools/previewConstants.ts create mode 100644 site/src/pages/AgentsPage/components/VideoLightbox.stories.tsx create mode 100644 site/src/pages/AgentsPage/components/VideoLightbox.tsx diff --git a/site/.storybook/main.ts b/site/.storybook/main.ts index 299987770d..84276d740e 100644 --- a/site/.storybook/main.ts +++ b/site/.storybook/main.ts @@ -11,7 +11,7 @@ export default { "@storybook/addon-vitest", ], - staticDirs: ["../static"], + staticDirs: ["../static", "./static"], framework: { name: "@storybook/react-vite", diff --git a/site/.storybook/static/tiny-recording.mp4 b/site/.storybook/static/tiny-recording.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6d002c73ed2685c4d72084a53b99f23cb03c3395 GIT binary patch literal 1582 zcmah}&x<2P6s}C1*~Jx9Sk^g=QsN38Ha*>)Om-Y--EmmVLBWIWVO2t^sJfQR)WdQrRxdm6vjNoL|G_(=8p^}VlN*Q*X= zjK_9brlwGgtuWG9zLNfJRXN%L1Y6>Bj1oytEPgwOcbPLcm* zt)P!LtrkNh-?GDc&*lFqjI4p)keRV{WY+17x2|n` zr=nCK$5nLp-F<}D*c;#0m8~So^qhlD(nv0B(l_82J&{M5BuwKSmF6*wixX9oF1L9( zn&mbkEeS(H`A5b&<1@SQg0X`P_FZGmVjzV1f`$f7vdlS)9OAWmFlmwe8!?NYKG%NgMzXQ7iAT?fi@ z%eSIISBNeWJqLizS6{pSa;w94Z|ngX$qsMqVcM83~NxL|SRT(fvY11M%J3JHv5e6>D6${!tl7p-Whk>`_?zmj!BAXf+xE1`$ z^E|F(kjRRg@c`CUUYcp((X+3kz;j#_;lWi}$*50^RMFQW-(SGKd~jOp`0Qe_*)CJkIxz|H=)dBIJQB@rJb~(Q_<2SDAHm7q+`nKFbH+b z6}}NkoxutGw&U^<^%J-buEeM$;#!L;=su5tv9b(0JS}m~m?_2pE-}IgX&~%2F5!#v y{r4{Xxqir=`(T4TbpEpsKbgy?;~(Ds|KEL3tnJ>}e&YULu+Ht?d407CZvPF#;bt2E literal 0 HcmV?d00001 diff --git a/site/src/pages/AgentsPage/components/ChatElements/tools/InlineDesktopPreview.tsx b/site/src/pages/AgentsPage/components/ChatElements/tools/InlineDesktopPreview.tsx index 13395d5473..64f9b935b9 100644 --- a/site/src/pages/AgentsPage/components/ChatElements/tools/InlineDesktopPreview.tsx +++ b/site/src/pages/AgentsPage/components/ChatElements/tools/InlineDesktopPreview.tsx @@ -6,12 +6,7 @@ import { type UseDesktopConnectionResult, useDesktopConnection, } from "#/pages/AgentsPage/hooks/useDesktopConnection"; - -/** Default aspect ratio used before the remote framebuffer size is known. */ -const DEFAULT_ASPECT = "16 / 9"; - -/** Fixed pixel height for the compact preview thumbnail. */ -const PREVIEW_HEIGHT = 128; +import { DEFAULT_ASPECT, PREVIEW_HEIGHT } from "./previewConstants"; /** * Non-interactive inline VNC desktop preview. The noVNC canvas is diff --git a/site/src/pages/AgentsPage/components/ChatElements/tools/RecordingPreview.stories.tsx b/site/src/pages/AgentsPage/components/ChatElements/tools/RecordingPreview.stories.tsx new file mode 100644 index 0000000000..24cd6b96d0 --- /dev/null +++ b/site/src/pages/AgentsPage/components/ChatElements/tools/RecordingPreview.stories.tsx @@ -0,0 +1,74 @@ +import type { Meta, StoryObj } from "@storybook/react-vite"; +import { expect, fireEvent, userEvent, waitFor, within } from "storybook/test"; +import { RecordingPreview } from "./RecordingPreview"; + +// The file is stored in site/.storybook/static/tiny-recording.mp4. +const TINY_MP4 = "/tiny-recording.mp4"; + +const meta: Meta = { + title: "pages/AgentsPage/ChatElements/tools/RecordingPreview", + component: RecordingPreview, + decorators: [ + (Story) => ( +
+ +
+ ), + ], +}; + +export default meta; +type Story = StoryObj; + +export const Default: Story = { + args: { + recordingFileId: "dummy-recording-id", + src: TINY_MP4, + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + expect( + canvas.getByRole("button", { name: "View recording" }), + ).toBeInTheDocument(); + }, +}; + +export const LightboxOpen: Story = { + args: { + recordingFileId: "dummy-recording-id", + src: TINY_MP4, + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + await userEvent.click( + canvas.getByRole("button", { name: "View recording" }), + ); + const doc = canvasElement.ownerDocument; + await waitFor(() => { + const video = doc.querySelector("dialog video, [role='dialog'] video"); + expect(video).toBeInTheDocument(); + expect(video).toHaveAttribute("controls"); + }); + }, +}; + +export const ThumbnailError: Story = { + args: { + recordingFileId: "dummy-recording-id", + src: TINY_MP4, + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + const video = canvasElement.querySelector("video"); + expect(video).not.toBeNull(); + fireEvent.error(video!); + await waitFor(() => { + expect(canvas.getByText("Thumbnail unavailable")).toBeInTheDocument(); + // The play button should still be available so the user can + // attempt to view the recording even when the thumbnail fails. + expect( + canvas.getByRole("button", { name: "View recording" }), + ).toBeInTheDocument(); + }); + }, +}; diff --git a/site/src/pages/AgentsPage/components/ChatElements/tools/RecordingPreview.tsx b/site/src/pages/AgentsPage/components/ChatElements/tools/RecordingPreview.tsx new file mode 100644 index 0000000000..8ce3602944 --- /dev/null +++ b/site/src/pages/AgentsPage/components/ChatElements/tools/RecordingPreview.tsx @@ -0,0 +1,77 @@ +import { ImageOffIcon, PlayIcon } from "lucide-react"; +import type React from "react"; +import { useState } from "react"; +import { VideoLightbox } from "../../VideoLightbox"; +import { DEFAULT_ASPECT, PREVIEW_HEIGHT } from "./previewConstants"; + +interface RecordingPreviewProps { + /** The chat file ID for the MP4 recording. */ + recordingFileId: string; + /** Optional video URL override. When provided, this is used + * directly instead of deriving the URL from recordingFileId. */ + src?: string; +} + +/** + * Inline recording thumbnail with a play icon overlay. Clicking the + * preview opens a full-screen VideoLightbox with native playback + * controls. If the thumbnail fails to load, a "Thumbnail unavailable" + * message is shown but the video remains playable. + */ +export const RecordingPreview: React.FC = ({ + recordingFileId, + src: srcOverride, +}) => { + const [showLightbox, setShowLightbox] = useState(false); + const [thumbnailError, setThumbnailError] = useState(false); + // Incremented each time the lightbox opens so the VideoLightbox + // component remounts and resets its internal error state. + const [lightboxKey, setLightboxKey] = useState(0); + + const thumbnailSrc = + // Seek to first frame so the browser renders a thumbnail preview. + srcOverride ?? `/api/experimental/chats/files/${recordingFileId}#t=0.001`; + const videoSrc = + srcOverride ?? `/api/experimental/chats/files/${recordingFileId}`; + + return ( +
+ {thumbnailError ? ( +
+ + Thumbnail unavailable +
+ ) : ( +
+ ); +}; diff --git a/site/src/pages/AgentsPage/components/ChatElements/tools/SubagentTool.tsx b/site/src/pages/AgentsPage/components/ChatElements/tools/SubagentTool.tsx index 2c82085b69..3b4e938f62 100644 --- a/site/src/pages/AgentsPage/components/ChatElements/tools/SubagentTool.tsx +++ b/site/src/pages/AgentsPage/components/ChatElements/tools/SubagentTool.tsx @@ -16,6 +16,7 @@ import { Response } from "../Response"; import { Shimmer } from "../Shimmer"; import { useDesktopPanel } from "./DesktopPanelContext"; import { InlineDesktopPreview } from "./InlineDesktopPreview"; +import { RecordingPreview } from "./RecordingPreview"; import { isSubagentSuccessStatus, shortDurationMs, @@ -58,6 +59,53 @@ const SUBAGENT_VERBS: Record< }, }; +/** + * Returns the label JSX for a sub-agent tool row. Extracted to keep + * the rendering logic for the three label variants readable. + */ +function getSubagentLabel( + showDesktopPreview: boolean | undefined, + toolStatus: ToolStatus, + variant: "default" | "computer-use", + toolName: string, + title: string, + isTimeout: boolean, +): React.ReactNode { + if (showDesktopPreview && toolStatus === "running") { + return ( + + Using the computer... + + ); + } + if ( + variant === "computer-use" && + toolName === "wait_agent" && + toolStatus === "completed" + ) { + return ( + <> + Used the computer{" "} + {title} + + ); + } + return ( + <> + {SUBAGENT_VERBS[toolName]?.[ + isTimeout + ? "timeout" + : toolStatus === "completed" + ? "completed" + : toolStatus === "error" + ? "error" + : "running" + ] ?? ""} + {title} + + ); +} + /** * Resolves a sub-agent status string and tool-level status into a * display icon. The sub-agent status in the tool result is a @@ -122,6 +170,8 @@ export const SubagentTool: React.FC<{ /** Show an inline VNC desktop preview (for computer-use subagents). */ showDesktopPreview?: boolean; variant?: "default" | "computer-use"; + /** File ID for a completed recording (shown after tool completes). */ + recordingFileId?: string; }> = ({ toolName, title, @@ -136,6 +186,7 @@ export const SubagentTool: React.FC<{ isTimeout = false, showDesktopPreview, variant = "default", + recordingFileId, }) => { const [expanded, setExpanded] = useState(false); const { desktopChatId, onOpenDesktop } = useDesktopPanel(); @@ -166,23 +217,13 @@ export const SubagentTool: React.FC<{ showDesktopPreview={showDesktopPreview} />{" "} - {showDesktopPreview && toolStatus === "running" ? ( - - Using the computer... - - ) : ( - <> - {SUBAGENT_VERBS[toolName]?.[ - isTimeout - ? "timeout" - : toolStatus === "completed" - ? "completed" - : toolStatus === "error" - ? "error" - : "running" - ] ?? ""} - {title} - + {getSubagentLabel( + showDesktopPreview, + toolStatus, + variant, + toolName, + title, + isTimeout, )} {chatId && ( - {showDesktopPreview && desktopChatId && ( + {showDesktopPreview && desktopChatId && toolStatus !== "completed" && (
)} + {recordingFileId && toolStatus === "completed" && ( +
+ +
+ )} {expanded && hasPrompt && ( ( + + + + ), + ], + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + expect(canvas.queryByRole("button", { name: "View recording" })).toBeNull(); + expect( + canvas.queryByRole("button", { name: "Open desktop tab" }), + ).toBeNull(); + }, +}; + +export const WaitAgentComputerUseTimedOutNoRecording: Story = { + args: { + name: "wait_agent", + status: "error", + isError: true, + args: { chat_id: "desktop-child-1" }, + result: { + chat_id: "desktop-child-1", + title: "Set up environment", + status: "pending", + error: "timed out waiting for agent", + }, + computerUseSubagentIds: new Set(["desktop-child-1"]), + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + expect(canvas.queryByRole("button", { name: "View recording" })).toBeNull(); + }, +}; + // --------------------------------------------------------------------------- // read_skill stories // --------------------------------------------------------------------------- diff --git a/site/src/pages/AgentsPage/components/ChatElements/tools/Tool.tsx b/site/src/pages/AgentsPage/components/ChatElements/tools/Tool.tsx index b97060341b..084f00e8b5 100644 --- a/site/src/pages/AgentsPage/components/ChatElements/tools/Tool.tsx +++ b/site/src/pages/AgentsPage/components/ChatElements/tools/Tool.tsx @@ -305,6 +305,7 @@ const SubagentRenderer: FC = ({ ? asNumber(rec.duration_ms, { parseString: true }) : undefined; const report = rec ? asString(rec.report) : ""; + const recordingFileId = rec ? asString(rec.recording_file_id) : ""; const prompt = parsedArgs ? asString(parsedArgs.prompt) : ""; const subagentMessage = parsedArgs ? asString(parsedArgs.message) : ""; const title = @@ -366,6 +367,7 @@ const SubagentRenderer: FC = ({ showDesktopPreviews && computerUseSubagentIds?.has(chatId) } variant={variant} + recordingFileId={recordingFileId || undefined} /> ); }; diff --git a/site/src/pages/AgentsPage/components/ChatElements/tools/previewConstants.ts b/site/src/pages/AgentsPage/components/ChatElements/tools/previewConstants.ts new file mode 100644 index 0000000000..7231a38a86 --- /dev/null +++ b/site/src/pages/AgentsPage/components/ChatElements/tools/previewConstants.ts @@ -0,0 +1,8 @@ +/** Fixed pixel height for the compact preview thumbnail. */ +export const PREVIEW_HEIGHT = 128; + +/** Default aspect ratio used before the actual media dimensions are known. */ +export const DEFAULT_ASPECT = "16 / 9"; + +/** Placeholder text shown when a recording cannot be loaded. */ +export const RECORDING_UNAVAILABLE_TEXT = "Recording unavailable"; diff --git a/site/src/pages/AgentsPage/components/VideoLightbox.stories.tsx b/site/src/pages/AgentsPage/components/VideoLightbox.stories.tsx new file mode 100644 index 0000000000..812197e2dd --- /dev/null +++ b/site/src/pages/AgentsPage/components/VideoLightbox.stories.tsx @@ -0,0 +1,71 @@ +import type { Meta, StoryObj } from "@storybook/react-vite"; +import { expect, fireEvent, fn, waitFor, within } from "storybook/test"; +import { RECORDING_UNAVAILABLE_TEXT } from "./ChatElements/tools/previewConstants"; +import { VideoLightbox } from "./VideoLightbox"; + +// The file is stored in site/.storybook/static/tiny-recording.mp4. +const TINY_MP4 = "/tiny-recording.mp4"; + +const meta: Meta = { + title: "components/VideoLightbox", + component: VideoLightbox, + decorators: [ + (Story) => ( +
+

Background content behind the lightbox overlay

+ +
+ ), + ], +}; + +export default meta; +type Story = StoryObj; + +export const Default: Story = { + args: { + src: TINY_MP4, + open: true, + onClose: fn(), + }, + play: async ({ canvasElement }) => { + const doc = canvasElement.ownerDocument; + const video = doc.querySelector("video"); + expect(video).toBeInTheDocument(); + expect(video).toHaveAttribute("controls"); + }, +}; + +export const AccessibleTitle: Story = { + args: { + src: TINY_MP4, + open: true, + onClose: fn(), + }, + play: async ({ canvasElement }) => { + const screen = within(canvasElement.ownerDocument.body); + expect( + screen.getByRole("dialog", { name: "Recording playback" }), + ).toBeInTheDocument(); + }, +}; + +export const VideoError: Story = { + args: { + src: TINY_MP4, + open: true, + onClose: fn(), + }, + play: async ({ canvasElement }) => { + const doc = canvasElement.ownerDocument; + const screen = within(doc.body); + const video = doc.querySelector("video"); + expect(video).not.toBeNull(); + fireEvent.error(video!); + await waitFor(() => { + expect(screen.getByText(RECORDING_UNAVAILABLE_TEXT)).toBeInTheDocument(); + }); + // The video element should be replaced by the error message. + expect(doc.querySelector("video")).toBeNull(); + }, +}; diff --git a/site/src/pages/AgentsPage/components/VideoLightbox.tsx b/site/src/pages/AgentsPage/components/VideoLightbox.tsx new file mode 100644 index 0000000000..28e75c1461 --- /dev/null +++ b/site/src/pages/AgentsPage/components/VideoLightbox.tsx @@ -0,0 +1,41 @@ +import { type FC, useState } from "react"; +import { Dialog, DialogContent, DialogTitle } from "#/components/Dialog/Dialog"; +import { RECORDING_UNAVAILABLE_TEXT } from "./ChatElements/tools/previewConstants"; + +interface VideoLightboxProps { + src: string; + open: boolean; + onClose: () => void; +} + +export const VideoLightbox: FC = ({ + src, + open, + onClose, +}) => { + const [videoError, setVideoError] = useState(false); + + return ( + !o && onClose()}> + + Recording playback + {videoError ? ( +
+ {RECORDING_UNAVAILABLE_TEXT} +
+ ) : ( + // biome-ignore lint/a11y/useMediaCaption: Screen recordings do not have caption tracks. + + + ); +};