feat(site): agents desktop recordings frontend (#23895)

This PR modifies the `wait_agent` tool call card to display screen
recordings of computer use subagents. The backend logic was added in
https://github.com/coder/coder/pull/23894.

There's one big inefficiency in the current implementation: to display
video thumbnails, the frontend downloads the entire video files from the
backend. Our backend does not support HTTP range requests to only fetch
the first frame. I'll be fixing that in a later PR.


https://github.com/user-attachments/assets/684cea8b-66a9-45f8-96b2-57433da41c1c

site/.storybook/main.ts

@@ -11,7 +11,7 @@ export default {
 		"@storybook/addon-vitest",
 	],
 
-	staticDirs: ["../static"],
+	staticDirs: ["../static", "./static"],
 
 	framework: {
 		name: "@storybook/react-vite",

site/src/pages/AgentsPage/components/ChatElements/tools/InlineDesktopPreview.tsx

@@ -6,12 +6,7 @@ import {
 	type UseDesktopConnectionResult,
 	useDesktopConnection,
 } from "#/pages/AgentsPage/hooks/useDesktopConnection";
-
-/** Default aspect ratio used before the remote framebuffer size is known. */
-const DEFAULT_ASPECT = "16 / 9";
-
-/** Fixed pixel height for the compact preview thumbnail. */
-const PREVIEW_HEIGHT = 128;
+import { DEFAULT_ASPECT, PREVIEW_HEIGHT } from "./previewConstants";
 
 /**
  * Non-interactive inline VNC desktop preview. The noVNC canvas is

site/src/pages/AgentsPage/components/ChatElements/tools/RecordingPreview.stories.tsx

@@ -0,0 +1,74 @@
+import type { Meta, StoryObj } from "@storybook/react-vite";
+import { expect, fireEvent, userEvent, waitFor, within } from "storybook/test";
+import { RecordingPreview } from "./RecordingPreview";
+
+// The file is stored in site/.storybook/static/tiny-recording.mp4.
+const TINY_MP4 = "/tiny-recording.mp4";
+
+const meta: Meta<typeof RecordingPreview> = {
+	title: "pages/AgentsPage/ChatElements/tools/RecordingPreview",
+	component: RecordingPreview,
+	decorators: [
+		(Story) => (
+			<div className="max-w-3xl rounded-lg border border-solid border-border-default bg-surface-primary p-4">
+				<Story />
+			</div>
+		),
+	],
+};
+
+export default meta;
+type Story = StoryObj<typeof RecordingPreview>;
+
+export const Default: Story = {
+	args: {
+		recordingFileId: "dummy-recording-id",
+		src: TINY_MP4,
+	},
+	play: async ({ canvasElement }) => {
+		const canvas = within(canvasElement);
+		expect(
+			canvas.getByRole("button", { name: "View recording" }),
+		).toBeInTheDocument();
+	},
+};
+
+export const LightboxOpen: Story = {
+	args: {
+		recordingFileId: "dummy-recording-id",
+		src: TINY_MP4,
+	},
+	play: async ({ canvasElement }) => {
+		const canvas = within(canvasElement);
+		await userEvent.click(
+			canvas.getByRole("button", { name: "View recording" }),
+		);
+		const doc = canvasElement.ownerDocument;
+		await waitFor(() => {
+			const video = doc.querySelector("dialog video, [role='dialog'] video");
+			expect(video).toBeInTheDocument();
+			expect(video).toHaveAttribute("controls");
+		});
+	},
+};
+
+export const ThumbnailError: Story = {
+	args: {
+		recordingFileId: "dummy-recording-id",
+		src: TINY_MP4,
+	},
+	play: async ({ canvasElement }) => {
+		const canvas = within(canvasElement);
+		const video = canvasElement.querySelector("video");
+		expect(video).not.toBeNull();
+		fireEvent.error(video!);
+		await waitFor(() => {
+			expect(canvas.getByText("Thumbnail unavailable")).toBeInTheDocument();
+			// The play button should still be available so the user can
+			// attempt to view the recording even when the thumbnail fails.
+			expect(
+				canvas.getByRole("button", { name: "View recording" }),
+			).toBeInTheDocument();
+		});
+	},
+};

site/src/pages/AgentsPage/components/ChatElements/tools/RecordingPreview.tsx

@@ -0,0 +1,77 @@
+import { ImageOffIcon, PlayIcon } from "lucide-react";
+import type React from "react";
+import { useState } from "react";
+import { VideoLightbox } from "../../VideoLightbox";
+import { DEFAULT_ASPECT, PREVIEW_HEIGHT } from "./previewConstants";
+
+interface RecordingPreviewProps {
+	/** The chat file ID for the MP4 recording. */
+	recordingFileId: string;
+	/** Optional video URL override. When provided, this is used
+	 * directly instead of deriving the URL from recordingFileId. */
+	src?: string;
+}
+
+/**
+ * Inline recording thumbnail with a play icon overlay. Clicking the
+ * preview opens a full-screen VideoLightbox with native playback
+ * controls. If the thumbnail fails to load, a "Thumbnail unavailable"
+ * message is shown but the video remains playable.
+ */
+export const RecordingPreview: React.FC<RecordingPreviewProps> = ({
+	recordingFileId,
+	src: srcOverride,
+}) => {
+	const [showLightbox, setShowLightbox] = useState(false);
+	const [thumbnailError, setThumbnailError] = useState(false);
+	// Incremented each time the lightbox opens so the VideoLightbox
+	// component remounts and resets its internal error state.
+	const [lightboxKey, setLightboxKey] = useState(0);
+
+	const thumbnailSrc =
+		// Seek to first frame so the browser renders a thumbnail preview.
+		srcOverride ?? `/api/experimental/chats/files/${recordingFileId}#t=0.001`;
+	const videoSrc =
+		srcOverride ?? `/api/experimental/chats/files/${recordingFileId}`;
+
+	return (
+		<div
+			className="relative overflow-hidden rounded-lg border border-solid border-border-default"
+			style={{ aspectRatio: DEFAULT_ASPECT, height: PREVIEW_HEIGHT }}
+		>
+			{thumbnailError ? (
+				<div className="flex h-full w-full items-center justify-center gap-1.5 bg-surface-secondary text-xs text-content-secondary">
+					<ImageOffIcon className="h-3 w-3" />
+					Thumbnail unavailable
+				</div>
+			) : (
+				<video
+					src={thumbnailSrc}
+					preload="metadata"
+					muted
+					className="h-full w-full pointer-events-none object-cover"
+					onError={() => setThumbnailError(true)}
+				/>
+			)}
+			<button
+				type="button"
+				aria-label="View recording"
+				onClick={() => {
+					setShowLightbox(true);
+					setLightboxKey((k) => k + 1);
+				}}
+				className="absolute inset-0 z-10 flex cursor-pointer items-center justify-center border-0 bg-black/0 p-0 transition-colors hover:bg-black/50"
+			>
+				<span className="flex h-10 w-10 items-center justify-center rounded-full bg-black/60">
+					<PlayIcon className="h-5 w-5 text-white" />
+				</span>
+			</button>
+			<VideoLightbox
+				key={lightboxKey}
+				src={videoSrc}
+				open={showLightbox}
+				onClose={() => setShowLightbox(false)}
+			/>
+		</div>
+	);
+};

site/src/pages/AgentsPage/components/ChatElements/tools/SubagentTool.tsx

@@ -16,6 +16,7 @@ import { Response } from "../Response";
 import { Shimmer } from "../Shimmer";
 import { useDesktopPanel } from "./DesktopPanelContext";
 import { InlineDesktopPreview } from "./InlineDesktopPreview";
+import { RecordingPreview } from "./RecordingPreview";
 import {
 	isSubagentSuccessStatus,
 	shortDurationMs,
@@ -58,6 +59,53 @@ const SUBAGENT_VERBS: Record<
 	},
 };
 
+/**
+ * Returns the label JSX for a sub-agent tool row. Extracted to keep
+ * the rendering logic for the three label variants readable.
+ */
+function getSubagentLabel(
+	showDesktopPreview: boolean | undefined,
+	toolStatus: ToolStatus,
+	variant: "default" | "computer-use",
+	toolName: string,
+	title: string,
+	isTimeout: boolean,
+): React.ReactNode {
+	if (showDesktopPreview && toolStatus === "running") {
+		return (
+			<Shimmer as="span" className="text-sm">
+				Using the computer...
+			</Shimmer>
+		);
+	}
+	if (
+		variant === "computer-use" &&
+		toolName === "wait_agent" &&
+		toolStatus === "completed"
+	) {
+		return (
+			<>
+				Used the computer{" "}
+				<span className="text-content-secondary opacity-60">{title}</span>
+			</>
+		);
+	}
+	return (
+		<>
+			{SUBAGENT_VERBS[toolName]?.[
+				isTimeout
+					? "timeout"
+					: toolStatus === "completed"
+						? "completed"
+						: toolStatus === "error"
+							? "error"
+							: "running"
+			] ?? ""}
+			<span className="text-content-secondary opacity-60">{title}</span>
+		</>
+	);
+}
+
 /**
  * Resolves a sub-agent status string and tool-level status into a
  * display icon. The sub-agent status in the tool result is a
@@ -122,6 +170,8 @@ export const SubagentTool: React.FC<{
 	/** Show an inline VNC desktop preview (for computer-use subagents). */
 	showDesktopPreview?: boolean;
 	variant?: "default" | "computer-use";
+	/** File ID for a completed recording (shown after tool completes). */
+	recordingFileId?: string;
 }> = ({
 	toolName,
 	title,
@@ -136,6 +186,7 @@ export const SubagentTool: React.FC<{
 	isTimeout = false,
 	showDesktopPreview,
 	variant = "default",
+	recordingFileId,
 }) => {
 	const [expanded, setExpanded] = useState(false);
 	const { desktopChatId, onOpenDesktop } = useDesktopPanel();
@@ -166,23 +217,13 @@ export const SubagentTool: React.FC<{
 					showDesktopPreview={showDesktopPreview}
 				/>{" "}
 				<span className="min-w-0 flex-1 truncate text-sm text-content-secondary">
-					{showDesktopPreview && toolStatus === "running" ? (
-						<Shimmer as="span" className="text-sm">
-							Using the computer...
-						</Shimmer>
-					) : (
-						<>
-							{SUBAGENT_VERBS[toolName]?.[
-								isTimeout
-									? "timeout"
-									: toolStatus === "completed"
-										? "completed"
-										: toolStatus === "error"
-											? "error"
-											: "running"
-							] ?? ""}
-							<span className="text-content-secondary opacity-60">{title}</span>
-						</>
+					{getSubagentLabel(
+						showDesktopPreview,
+						toolStatus,
+						variant,
+						toolName,
+						title,
+						isTimeout,
 					)}
 					{chatId && (
 						<Link
@@ -210,7 +251,7 @@ export const SubagentTool: React.FC<{
 				)}
 			</button>
 
-			{showDesktopPreview && desktopChatId && (
+			{showDesktopPreview && desktopChatId && toolStatus !== "completed" && (
 				<div className="mt-1.5 w-fit overflow-hidden rounded-lg border border-solid border-border-default">
 					<InlineDesktopPreview
 						chatId={desktopChatId}
@@ -219,6 +260,11 @@ export const SubagentTool: React.FC<{
 				</div>
 			)}
 
+			{recordingFileId && toolStatus === "completed" && (
+				<div className="mt-1.5 w-fit">
+					<RecordingPreview recordingFileId={recordingFileId} />
+				</div>
+			)}
 			{expanded && hasPrompt && (
 				<ScrollArea
 					className="mt-1.5 rounded-md border border-solid border-border-default"

site/src/pages/AgentsPage/components/ChatElements/tools/Tool.stories.tsx

@@ -1258,6 +1258,60 @@ export const WaitAgentComputerUseRunning: Story = {
 	},
 };
 
+export const WaitAgentComputerUseCompletedNoRecording: Story = {
+	args: {
+		name: "wait_agent",
+		status: "completed",
+		args: { chat_id: "desktop-child-1" },
+		result: {
+			chat_id: "desktop-child-1",
+			title: "Set up environment",
+			status: "waiting",
+			report: "Configured the dev environment.",
+		},
+		computerUseSubagentIds: new Set(["desktop-child-1"]),
+	},
+	decorators: [
+		(Story) => (
+			<DesktopPanelContext.Provider
+				value={{
+					desktopChatId: "desktop-child-1",
+					onOpenDesktop: fn(),
+				}}
+			>
+				<Story />
+			</DesktopPanelContext.Provider>
+		),
+	],
+	play: async ({ canvasElement }) => {
+		const canvas = within(canvasElement);
+		expect(canvas.queryByRole("button", { name: "View recording" })).toBeNull();
+		expect(
+			canvas.queryByRole("button", { name: "Open desktop tab" }),
+		).toBeNull();
+	},
+};
+
+export const WaitAgentComputerUseTimedOutNoRecording: Story = {
+	args: {
+		name: "wait_agent",
+		status: "error",
+		isError: true,
+		args: { chat_id: "desktop-child-1" },
+		result: {
+			chat_id: "desktop-child-1",
+			title: "Set up environment",
+			status: "pending",
+			error: "timed out waiting for agent",
+		},
+		computerUseSubagentIds: new Set(["desktop-child-1"]),
+	},
+	play: async ({ canvasElement }) => {
+		const canvas = within(canvasElement);
+		expect(canvas.queryByRole("button", { name: "View recording" })).toBeNull();
+	},
+};
+
 // ---------------------------------------------------------------------------
 // read_skill stories
 // ---------------------------------------------------------------------------

site/src/pages/AgentsPage/components/ChatElements/tools/Tool.tsx

@@ -305,6 +305,7 @@ const SubagentRenderer: FC<ToolRendererProps> = ({
 		? asNumber(rec.duration_ms, { parseString: true })
 		: undefined;
 	const report = rec ? asString(rec.report) : "";
+	const recordingFileId = rec ? asString(rec.recording_file_id) : "";
 	const prompt = parsedArgs ? asString(parsedArgs.prompt) : "";
 	const subagentMessage = parsedArgs ? asString(parsedArgs.message) : "";
 	const title =
@@ -366,6 +367,7 @@ const SubagentRenderer: FC<ToolRendererProps> = ({
 				showDesktopPreviews && computerUseSubagentIds?.has(chatId)
 			}
 			variant={variant}
+			recordingFileId={recordingFileId || undefined}
 		/>
 	);
 };

site/src/pages/AgentsPage/components/ChatElements/tools/previewConstants.ts

@@ -0,0 +1,8 @@
+/** Fixed pixel height for the compact preview thumbnail. */
+export const PREVIEW_HEIGHT = 128;
+
+/** Default aspect ratio used before the actual media dimensions are known. */
+export const DEFAULT_ASPECT = "16 / 9";
+
+/** Placeholder text shown when a recording cannot be loaded. */
+export const RECORDING_UNAVAILABLE_TEXT = "Recording unavailable";

site/src/pages/AgentsPage/components/VideoLightbox.stories.tsx

@@ -0,0 +1,71 @@
+import type { Meta, StoryObj } from "@storybook/react-vite";
+import { expect, fireEvent, fn, waitFor, within } from "storybook/test";
+import { RECORDING_UNAVAILABLE_TEXT } from "./ChatElements/tools/previewConstants";
+import { VideoLightbox } from "./VideoLightbox";
+
+// The file is stored in site/.storybook/static/tiny-recording.mp4.
+const TINY_MP4 = "/tiny-recording.mp4";
+
+const meta: Meta<typeof VideoLightbox> = {
+	title: "components/VideoLightbox",
+	component: VideoLightbox,
+	decorators: [
+		(Story) => (
+			<div className="flex min-h-64 items-center justify-center p-8 text-content-primary">
+				<p>Background content behind the lightbox overlay</p>
+				<Story />
+			</div>
+		),
+	],
+};
+
+export default meta;
+type Story = StoryObj<typeof VideoLightbox>;
+
+export const Default: Story = {
+	args: {
+		src: TINY_MP4,
+		open: true,
+		onClose: fn(),
+	},
+	play: async ({ canvasElement }) => {
+		const doc = canvasElement.ownerDocument;
+		const video = doc.querySelector("video");
+		expect(video).toBeInTheDocument();
+		expect(video).toHaveAttribute("controls");
+	},
+};
+
+export const AccessibleTitle: Story = {
+	args: {
+		src: TINY_MP4,
+		open: true,
+		onClose: fn(),
+	},
+	play: async ({ canvasElement }) => {
+		const screen = within(canvasElement.ownerDocument.body);
+		expect(
+			screen.getByRole("dialog", { name: "Recording playback" }),
+		).toBeInTheDocument();
+	},
+};
+
+export const VideoError: Story = {
+	args: {
+		src: TINY_MP4,
+		open: true,
+		onClose: fn(),
+	},
+	play: async ({ canvasElement }) => {
+		const doc = canvasElement.ownerDocument;
+		const screen = within(doc.body);
+		const video = doc.querySelector("video");
+		expect(video).not.toBeNull();
+		fireEvent.error(video!);
+		await waitFor(() => {
+			expect(screen.getByText(RECORDING_UNAVAILABLE_TEXT)).toBeInTheDocument();
+		});
+		// The video element should be replaced by the error message.
+		expect(doc.querySelector("video")).toBeNull();
+	},
+};

site/src/pages/AgentsPage/components/VideoLightbox.tsx

@@ -0,0 +1,41 @@
+import { type FC, useState } from "react";
+import { Dialog, DialogContent, DialogTitle } from "#/components/Dialog/Dialog";
+import { RECORDING_UNAVAILABLE_TEXT } from "./ChatElements/tools/previewConstants";
+
+interface VideoLightboxProps {
+	src: string;
+	open: boolean;
+	onClose: () => void;
+}
+
+export const VideoLightbox: FC<VideoLightboxProps> = ({
+	src,
+	open,
+	onClose,
+}) => {
+	const [videoError, setVideoError] = useState(false);
+
+	return (
+		<Dialog open={open} onOpenChange={(o) => !o && onClose()}>
+			<DialogContent
+				className="max-h-[85vh] max-w-[90vw] w-fit border-0 bg-transparent p-0 shadow-none"
+				aria-describedby={undefined}
+			>
+				<DialogTitle className="sr-only">Recording playback</DialogTitle>
+				{videoError ? (
+					<div className="flex items-center justify-center rounded bg-surface-secondary p-8 text-sm text-content-secondary">
+						{RECORDING_UNAVAILABLE_TEXT}
+					</div>
+				) : (
+					// biome-ignore lint/a11y/useMediaCaption: Screen recordings do not have caption tracks.
+					<video
+						src={src}
+						controls
+						className="max-h-[85vh] max-w-[90vw] rounded"
+						onError={() => setVideoError(true)}
+					/>
+				)}
+			</DialogContent>
+		</Dialog>
+	);
+};