feat(site): enable copy and paste in agents desktop (#23686)

site/src/pages/AgentsPage/hooks/useDesktopConnection.test.ts

@@ -15,7 +15,9 @@ vi.mock("api/api", () => ({
 interface MockRFBInstance {
 	scaleViewport: boolean;
 	resizeSession: boolean;
+	clipboardPasteFrom: ReturnType<typeof vi.fn>;
 	disconnect: ReturnType<typeof vi.fn>;
+	sendKey: ReturnType<typeof vi.fn>;
 	addEventListener: ReturnType<typeof vi.fn>;
 	listeners: Map<string, (ev: unknown) => void>;
 	simulateEvent: (type: string, detail?: unknown) => void;
@@ -34,7 +36,9 @@ const { FakeRFB, lastInstance } = vi.hoisted(() => {
 
 		scaleViewport = false;
 		resizeSession = true;
+		clipboardPasteFrom = vi.fn();
 		disconnect = vi.fn();
+		sendKey = vi.fn();
 		addEventListener = vi.fn((type: string, handler: (ev: unknown) => void) => {
 			this.listeners.set(type, handler);
 		});
@@ -66,6 +70,8 @@ vi.mock("@novnc/novnc/lib/rfb", () => ({
 import { watchChatDesktop } from "#/api/api";
 
 const mockWatchChatDesktop = vi.mocked(watchChatDesktop);
+const mockClipboardReadText = vi.fn<() => Promise<string>>();
+const mockClipboardWriteText = vi.fn<(text: string) => Promise<void>>();
 
 // ---- Mock ResizeObserver ----------------------------------------------------
 
@@ -135,6 +141,17 @@ describe("useDesktopConnection", () => {
 		resizeObserverInstances = [];
 		globalThis.ResizeObserver =
 			MockResizeObserver as unknown as typeof ResizeObserver;
+		mockClipboardReadText.mockReset();
+		mockClipboardReadText.mockResolvedValue("");
+		mockClipboardWriteText.mockReset();
+		mockClipboardWriteText.mockResolvedValue();
+		Object.defineProperty(navigator, "clipboard", {
+			configurable: true,
+			value: {
+				readText: mockClipboardReadText,
+				writeText: mockClipboardWriteText,
+			},
+		});
 	});
 
 	afterEach(() => {
@@ -394,6 +411,21 @@ describe("useDesktopConnection", () => {
 		expect(mockWatchChatDesktop).toHaveBeenCalledTimes(1);
 	});
 
+	it("stores remote clipboard text when the server sends it", async () => {
+		const { result } = renderHook(() =>
+			useDesktopConnection({ chatId: "chat-1" }),
+		);
+
+		const rfb = getLastRFBInstance();
+		act(() => rfb.simulateEvent("connect"));
+		act(() => rfb.simulateEvent("clipboard", { text: "from remote" }));
+
+		expect(result.current.remoteClipboardText).toBe("from remote");
+		await vi.waitFor(() => {
+			expect(mockClipboardWriteText).toHaveBeenCalledWith("from remote");
+		});
+	});
+
 	it("does not retry when disconnect fires before connect (desktop unavailable)", () => {
 		vi.useFakeTimers();
 

site/src/pages/AgentsPage/hooks/useDesktopConnection.ts

@@ -1,5 +1,7 @@
 import RFB from "@novnc/novnc/lib/rfb";
+import { useClipboard } from "hooks/useClipboard";
 import { useEffect, useRef, useState } from "react";
+import { toast } from "sonner";
 import { watchChatDesktop } from "#/api/api";
 
 interface UseDesktopConnectionOptions {
@@ -30,6 +32,8 @@ interface UseDesktopConnectionResult {
 	 * without reconnecting.
 	 */
 	attach: (container: HTMLElement) => void;
+	/** Latest text received from the remote desktop clipboard. */
+	remoteClipboardText: string | null;
 	/** The underlying RFB instance, if connected. */
 	rfb: RFB | null;
 }
@@ -39,11 +43,48 @@ const MAX_RECONNECT_ATTEMPTS = 10;
 const STABLE_CONNECTION_MS = 3_000;
 const CONNECT_TIMEOUT_MS = 30_000;
 
+// X11 keysym values sent to the remote desktop via the RFB
+// protocol. Full list: https://www.cl.cam.ac.uk/~mgk25/ucs/keysymdef.h
+const XK_Control_L = 0xffe3; // Left Control modifier.
+const XK_Control_R = 0xffe4; // Right Control modifier.
+const XK_Meta_L = 0xffeb; // Left Meta (Cmd on macOS) modifier.
+const XK_Meta_R = 0xffec; // Right Meta modifier.
+const XK_c = 0x0063; // Latin lowercase 'c'.
+const XK_v = 0x0076; // Latin lowercase 'v'.
+const XK_x = 0x0078; // Latin lowercase 'x'.
+const XK_Shift_L = 0xffe1; // Left Shift modifier.
+const XK_Shift_R = 0xffe2; // Right Shift modifier.
+
+const isPasteShortcut = (event: KeyboardEvent): boolean => {
+	const key = event.key.toLowerCase();
+	return (
+		(key === "v" && (event.ctrlKey || event.metaKey) && !event.altKey) ||
+		(key === "insert" && event.shiftKey && !event.ctrlKey && !event.metaKey)
+	);
+};
+
+// Detect Cmd+C on macOS so we can remap it to Ctrl+C on the
+// remote desktop. Plain Ctrl+C already travels through noVNC
+// correctly, so we only need to intercept the Meta variant.
+const isMacCopyShortcut = (event: KeyboardEvent): boolean => {
+	const key = event.key.toLowerCase();
+	return key === "c" && event.metaKey && !event.ctrlKey && !event.altKey;
+};
+
+// Detect Cmd+X on macOS — same remapping rationale as copy.
+const isMacCutShortcut = (event: KeyboardEvent): boolean => {
+	const key = event.key.toLowerCase();
+	return key === "x" && event.metaKey && !event.ctrlKey && !event.altKey;
+};
+
 export function useDesktopConnection({
 	chatId,
 }: UseDesktopConnectionOptions): UseDesktopConnectionResult {
 	const [status, setStatus] = useState<DesktopConnectionStatus>("idle");
 	const [hasConnected, setHasConnected] = useState(false);
+	const [remoteClipboardText, setRemoteClipboardText] = useState<string | null>(
+		null,
+	);
 
 	// rfbRef provides synchronous access for cleanup and event
 	// handlers. rfbInstance (state) provides reactivity so consumers
@@ -74,6 +115,20 @@ export function useDesktopConnection({
 	const reconnect = () => {
 		restartRef.current?.();
 	};
+	const { copyToClipboard: syncRemoteClipboardToLocal } = useClipboard({
+		onError: () => {
+			toast.error(
+				"Failed to sync the remote clipboard to your local clipboard.",
+			);
+		},
+	});
+	// Stable ref so the effect can call the latest
+	// syncRemoteClipboardToLocal without listing it as a
+	// dependency (its identity may change across renders).
+	const syncClipboardRef = useRef(syncRemoteClipboardToLocal);
+	useEffect(() => {
+		syncClipboardRef.current = syncRemoteClipboardToLocal;
+	}, [syncRemoteClipboardToLocal]);
 
 	const attach = (container: HTMLElement) => {
 		const screen = offscreenContainerRef.current;
@@ -81,7 +136,6 @@ export function useDesktopConnection({
 			container.appendChild(screen);
 		}
 	};
-
 	// Single lifecycle effect that owns the entire connection.
 	// Connects on mount, tears down on unmount, and resets when
 	// chatId changes.
@@ -93,7 +147,24 @@ export function useDesktopConnection({
 	useEffect(() => {
 		let visibilityObserver: ResizeObserver | null = null;
 
+		setRemoteClipboardText(null);
+
+		let clipboardKeyListener: ((event: KeyboardEvent) => void) | null = null;
+		let clipboardKeyUpListener: ((event: KeyboardEvent) => void) | null = null;
+		const removeClipboardKeyListener = () => {
+			const screen = offscreenContainerRef.current;
+			if (screen && clipboardKeyListener) {
+				screen.removeEventListener("keydown", clipboardKeyListener, true);
+				clipboardKeyListener = null;
+			}
+			if (screen && clipboardKeyUpListener) {
+				screen.removeEventListener("keyup", clipboardKeyUpListener, true);
+				clipboardKeyUpListener = null;
+			}
+		};
+
 		const cleanupRfb = () => {
+			removeClipboardKeyListener();
 			if (rfbRef.current) {
 				try {
 					rfbRef.current.disconnect();
@@ -160,6 +231,7 @@ export function useDesktopConnection({
 			offscreenContainerRef.current = document.createElement("div");
 			offscreenContainerRef.current.style.width = "100%";
 			offscreenContainerRef.current.style.height = "100%";
+			offscreenContainerRef.current.style.position = "relative";
 
 			const socket = watchChatDesktop(chatId);
 
@@ -170,6 +242,7 @@ export function useDesktopConnection({
 
 				rfb.scaleViewport = true;
 				rfb.resizeSession = false;
+				rfb.focusOnClick = true;
 
 				// Per-session flags scoped to this RFB instance.
 				// NOT refs — each doConnect() gets fresh copies so
@@ -177,6 +250,122 @@ export function useDesktopConnection({
 				let sessionConnected = false;
 				let securityFailed = false;
 
+				rfb.addEventListener("clipboard", (event) => {
+					const text = event.detail.text ?? "";
+					if (gen !== generationRef.current || !sessionConnected) {
+						return;
+					}
+					setRemoteClipboardText(text);
+					syncClipboardRef.current(text).catch((err) => {
+						console.error("Failed to sync remote clipboard to local:", err);
+					});
+				});
+
+				// Capture-phase keydown on the container fires before
+				// noVNC's own handlers on the canvas. This lets us
+				// intercept clipboard shortcuts before noVNC swallows
+				// them without any hidden-textarea focus tricks.
+				// Track which keys were intercepted on keydown so
+				// the corresponding keyup events can be suppressed.
+				const interceptedKeys = new Set<string>();
+
+				clipboardKeyListener = (event) => {
+					if (gen !== generationRef.current) {
+						return;
+					}
+					if (!sessionConnected) {
+						return;
+					}
+					// Remap Cmd+C / Cmd+X on macOS to Ctrl+C / Ctrl+X
+					// on the remote desktop. Release the stale Meta
+					// modifier first because its keydown already reached
+					// the remote before we could intercept it.
+					if (isMacCopyShortcut(event) || isMacCutShortcut(event)) {
+						const targetKeysym = isMacCopyShortcut(event) ? XK_c : XK_x;
+						const keyName = isMacCopyShortcut(event) ? "KeyC" : "KeyX";
+						event.preventDefault();
+						event.stopPropagation();
+						interceptedKeys.add(event.code);
+						rfb.sendKey(XK_Meta_L, "MetaLeft", false);
+						rfb.sendKey(XK_Meta_R, "MetaRight", false);
+						rfb.sendKey(XK_Control_L, "ControlLeft", true);
+						rfb.sendKey(targetKeysym, keyName, true);
+						rfb.sendKey(targetKeysym, keyName, false);
+						rfb.sendKey(XK_Control_L, "ControlLeft", false);
+						return;
+					}
+
+					if (!isPasteShortcut(event)) {
+						return;
+					}
+					event.preventDefault();
+					event.stopPropagation();
+					interceptedKeys.add(event.code);
+
+					const doPaste = async () => {
+						// Check clipboard permission state before
+						// attempting readText(). This lets us show a
+						// clear message when permission is denied
+						// instead of silently hanging.
+						try {
+							const permStatus = await navigator.permissions.query({
+								name: "clipboard-read" as PermissionName,
+							});
+							if (permStatus.state === "denied") {
+								toast.error(
+									"Clipboard permission denied. Allow clipboard access in your browser settings to paste.",
+								);
+								return;
+							}
+						} catch {
+							// permissions.query may not support
+							// clipboard-read in all browsers.
+							// Continue with readText anyway.
+						}
+
+						const text = await navigator.clipboard.readText();
+						if (!text) {
+							return;
+						}
+						if (gen !== generationRef.current) {
+							return;
+						}
+						rfb.clipboardPasteFrom(text);
+						rfb.sendKey(XK_Shift_L, "ShiftLeft", false);
+						rfb.sendKey(XK_Shift_R, "ShiftRight", false);
+						rfb.sendKey(XK_Meta_L, "MetaLeft", false);
+						rfb.sendKey(XK_Meta_R, "MetaRight", false);
+						rfb.sendKey(XK_Control_L, "ControlLeft", false);
+						rfb.sendKey(XK_Control_R, "ControlRight", false);
+						rfb.sendKey(XK_Control_L, "ControlLeft", true);
+						rfb.sendKey(XK_v, "KeyV", true);
+						rfb.sendKey(XK_v, "KeyV", false);
+						rfb.sendKey(XK_Control_L, "ControlLeft", false);
+					};
+					doPaste().catch((err) => {
+						console.error("Paste into remote desktop failed:", err);
+					});
+				};
+
+				clipboardKeyUpListener = (event) => {
+					if (interceptedKeys.has(event.code)) {
+						interceptedKeys.delete(event.code);
+						event.preventDefault();
+						event.stopPropagation();
+					}
+				};
+
+				offscreenContainerRef.current.addEventListener(
+					"keydown",
+					clipboardKeyListener,
+					true,
+				);
+				offscreenContainerRef.current.addEventListener(
+					"keyup",
+					clipboardKeyUpListener,
+					true,
+				);
+
 				// Fail if the VNC handshake doesn't complete within
 				// a reasonable window. Bump the generation before
 				// cleanupRfb() so the resulting noVNC "disconnect"
@@ -318,6 +507,7 @@ export function useDesktopConnection({
 		hasConnected,
 		reconnect,
 		attach,
+		remoteClipboardText,
 		rfb: rfbInstance,
 	};
 }