diff --git a/apps/cli/src/runtime/run-interactive.ts b/apps/cli/src/runtime/run-interactive.ts index 037e09179a..cc6e0143b1 100644 --- a/apps/cli/src/runtime/run-interactive.ts +++ b/apps/cli/src/runtime/run-interactive.ts @@ -1,4 +1,5 @@ import { + createComputerUseToolFromEnv, getCurrentContextSize, type ProviderSettings, ProviderSettingsManager, @@ -258,7 +259,15 @@ export async function runInteractive( tuiModeChanged, }); - config.extraTools = config.mode === "plan" ? [switchToActModeTool] : []; + // Proof-of-concept computer-use support: only enabled when + // CLINE_COMPUTER_USE_PORT points at a running backend. See + // @cline/core's extensions/computer-use/README.md for the wire protocol. + const computerUseTool = await createComputerUseToolFromEnv(); + + config.extraTools = [ + ...(config.mode === "plan" ? [switchToActModeTool] : []), + ...(computerUseTool ? [computerUseTool] : []), + ]; const uiEvents = getUIEventEmitter(); const chatCommandState: ChatCommandState = { diff --git a/sdk/packages/core/src/extensions/computer-use/README.md b/sdk/packages/core/src/extensions/computer-use/README.md new file mode 100644 index 0000000000..aae923418b --- /dev/null +++ b/sdk/packages/core/src/extensions/computer-use/README.md @@ -0,0 +1,156 @@ +# Computer-use tool + +Bridges Anthropic's [computer-use tool](https://platform.claude.com/docs/en/agents-and-tools/tool-use/computer-use-tool) +to an external screen-capture/input backend (developed out-of-tree, in Rust) +over a small JSON-L socket protocol. + +This is a genuine `@cline/core` extension: `createComputerUseTool()` returns +a plain `AgentTool`, the same contract used by every other tool in +`sdk/packages/core/src/extensions/tools/`. Any host that builds a +`CoreSessionConfig` (CLI, the VSCode adapter, a future standalone script, +...) can add it via `config.extraTools` — see `createComputerUseToolFromEnv()` +in `env.ts` for the zero-config way to opt in from `CLINE_COMPUTER_USE_PORT`. + +This folder is deliberately isolated from the rest of `@cline/core`: + +- It only imports `AgentTool`/`AgentToolContext`/`createTool` from + `@cline/shared`. +- It has no dependency on MCP, the plugin-sandbox system, or any + host-specific services. + +This makes it straightforward to lift into a real out-of-tree Cline plugin +later (see `sdk/packages/core/src/extensions/plugin/`) without touching call +sites beyond wherever it's added to `extraTools`. + +## Why not MCP? + +MCP is Cline's real plugin/tool-integration mechanism, and it was +considered. It was not used here because: + +- MCP requires a JSON-RPC 2.0 handshake (`initialize`/`initialized`), + capability negotiation, and a tool-discovery round trip before the first + real call can be made. None of that is useful for a single, fixed tool + (`computer`) talking to a single, purpose-built backend process. +- MCP's transports (stdio framed messages, or SSE/HTTP) add either process + lifecycle management or an HTTP server to what is fundamentally a + low-latency "send an input event, get a screenshot back" loop, called on + essentially every model turn during a computer-use session. +- MCP tool names get namespaced (`server__tool`) and go through the generic + `McpHub`/`createMcpTools` plumbing (see `sdk/packages/core/src/extensions/mcp/`), + which is the right call for arbitrary third-party servers but is extra + indirection for a first-party, tightly-coupled bridge. + +A direct socket client keeps the round-trip cost to "one write, one read" +and keeps the protocol trivial to implement in Rust with just `tokio::net` +and `serde_json` — no MCP SDK dependency needed on the Rust side. If/when +this becomes a real out-of-tree plugin, it can still register as a normal +`AgentTool` from a plugin's `setup()`; moving away from MCP was about +avoiding protocol overhead, not about avoiding the plugin system. + +## Wire protocol + +- **Transport:** plain TCP to `127.0.0.1:` (loopback only). TCP is used + instead of a Unix domain socket / Windows named pipe so the exact same + client code works unmodified on Windows, macOS, and Linux. +- **Framing:** newline-delimited JSON ("JSON Lines" / JSON-L). Every request + or response is exactly one JSON value serialized on one line, terminated + by `\n`. No `Content-Length` headers, no multipart framing — this is the + simplest framing that still works correctly, since JSON string values + always escape embedded newlines. +- **Multiplexing:** every request carries a numeric `id`; the backend must + echo it back on the matching response. The client does not assume + in-order responses. + +### Request (Cline -> backend) + +```jsonc +{ + "id": 1, + "action": "screenshot" | "cursor_position" | "mouse_move" | "left_click" | + "left_click_drag" | "right_click" | "middle_click" | + "double_click" | "triple_click" | "left_mouse_down" | + "left_mouse_up" | "key" | "hold_key" | "type" | "scroll" | + "wait" | "zoom" | + // Internal query, not one of Anthropic's `computer` tool + // actions. Sent once at startup to build the tool's + // description/schema with the real, native display size + // instead of a guessed default. See "Display size" below. + "get_display_info", + "coordinate": [x, y], // optional, pixel coordinate + "startCoordinate": [x, y], // optional, for left_click_drag + "text": "hello world", // optional, for "type" + "keys": "ctrl+alt+delete", // optional, for "key" / "hold_key" + "durationSeconds": 0.5, // optional, for "hold_key" / "wait" + "scrollDirection": "down", // optional, for "scroll" + "scrollAmount": 3, // optional, for "scroll" + "region": [x, y, width, height] // optional, for "zoom" +} +``` + +### Response (backend -> Cline) + +```jsonc +{ + "id": 1, + "ok": true, + "text": "optional human-readable result (e.g. cursor position)", + "image": { // present for "screenshot" and any + "data": "", // action that returns a fresh screenshot + "mediaType": "image/png" + }, + "display": { // present for "get_display_info" + "widthPx": 1920, + "heightPx": 1080 + } +} +``` + +On failure: + +```jsonc +{ "id": 1, "ok": false, "error": "description of what went wrong" } +``` + +See `protocol.ts` for the exact TypeScript types and `client.ts` for the +client-side framing/pending-request implementation. + +## Display size + +The `computer` tool's description embeds the display's pixel dimensions, +which Anthropic's model uses to reason about coordinates. Since the backend +is the one actually capturing the screen, it — not Cline — is the source of +truth for those dimensions. + +`createComputerUseTool()` is therefore `async`: unless both +`displayWidthPx`/`displayHeightPx` are passed explicitly, it calls +`ComputerUseClient.getDisplayInfo()` (a `"get_display_info"` request) once at +construction time and uses the backend's reported size. A single override +(e.g. only `displayWidthPx`) still queries the backend for the other +dimension. + +## Opting in from environment variables + +`createComputerUseToolFromEnv()` (`env.ts`) returns a ready-to-use tool (or +`undefined` if computer-use isn't configured for the current process) and is +also `async` for the same reason: + +| Variable | Required | Default | Meaning | +| --- | --- | --- | --- | +| `CLINE_COMPUTER_USE_PORT` | yes (enables the tool) | — | Backend TCP port | +| `CLINE_COMPUTER_USE_HOST` | no | `127.0.0.1` | Backend host | +| `CLINE_COMPUTER_USE_DISPLAY_WIDTH` | no | queried from backend | Override display width, px | +| `CLINE_COMPUTER_USE_DISPLAY_HEIGHT` | no | queried from backend | Override display height, px | + +## Not yet done + +- No auth/handshake — this assumes the backend is a locally-spawned trusted + process on loopback. Do not bind the backend to a non-loopback address. +- No reconnect/backoff policy in the client; a dropped connection fails all + in-flight requests and reconnects lazily on the next call. +- No persisted setting/UI toggle; env-var opt-in only, matching this being a + proof of concept. +- The Anthropic `anthropic-beta: computer-use-2025-11-24` header is sent + unconditionally for the direct `anthropic` provider (see + `sdk/packages/llms/src/providers/routing/anthropic-compatible.ts`) rather + than gated on whether the `computer` tool is actually part of the current + request. Fine for this proof of concept; revisit before shipping. diff --git a/sdk/packages/core/src/extensions/computer-use/client.test.ts b/sdk/packages/core/src/extensions/computer-use/client.test.ts new file mode 100644 index 0000000000..06c0a36d10 --- /dev/null +++ b/sdk/packages/core/src/extensions/computer-use/client.test.ts @@ -0,0 +1,166 @@ +import { type AddressInfo, createServer, type Server, type Socket } from "node:net"; +import { afterEach, describe, expect, it } from "vitest"; +import { ComputerUseClient } from "./client"; +import type { ComputerUseResponse } from "./protocol"; + +/** + * Minimal fake computer-use backend for tests: a real TCP server that reads + * newline-delimited JSON requests and replies according to `respond`. + */ +function startFakeBackend( + respond: (request: Record) => ComputerUseResponse, +): Promise<{ server: Server; port: number }> { + return new Promise((resolve) => { + const server = createServer((socket: Socket) => { + let buffer = ""; + socket.setEncoding("utf8"); + socket.on("data", (chunk: string) => { + buffer += chunk; + let newlineIndex = buffer.indexOf("\n"); + while (newlineIndex >= 0) { + const line = buffer.slice(0, newlineIndex); + buffer = buffer.slice(newlineIndex + 1); + if (line.trim().length > 0) { + const request = JSON.parse(line) as Record; + const response = respond(request); + socket.write(`${JSON.stringify(response)}\n`); + } + newlineIndex = buffer.indexOf("\n"); + } + }); + }); + server.listen(0, "127.0.0.1", () => { + const address = server.address() as AddressInfo; + resolve({ server, port: address.port }); + }); + }); +} + +describe("ComputerUseClient", () => { + let server: Server | undefined; + + afterEach(async () => { + if (!server) { + return; + } + await new Promise((resolve) => server?.close(() => resolve())); + }); + + it("sends a request and resolves with the matching response", async () => { + const started = await startFakeBackend((request) => ({ + id: request.id as number, + ok: true, + text: `handled ${request.action as string}`, + })); + server = started.server; + + const client = new ComputerUseClient({ port: started.port }); + const response = await client.send({ action: "screenshot" }); + + expect(response.ok).toBe(true); + expect(response.text).toBe("handled screenshot"); + client.close(); + }); + + it("matches responses to requests by id across multiple in-flight calls", async () => { + const started = await startFakeBackend((request) => ({ + id: request.id as number, + ok: true, + text: `id=${request.id}`, + })); + server = started.server; + + const client = new ComputerUseClient({ port: started.port }); + const [first, second] = await Promise.all([ + client.send({ action: "cursor_position" }), + client.send({ action: "screenshot" }), + ]); + + expect(first.text).toBe("id=1"); + expect(second.text).toBe("id=2"); + client.close(); + }); + + it("surfaces backend error responses", async () => { + const started = await startFakeBackend((request) => ({ + id: request.id as number, + ok: false, + error: "backend exploded", + })); + server = started.server; + + const client = new ComputerUseClient({ port: started.port }); + const response = await client.send({ action: "left_click", coordinate: [1, 2] }); + + expect(response.ok).toBe(false); + expect(response.error).toBe("backend exploded"); + client.close(); + }); + + it("getDisplayInfo resolves with the backend-reported dimensions", async () => { + const started = await startFakeBackend((request) => ({ + id: request.id as number, + ok: true, + display: { widthPx: 1920, heightPx: 1080 }, + })); + server = started.server; + + const client = new ComputerUseClient({ port: started.port }); + const info = await client.getDisplayInfo(); + + expect(info).toEqual({ widthPx: 1920, heightPx: 1080 }); + client.close(); + }); + + it("getDisplayInfo rejects when the backend omits display info", async () => { + const started = await startFakeBackend((request) => ({ + id: request.id as number, + ok: true, + })); + server = started.server; + + const client = new ComputerUseClient({ port: started.port }); + await expect(client.getDisplayInfo()).rejects.toThrow(); + client.close(); + }); + + it("rejects when connecting to a closed port", async () => { + const started = await startFakeBackend((request) => ({ + id: request.id as number, + ok: true, + })); + server = started.server; + const { port } = started; + const startedServer = started.server; + await new Promise((resolve) => startedServer.close(() => resolve())); + + const client = new ComputerUseClient({ + port, + connectTimeoutMs: 500, + }); + await expect(client.send({ action: "screenshot" })).rejects.toThrow(); + }); + + it("times out a request the backend never answers", async () => { + const started = await startFakeBackend(() => { + throw new Error("never called: server intentionally does not respond"); + }); + server = started.server; + // Override respond to swallow requests without answering. + server.removeAllListeners("connection"); + server.on("connection", (socket) => { + socket.on("data", () => { + /* intentionally never respond */ + }); + }); + + const client = new ComputerUseClient({ + port: started.port, + requestTimeoutMs: 200, + }); + await expect(client.send({ action: "wait", durationSeconds: 1 })).rejects.toThrow( + /timed out/, + ); + client.close(); + }); +}); diff --git a/sdk/packages/core/src/extensions/computer-use/client.ts b/sdk/packages/core/src/extensions/computer-use/client.ts new file mode 100644 index 0000000000..17965688f2 --- /dev/null +++ b/sdk/packages/core/src/extensions/computer-use/client.ts @@ -0,0 +1,200 @@ +import { type Socket, connect } from "node:net"; +import { + type ComputerUseDisplayInfo, + GET_DISPLAY_INFO_ACTION, + type ComputerUseRequest, + type ComputerUseResponse, + isComputerUseResponse, +} from "./protocol"; + +export interface ComputerUseClientOptions { + /** Backend host, defaults to loopback only. */ + host?: string; + /** Backend TCP port. */ + port: number; + /** Per-request timeout in milliseconds. */ + requestTimeoutMs?: number; + /** Timeout for establishing the initial connection, in milliseconds. */ + connectTimeoutMs?: number; +} + +const DEFAULT_REQUEST_TIMEOUT_MS = 15_000; +const DEFAULT_CONNECT_TIMEOUT_MS = 3_000; + +interface PendingRequest { + resolve: (response: ComputerUseResponse) => void; + reject: (error: Error) => void; + timeout: ReturnType; +} + +/** + * Minimal client for the computer-use backend's JSON-L-over-TCP protocol. + * + * Deliberately lightweight: no reconnect/backoff policy, no multiplexed + * transport negotiation, no schema registry. Connects lazily on first use, + * reuses the connection across calls, and reconnects on the next call after + * a disconnect. See ./protocol.ts for the wire format and the rationale for + * not using MCP here. + */ +export class ComputerUseClient { + private socket: Socket | undefined; + private connectPromise: Promise | undefined; + private buffer = ""; + private nextRequestId = 1; + private readonly pending = new Map(); + + constructor(private readonly options: ComputerUseClientOptions) {} + + /** Sends a request and resolves with the matching response. */ + async send( + request: Omit, + ): Promise { + const socket = await this.ensureConnected(); + const id = this.nextRequestId++; + const fullRequest: ComputerUseRequest = { ...request, id }; + const line = `${JSON.stringify(fullRequest)}\n`; + + const timeoutMs = this.options.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS; + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + this.pending.delete(id); + reject( + new Error( + `Computer-use request ${id} (${request.action}) timed out after ${timeoutMs}ms`, + ), + ); + }, timeoutMs); + + this.pending.set(id, { resolve, reject, timeout }); + + socket.write(line, (error) => { + if (error) { + this.pending.delete(id); + clearTimeout(timeout); + reject(error); + } + }); + }); + } + + /** + * Queries the backend for the native display dimensions. This is not one + * of Anthropic's `computer` tool actions — it's a one-time startup query + * used to build the tool's description/schema with real values instead + * of guessed defaults, since the tool's definition is static once built. + */ + async getDisplayInfo(): Promise { + const response = await this.send({ action: GET_DISPLAY_INFO_ACTION }); + if (!response.ok || !response.display) { + throw new Error( + response.error ?? "Computer-use backend did not return display info", + ); + } + return response.display; + } + + /** Closes the underlying socket, if any. Safe to call multiple times. */ + close(): void { + this.socket?.destroy(); + this.socket = undefined; + this.connectPromise = undefined; + this.failAllPending(new Error("Computer-use client closed")); + } + + private async ensureConnected(): Promise { + if (this.socket && !this.socket.destroyed) { + return this.socket; + } + if (!this.connectPromise) { + this.connectPromise = this.connectSocket(); + } + return this.connectPromise; + } + + private connectSocket(): Promise { + const host = this.options.host ?? "127.0.0.1"; + const port = this.options.port; + const connectTimeoutMs = + this.options.connectTimeoutMs ?? DEFAULT_CONNECT_TIMEOUT_MS; + + return new Promise((resolve, reject) => { + const socket = connect({ host, port }); + const onConnectTimeout = setTimeout(() => { + socket.destroy(); + reject( + new Error( + `Timed out connecting to computer-use backend at ${host}:${port}`, + ), + ); + }, connectTimeoutMs); + + socket.once("connect", () => { + clearTimeout(onConnectTimeout); + this.socket = socket; + this.buffer = ""; + // Don't let an idle backend connection keep the host process + // alive on its own; the socket is reference-counted back in + // while a request is in flight via the write/response cycle. + socket.unref(); + resolve(socket); + }); + + socket.once("error", (error) => { + clearTimeout(onConnectTimeout); + this.connectPromise = undefined; + this.failAllPending(error); + reject(error); + }); + + socket.once("close", () => { + this.connectPromise = undefined; + this.failAllPending(new Error("Computer-use backend connection closed")); + }); + + socket.setEncoding("utf8"); + socket.on("data", (chunk: string) => this.onData(chunk)); + }); + } + + private onData(chunk: string): void { + this.buffer += chunk; + let newlineIndex = this.buffer.indexOf("\n"); + while (newlineIndex >= 0) { + const line = this.buffer.slice(0, newlineIndex).trim(); + this.buffer = this.buffer.slice(newlineIndex + 1); + if (line.length > 0) { + this.handleLine(line); + } + newlineIndex = this.buffer.indexOf("\n"); + } + } + + private handleLine(line: string): void { + let parsed: unknown; + try { + parsed = JSON.parse(line); + } catch { + // Malformed line from the backend; ignore rather than crash the + // connection, since this is best-effort POC plumbing. + return; + } + if (!isComputerUseResponse(parsed)) { + return; + } + const pending = this.pending.get(parsed.id); + if (!pending) { + return; + } + this.pending.delete(parsed.id); + clearTimeout(pending.timeout); + pending.resolve(parsed); + } + + private failAllPending(error: Error): void { + for (const [id, pending] of this.pending) { + clearTimeout(pending.timeout); + pending.reject(error); + this.pending.delete(id); + } + } +} diff --git a/sdk/packages/core/src/extensions/computer-use/env.test.ts b/sdk/packages/core/src/extensions/computer-use/env.test.ts new file mode 100644 index 0000000000..095f811beb --- /dev/null +++ b/sdk/packages/core/src/extensions/computer-use/env.test.ts @@ -0,0 +1,122 @@ +import { type AddressInfo, createServer, type Server, type Socket } from "node:net"; +import { afterEach, describe, expect, it } from "vitest"; +import { createComputerUseToolFromEnv } from "./env"; +import type { ComputerUseResponse } from "./protocol"; + +/** + * Like the fake backend in client.test.ts/tool.test.ts, but also tracks + * accepted sockets so the test can force-close them. Unlike those files, + * `createComputerUseToolFromEnv` builds its own internal `ComputerUseClient` + * with no handle exposed back to the test, so there's no `client.close()` to + * call — `server.close()` alone would otherwise hang forever waiting for the + * (unref'd but still open) client connection to end. + */ +function startFakeBackend( + respond: (request: Record) => ComputerUseResponse, +): Promise<{ server: Server; port: number; destroyConnections: () => void }> { + const sockets = new Set(); + return new Promise((resolve) => { + const server = createServer((socket: Socket) => { + sockets.add(socket); + socket.on("close", () => sockets.delete(socket)); + let buffer = ""; + socket.setEncoding("utf8"); + socket.on("data", (chunk: string) => { + buffer += chunk; + let newlineIndex = buffer.indexOf("\n"); + while (newlineIndex >= 0) { + const line = buffer.slice(0, newlineIndex); + buffer = buffer.slice(newlineIndex + 1); + if (line.trim().length > 0) { + const request = JSON.parse(line) as Record; + socket.write(`${JSON.stringify(respond(request))}\n`); + } + newlineIndex = buffer.indexOf("\n"); + } + }); + }); + server.listen(0, "127.0.0.1", () => { + const address = server.address() as AddressInfo; + resolve({ + server, + port: address.port, + destroyConnections: () => { + for (const socket of sockets) { + socket.destroy(); + } + }, + }); + }); + }); +} + +function fakeDisplayInfoBackend( + widthPx: number, + heightPx: number, +): (request: Record) => ComputerUseResponse { + return (request) => ({ + id: request.id as number, + ok: true, + display: { widthPx, heightPx }, + }); +} + +describe("createComputerUseToolFromEnv", () => { + let server: Server | undefined; + let destroyConnections: (() => void) | undefined; + + afterEach(async () => { + destroyConnections?.(); + destroyConnections = undefined; + if (!server) { + return; + } + await new Promise((resolve) => server?.close(() => resolve())); + server = undefined; + }); + + it("returns undefined when the port variable is unset", async () => { + await expect(createComputerUseToolFromEnv({})).resolves.toBeUndefined(); + }); + + it("returns undefined when the port variable is not a positive integer", async () => { + await expect( + createComputerUseToolFromEnv({ CLINE_COMPUTER_USE_PORT: "not-a-port" }), + ).resolves.toBeUndefined(); + await expect( + createComputerUseToolFromEnv({ CLINE_COMPUTER_USE_PORT: "0" }), + ).resolves.toBeUndefined(); + await expect( + createComputerUseToolFromEnv({ CLINE_COMPUTER_USE_PORT: "-1" }), + ).resolves.toBeUndefined(); + }); + + it("queries the backend for display size when no override is set", async () => { + const started = await startFakeBackend(fakeDisplayInfoBackend(1920, 1080)); + server = started.server; + destroyConnections = started.destroyConnections; + + const tool = await createComputerUseToolFromEnv({ + CLINE_COMPUTER_USE_PORT: String(started.port), + }); + + expect(tool).toBeDefined(); + expect(tool?.name).toBe("computer"); + expect(tool?.description).toContain("1920x1080"); + }); + + it("honors display size overrides without querying the backend", async () => { + const started = await startFakeBackend(() => { + throw new Error("backend should not be queried when overrides are set"); + }); + server = started.server; + + const tool = await createComputerUseToolFromEnv({ + CLINE_COMPUTER_USE_PORT: String(started.port), + CLINE_COMPUTER_USE_DISPLAY_WIDTH: "1024", + CLINE_COMPUTER_USE_DISPLAY_HEIGHT: "768", + }); + + expect(tool?.description).toContain("1024x768"); + }); +}); diff --git a/sdk/packages/core/src/extensions/computer-use/env.ts b/sdk/packages/core/src/extensions/computer-use/env.ts new file mode 100644 index 0000000000..9f8a7335e7 --- /dev/null +++ b/sdk/packages/core/src/extensions/computer-use/env.ts @@ -0,0 +1,43 @@ +import type { AgentTool } from "@cline/shared"; +import { createComputerUseTool } from "./tool"; + +const PORT_ENV_VAR = "CLINE_COMPUTER_USE_PORT"; +const HOST_ENV_VAR = "CLINE_COMPUTER_USE_HOST"; +const DISPLAY_WIDTH_ENV_VAR = "CLINE_COMPUTER_USE_DISPLAY_WIDTH"; +const DISPLAY_HEIGHT_ENV_VAR = "CLINE_COMPUTER_USE_DISPLAY_HEIGHT"; + +function parsePositiveInt(value: string | undefined): number | undefined { + if (!value) { + return undefined; + } + const parsed = Number.parseInt(value, 10); + return Number.isFinite(parsed) && parsed > 0 ? parsed : undefined; +} + +/** + * Builds the `computer` tool from environment variables, or returns + * `undefined` if computer-use isn't configured for this process. + * + * This is a proof-of-concept convenience for hosts (starting with the CLI) + * that want to opt in without any config plumbing of their own: set + * `CLINE_COMPUTER_USE_PORT` to the backend's TCP port and the tool becomes + * available. Display size is queried from the backend by default; the + * `CLINE_COMPUTER_USE_DISPLAY_WIDTH`/`_HEIGHT` variables only need to be set + * to override that. There is intentionally no persisted setting/toggle yet + * — see ./README.md. + */ +export async function createComputerUseToolFromEnv( + env: NodeJS.ProcessEnv = process.env, +): Promise { + const port = parsePositiveInt(env[PORT_ENV_VAR]); + if (!port) { + return undefined; + } + + return createComputerUseTool({ + host: env[HOST_ENV_VAR] || undefined, + port, + displayWidthPx: parsePositiveInt(env[DISPLAY_WIDTH_ENV_VAR]), + displayHeightPx: parsePositiveInt(env[DISPLAY_HEIGHT_ENV_VAR]), + }); +} diff --git a/sdk/packages/core/src/extensions/computer-use/index.ts b/sdk/packages/core/src/extensions/computer-use/index.ts new file mode 100644 index 0000000000..979bf31ef8 --- /dev/null +++ b/sdk/packages/core/src/extensions/computer-use/index.ts @@ -0,0 +1,24 @@ +/** + * Computer-use tool integration. + * + * A genuine `@cline/core` extension (an `AgentTool`, usable via + * `CoreSessionConfig.extraTools` from any host), kept in its own folder with + * a minimal dependency surface (`@cline/shared` types only) so it can be + * extracted into a standalone Cline plugin later with minimal changes. See + * ./protocol.ts for the wire format and ./README.md for the backend + * contract and design rationale. + */ +export { ComputerUseClient, type ComputerUseClientOptions } from "./client"; +export { + createComputerUseToolFromEnv, +} from "./env"; +export type { + ComputerUseAction, + ComputerUseCoordinate, + ComputerUseDisplayInfo, + ComputerUseImage, + ComputerUseRequest, + ComputerUseResponse, +} from "./protocol"; +export { GET_DISPLAY_INFO_ACTION, isComputerUseResponse } from "./protocol"; +export { createComputerUseTool, type ComputerUseToolOptions } from "./tool"; diff --git a/sdk/packages/core/src/extensions/computer-use/protocol.ts b/sdk/packages/core/src/extensions/computer-use/protocol.ts new file mode 100644 index 0000000000..df3c399352 --- /dev/null +++ b/sdk/packages/core/src/extensions/computer-use/protocol.ts @@ -0,0 +1,121 @@ +/** + * Wire protocol for the Cline <-> computer-use backend socket connection. + * + * This is intentionally NOT MCP. The computer-use backend (a Rust process, + * developed out-of-tree) is a single-purpose, low-latency screen/input + * bridge: connect once, send an action, get a screenshot back. MCP's + * JSON-RPC 2.0 handshake, capability negotiation, and stdio/SSE transport + * machinery buy nothing here and add real latency + complexity for a tool + * that's called on every turn of an agentic loop. A plain newline-delimited + * JSON ("JSON Lines" / JSON-L) socket protocol keeps the dependency light + * and easy to reimplement in Rust with nothing more than `tokio::net` + + * `serde_json`. + * + * Framing: every message is a single JSON value serialized on one line, + * terminated by "\n". No Content-Length headers, no multipart framing. + * Newlines inside string values MUST be JSON-escaped (this is automatic with + * `JSON.stringify`/`serde_json::to_string`), so a bare "\n" always marks a + * message boundary. + * + * Transport: plain TCP, localhost only. TCP (rather than a Unix domain + * socket / Windows named pipe) is used so the same client code works + * unmodified on Windows, macOS, and Linux. This is a local trust boundary + * (loopback only, never bound to 0.0.0.0). + */ + +/** + * Actions understood by the computer-use backend, mirroring Anthropic's + * `computer` tool action set (see + * https://platform.claude.com/docs/en/agents-and-tools/tool-use/computer-use-tool). + */ +export type ComputerUseAction = + | "screenshot" + | "cursor_position" + | "mouse_move" + | "left_click" + | "left_click_drag" + | "right_click" + | "middle_click" + | "double_click" + | "triple_click" + | "left_mouse_down" + | "left_mouse_up" + | "key" + | "hold_key" + | "type" + | "scroll" + | "wait" + | "zoom"; + +/** + * Internal query (not one of Anthropic's `computer` tool actions) used to + * ask the backend for the real, native display dimensions before the + * `computer` tool is built. The tool's description/schema is static once + * built, so this must be resolved once at startup rather than per model + * turn — see `ComputerUseClient.getDisplayInfo()`. + */ +export const GET_DISPLAY_INFO_ACTION = "get_display_info"; + +/** Native display dimensions reported by the backend. */ +export interface ComputerUseDisplayInfo { + widthPx: number; + heightPx: number; +} + +/** A single [x, y] pixel coordinate in the (possibly scaled) display space. */ +export type ComputerUseCoordinate = readonly [number, number]; + +/** Request envelope sent from Cline to the computer-use backend, one per line. */ +export interface ComputerUseRequest { + /** Monotonically increasing id used to match responses to requests. */ + id: number; + action: ComputerUseAction | typeof GET_DISPLAY_INFO_ACTION; + coordinate?: ComputerUseCoordinate; + startCoordinate?: ComputerUseCoordinate; + text?: string; + /** Key name(s) for "key"/"hold_key", e.g. "ctrl+alt+delete". */ + keys?: string; + /** Duration in seconds, used by "hold_key" and "wait". */ + durationSeconds?: number; + scrollDirection?: "up" | "down" | "left" | "right"; + scrollAmount?: number; + /** Region [x, y, width, height] for "zoom". */ + region?: readonly [number, number, number, number]; +} + +/** A single image returned by the backend (typically a screenshot). */ +export interface ComputerUseImage { + /** Base64-encoded image bytes. */ + data: string; + /** MIME type, e.g. "image/png". */ + mediaType: string; +} + +/** Response envelope received from the computer-use backend, one per line. */ +export interface ComputerUseResponse { + /** Echoes the request id this response answers. */ + id: number; + ok: boolean; + /** Human-readable result text (e.g. cursor position, ack message). */ + text?: string; + /** + * Present for actions that capture the screen ("screenshot", and + * optionally others that return a post-action screenshot). + */ + image?: ComputerUseImage; + /** Present in the response to a "get_display_info" request. */ + display?: ComputerUseDisplayInfo; + /** Present when ok is false. */ + error?: string; +} + +/** Type guard for parsed JSON-L lines from the backend. */ +export function isComputerUseResponse( + value: unknown, +): value is ComputerUseResponse { + if (!value || typeof value !== "object") { + return false; + } + const candidate = value as Record; + return typeof candidate.id === "number" && typeof candidate.ok === "boolean"; +} diff --git a/sdk/packages/core/src/extensions/computer-use/tool.test.ts b/sdk/packages/core/src/extensions/computer-use/tool.test.ts new file mode 100644 index 0000000000..e7cdc1f3c9 --- /dev/null +++ b/sdk/packages/core/src/extensions/computer-use/tool.test.ts @@ -0,0 +1,204 @@ +import type { AgentToolContext } from "@cline/shared"; +import { type AddressInfo, createServer, type Server, type Socket } from "node:net"; +import { afterEach, describe, expect, it } from "vitest"; +import { ComputerUseClient } from "./client"; +import type { ComputerUseResponse } from "./protocol"; +import { createComputerUseTool } from "./tool"; + +function startFakeBackend( + respond: (request: Record) => ComputerUseResponse, +): Promise<{ server: Server; port: number }> { + return new Promise((resolve) => { + const server = createServer((socket: Socket) => { + let buffer = ""; + socket.setEncoding("utf8"); + socket.on("data", (chunk: string) => { + buffer += chunk; + let newlineIndex = buffer.indexOf("\n"); + while (newlineIndex >= 0) { + const line = buffer.slice(0, newlineIndex); + buffer = buffer.slice(newlineIndex + 1); + if (line.trim().length > 0) { + const request = JSON.parse(line) as Record; + socket.write(`${JSON.stringify(respond(request))}\n`); + } + newlineIndex = buffer.indexOf("\n"); + } + }); + }); + server.listen(0, "127.0.0.1", () => { + const address = server.address() as AddressInfo; + resolve({ server, port: address.port }); + }); + }); +} + +const ctx: AgentToolContext = { + agentId: "agent-1", + conversationId: "conv-1", + iteration: 1, +}; + +describe("createComputerUseTool", () => { + let server: Server | undefined; + let client: ComputerUseClient | undefined; + + afterEach(async () => { + client?.close(); + client = undefined; + if (!server) { + return; + } + await new Promise((resolve) => server?.close(() => resolve())); + server = undefined; + }); + + it("exposes the computer tool name and an object input schema", async () => { + const started = await startFakeBackend((request) => ({ + id: request.id as number, + ok: true, + })); + server = started.server; + client = new ComputerUseClient({ port: started.port }); + + const tool = await createComputerUseTool({ + port: started.port, + displayWidthPx: 1024, + displayHeightPx: 768, + client, + }); + + expect(tool.name).toBe("computer"); + expect(tool.inputSchema.type).toBe("object"); + expect(tool.description).toContain("1024x768"); + }); + + it("queries the backend for display size when no override is provided", async () => { + const started = await startFakeBackend((request) => ({ + id: request.id as number, + ok: true, + display: { widthPx: 1920, heightPx: 1080 }, + })); + server = started.server; + client = new ComputerUseClient({ port: started.port }); + + const tool = await createComputerUseTool({ + port: started.port, + client, + }); + + expect(tool.description).toContain("1920x1080"); + }); + + it("uses a single override without querying the backend for the other dimension", async () => { + const started = await startFakeBackend((request) => ({ + id: request.id as number, + ok: true, + display: { widthPx: 1920, heightPx: 1080 }, + })); + server = started.server; + client = new ComputerUseClient({ port: started.port }); + + const tool = await createComputerUseTool({ + port: started.port, + displayWidthPx: 640, + client, + }); + + expect(tool.description).toContain("640x1080"); + }); + + it("returns a screenshot as multimodal text+image content", async () => { + const started = await startFakeBackend((request) => ({ + id: request.id as number, + ok: true, + text: "screenshot taken", + image: { data: "ZmFrZS1wbmc=", mediaType: "image/png" }, + })); + server = started.server; + client = new ComputerUseClient({ port: started.port }); + + const tool = await createComputerUseTool({ + port: started.port, + displayWidthPx: 1024, + displayHeightPx: 768, + client, + }); + + const result = await tool.execute({ action: "screenshot" }, ctx); + + expect(result).toEqual([ + { type: "text", text: "screenshot taken" }, + { type: "image", data: "ZmFrZS1wbmc=", mediaType: "image/png" }, + ]); + }); + + it("returns plain text when the backend does not return an image", async () => { + const started = await startFakeBackend((request) => ({ + id: request.id as number, + ok: true, + text: "clicked at (10, 20)", + })); + server = started.server; + client = new ComputerUseClient({ port: started.port }); + + const tool = await createComputerUseTool({ + port: started.port, + displayWidthPx: 1024, + displayHeightPx: 768, + client, + }); + + const result = await tool.execute( + { action: "left_click", coordinate: [10, 20] }, + ctx, + ); + + expect(result).toBe("clicked at (10, 20)"); + }); + + it("maps key/hold_key text into the keys field, not the text field", async () => { + const started = await startFakeBackend((request) => ({ + id: request.id as number, + ok: true, + text: `keys=${request.keys ?? "none"} text=${request.text ?? "none"}`, + })); + server = started.server; + client = new ComputerUseClient({ port: started.port }); + + const tool = await createComputerUseTool({ + port: started.port, + displayWidthPx: 1024, + displayHeightPx: 768, + client, + }); + + const result = await tool.execute( + { action: "key", text: "ctrl+alt+delete" }, + ctx, + ); + + expect(result).toBe("keys=ctrl+alt+delete text=none"); + }); + + it("throws with the backend error message on failure", async () => { + const started = await startFakeBackend((request) => ({ + id: request.id as number, + ok: false, + error: "no display attached", + })); + server = started.server; + client = new ComputerUseClient({ port: started.port }); + + const tool = await createComputerUseTool({ + port: started.port, + displayWidthPx: 1024, + displayHeightPx: 768, + client, + }); + + await expect(tool.execute({ action: "screenshot" }, ctx)).rejects.toThrow( + "no display attached", + ); + }); +}); diff --git a/sdk/packages/core/src/extensions/computer-use/tool.ts b/sdk/packages/core/src/extensions/computer-use/tool.ts new file mode 100644 index 0000000000..512e7c6620 --- /dev/null +++ b/sdk/packages/core/src/extensions/computer-use/tool.ts @@ -0,0 +1,192 @@ +import { createTool } from "@cline/shared"; +import type { AgentTool, AgentToolContext } from "@cline/shared"; +import { ComputerUseClient, type ComputerUseClientOptions } from "./client"; +import type { ComputerUseAction, ComputerUseCoordinate } from "./protocol"; + +export interface ComputerUseToolOptions extends ComputerUseClientOptions { + /** + * Override for the controlled display's width in pixels, reported in the + * tool description. When omitted, this is queried from the backend at + * construction time via `ComputerUseClient.getDisplayInfo()` — the + * backend is the source of truth for the real, native screen size. + */ + displayWidthPx?: number; + /** Override for display height in pixels. See `displayWidthPx`. */ + displayHeightPx?: number; + /** + * Optional pre-built client. Mainly useful for tests and for hosts that + * want explicit control over the connection's lifecycle (e.g. calling + * `client.close()` on shutdown). When omitted, a client is constructed + * from the other options. + */ + client?: ComputerUseClient; +} + +/** Raw tool input shape as sent by the model (mirrors Anthropic's `computer` tool). */ +interface ComputerToolInput { + action: ComputerUseAction; + coordinate?: ComputerUseCoordinate; + start_coordinate?: ComputerUseCoordinate; + text?: string; + duration?: number; + scroll_direction?: "up" | "down" | "left" | "right"; + scroll_amount?: number; + region?: readonly [number, number, number, number]; +} + +const COMPUTER_TOOL_NAME = "computer"; + +const COMPUTER_TOOL_INPUT_SCHEMA: Record = { + type: "object", + properties: { + action: { + type: "string", + enum: [ + "screenshot", + "cursor_position", + "mouse_move", + "left_click", + "left_click_drag", + "right_click", + "middle_click", + "double_click", + "triple_click", + "left_mouse_down", + "left_mouse_up", + "key", + "hold_key", + "type", + "scroll", + "wait", + "zoom", + ], + description: "The action to perform.", + }, + coordinate: { + type: "array", + items: { type: "number" }, + minItems: 2, + maxItems: 2, + description: + "(x, y) pixel coordinate, required for mouse_move, left_click, left_click_drag (end point), right_click, middle_click, double_click, triple_click, left_mouse_down, left_mouse_up, and scroll (scroll origin).", + }, + start_coordinate: { + type: "array", + items: { type: "number" }, + minItems: 2, + maxItems: 2, + description: "(x, y) start coordinate, required for left_click_drag.", + }, + text: { + type: "string", + description: + "Text to type (for the type action) or key combination to press (for key/hold_key, e.g. 'ctrl+alt+delete').", + }, + duration: { + type: "number", + description: "Duration in seconds, used by hold_key and wait.", + }, + scroll_direction: { + type: "string", + enum: ["up", "down", "left", "right"], + description: "Direction to scroll, required for the scroll action.", + }, + scroll_amount: { + type: "number", + description: "Number of scroll clicks, required for the scroll action.", + }, + region: { + type: "array", + items: { type: "number" }, + minItems: 4, + maxItems: 4, + description: + "(x, y, width, height) region to zoom into, required for the zoom action.", + }, + }, + required: ["action"], +}; + +function toComputerUseRequest(input: ComputerToolInput) { + return { + action: input.action, + coordinate: input.coordinate, + startCoordinate: input.start_coordinate, + text: input.text, + durationSeconds: input.duration, + scrollDirection: input.scroll_direction, + scrollAmount: input.scroll_amount, + region: input.region, + }; +} + +/** + * Builds the `computer` tool that forwards Anthropic computer-use tool calls + * to an external, lightweight backend (a Rust process developed out-of-tree) + * over a plain JSON-L TCP socket. + * + * This is async because the tool's description embeds the display size, + * which is queried from the backend (the source of truth for the real, + * native screen dimensions) unless both `displayWidthPx`/`displayHeightPx` + * are explicitly overridden in `options`. + * + * This is a genuine `@cline/core` `AgentTool` — usable from any host that + * builds a `CoreSessionConfig` (CLI, VSCode adapter, etc.) via + * `config.extraTools`. It's deliberately isolated within its own folder + * (only depends on `@cline/shared`'s `AgentTool` contract) so it can be + * lifted out into a standalone Cline plugin later with minimal changes. See + * ./README.md for the wire protocol and rationale. + */ +export async function createComputerUseTool( + options: ComputerUseToolOptions, +): Promise { + const client = options.client ?? new ComputerUseClient(options); + + const { displayWidthPx, displayHeightPx } = + options.displayWidthPx !== undefined && options.displayHeightPx !== undefined + ? { displayWidthPx: options.displayWidthPx, displayHeightPx: options.displayHeightPx } + : await client.getDisplayInfo().then((info) => ({ + displayWidthPx: options.displayWidthPx ?? info.widthPx, + displayHeightPx: options.displayHeightPx ?? info.heightPx, + })); + + return createTool({ + name: COMPUTER_TOOL_NAME, + description: + `Control the screen and keyboard/mouse of a remote computer environment. ` + + `The display is ${displayWidthPx}x${displayHeightPx} pixels. ` + + `Use "screenshot" to see the current screen before acting, since the environment ` + + `may change between turns. Coordinates are [x, y] pixels from the top-left corner.`, + inputSchema: COMPUTER_TOOL_INPUT_SCHEMA, + // Screenshots and round trips to an external process are slower than + // in-process tools; give this more room than the SDK's 30s default. + timeoutMs: 30_000, + retryable: false, + execute: async (input: unknown, _context: AgentToolContext) => { + const parsedInput = input as ComputerToolInput; + const response = await client.send(toComputerUseRequest(parsedInput)); + + if (!response.ok) { + throw new Error( + response.error ?? `Computer-use action "${parsedInput.action}" failed`, + ); + } + + if (!response.image) { + return response.text ?? `Action "${parsedInput.action}" completed.`; + } + + return [ + { + type: "text" as const, + text: response.text ?? `Action "${parsedInput.action}" completed.`, + }, + { + type: "image" as const, + data: response.image.data, + mediaType: response.image.mediaType, + }, + ]; + }, + }); +} diff --git a/sdk/packages/core/src/index.ts b/sdk/packages/core/src/index.ts index b29d7a9cee..fb97f92241 100644 --- a/sdk/packages/core/src/index.ts +++ b/sdk/packages/core/src/index.ts @@ -1042,6 +1042,21 @@ export { export * from "./remote/remote-environments"; export { ensureLoginShellPath } from "./remote/shell-path"; export { isClineAccountFeatureEnabled } from "./services/feature-flags/cline-account-feature-flags"; +export { + ComputerUseClient, + type ComputerUseClientOptions, + type ComputerUseAction, + type ComputerUseCoordinate, + type ComputerUseDisplayInfo, + type ComputerUseImage, + type ComputerUseRequest, + type ComputerUseResponse, + createComputerUseTool, + createComputerUseToolFromEnv, + GET_DISPLAY_INFO_ACTION, + isComputerUseResponse, + type ComputerUseToolOptions, +} from "./extensions/computer-use"; export { applyClineFeaturedModels, type ClineRecommendedModel, diff --git a/sdk/packages/llms/src/providers/routing/anthropic-compatible.test.ts b/sdk/packages/llms/src/providers/routing/anthropic-compatible.test.ts index ea6d1cdb6f..b3e27e3d0d 100644 --- a/sdk/packages/llms/src/providers/routing/anthropic-compatible.test.ts +++ b/sdk/packages/llms/src/providers/routing/anthropic-compatible.test.ts @@ -2,6 +2,7 @@ import type { GatewayModelRoute, GatewayProviderContext, GatewayProviderManifest, + GatewayStreamRequest, } from "@cline/shared"; import { describe, expect, it } from "vitest"; import { @@ -14,6 +15,7 @@ import { } from "../model-facts"; import { applyPromptCacheToLastTextPart, + buildAnthropicProviderOptions, resolveAnthropicReasoningRequestPolicy, resolvePromptCacheRoute, shouldApplyPromptCache, @@ -652,3 +654,51 @@ describe("anthropic-compatible routing helpers", () => { }); }); }); + +describe("buildAnthropicProviderOptions computer-use beta header", () => { + function makeRequest( + overrides: Partial = {}, + ): GatewayStreamRequest { + return { + providerId: "anthropic", + modelId: "claude-sonnet-4-6", + messages: [], + ...overrides, + }; + } + + it("sends the computer-use beta header for the direct anthropic wire target", () => { + const options = buildAnthropicProviderOptions( + makeRequest(), + makeContext("anthropic"), + "anthropic", + ); + + expect(options.anthropicBeta).toEqual(["computer-use-2025-11-24"]); + }); + + it("does not send the beta header for other Anthropic-lineage wire targets", () => { + const bedrockOptions = buildAnthropicProviderOptions( + makeRequest({ providerId: "bedrock" }), + makeContext("anthropic"), + "bedrock", + ); + expect(bedrockOptions.anthropicBeta).toBeUndefined(); + + const vertexOptions = buildAnthropicProviderOptions( + makeRequest({ providerId: "vertex" }), + makeContext("anthropic"), + "vertex", + ); + expect(vertexOptions.anthropicBeta).toBeUndefined(); + }); + + it("does not send the beta header when target is omitted", () => { + const options = buildAnthropicProviderOptions( + makeRequest(), + makeContext("anthropic"), + ); + + expect(options.anthropicBeta).toBeUndefined(); + }); +}); diff --git a/sdk/packages/llms/src/providers/routing/anthropic-compatible.ts b/sdk/packages/llms/src/providers/routing/anthropic-compatible.ts index 9a09713323..9d2daf9b22 100644 --- a/sdk/packages/llms/src/providers/routing/anthropic-compatible.ts +++ b/sdk/packages/llms/src/providers/routing/anthropic-compatible.ts @@ -15,12 +15,26 @@ import { resolveClaudeThinkingEra, resolveModelFamily, } from "../model-facts"; +import type { AiSdkProviderOptionsTarget } from "./provider-options-types"; import { createEphemeralCacheControl, toProviderOptionsBucket, toProviderOptionsKey, } from "./utils"; +/** + * Anthropic beta features gated behind the `anthropic-beta` request header. + * + * `computer-use-2025-11-24` unlocks the extended computer-use action set + * (scroll, wait, zoom, etc.) on the direct Anthropic Messages API. This is + * always sent for the direct `anthropic` wire target for now (proof of + * concept); it is intentionally NOT sent for other Anthropic-lineage wire + * adapters (Bedrock, Vertex, MiniMax, SAP AI Core, ...) since those go + * through different request shapes/protocols that don't necessarily accept + * this header the same way. + */ +const ANTHROPIC_BETA_FEATURES: readonly string[] = ["computer-use-2025-11-24"]; + const ANTHROPIC_DEFAULT_THINKING_BUDGET_TOKENS = 1024; const ANTHROPIC_MAX_THINKING_BUDGET_TOKENS = 128000; @@ -367,6 +381,7 @@ export function resolveAnthropicReasoningRequestPolicy( export function buildAnthropicProviderOptions( request: GatewayStreamRequest, context: GatewayProviderContext, + target?: AiSdkProviderOptionsTarget, ) { const explicitBudget = request.reasoning?.enabled === false @@ -399,6 +414,12 @@ export function buildAnthropicProviderOptions( ...(shouldApplyAnthropicCacheBucket(request, context) ? createEphemeralCacheControl() : {}), + // POC: always send the computer-use beta header on the direct + // Anthropic wire target. Not gated per-model/per-tool yet; revisit + // before this leaves proof-of-concept status. + ...(target === "anthropic" + ? { anthropicBeta: [...ANTHROPIC_BETA_FEATURES] } + : {}), }; } diff --git a/sdk/packages/llms/src/providers/routing/provider-options.ts b/sdk/packages/llms/src/providers/routing/provider-options.ts index 3b655da1aa..24189772a8 100644 --- a/sdk/packages/llms/src/providers/routing/provider-options.ts +++ b/sdk/packages/llms/src/providers/routing/provider-options.ts @@ -98,6 +98,7 @@ export function composeAiSdkProviderOptions( const anthropicOptions = buildAnthropicProviderOptions( normalizedRequest, context, + target, ); const buildInput = { ...matchInput,