feat: chat desktop backend (#23005)

Implement the backend for the desktop feature for agents. - Adds a new `/api/experimental/chats/$id/desktop` endpoint to coderd which exposes a VNC stream from a [portabledesktop](https://github.com/coder/portabledesktop) process running inside the workspace - Adds a new `spawn_computer_use_agent` tool to chatd, which spawns a subagent that has access to the `computer` tool which lets it interact with the `portabledesktop` process running inside the workspace - Adds the plumbing to make the above possible There's a follow up frontend PR here: https://github.com/coder/coder/pull/23006
2026-09-24 15:04:27 +08:00 · 2026-03-13 19:49:34 +01:00
parent 67f5494665
commit 84527390c6
34 changed files with 4465 additions and 49 deletions
@@ -0,0 +1,220 @@
+package chattool
+
+import (
+	"context"
+	"fmt"
+	"math"
+	"time"
+
+	"charm.land/fantasy"
+	fantasyanthropic "charm.land/fantasy/providers/anthropic"
+
+	"github.com/coder/coder/v2/codersdk/workspacesdk"
+	"github.com/coder/quartz"
+)
+
+const (
+	// ComputerUseModelProvider is the provider for the computer
+	// use model.
+	ComputerUseModelProvider = "anthropic"
+	// ComputerUseModelName is the model used for computer use
+	// subagents.
+	ComputerUseModelName = "claude-opus-4-6"
+)
+
+// computerUseTool implements fantasy.AgentTool and
+// chatloop.ToolDefiner for Anthropic computer use.
+type computerUseTool struct {
+	displayWidth     int
+	displayHeight    int
+	getWorkspaceConn func(ctx context.Context) (workspacesdk.AgentConn, error)
+	providerOptions  fantasy.ProviderOptions
+	clock            quartz.Clock
+}
+
+// NewComputerUseTool creates a computer use AgentTool that
+// delegates to the agent's desktop endpoints.
+func NewComputerUseTool(
+	displayWidth, displayHeight int,
+	getWorkspaceConn func(ctx context.Context) (workspacesdk.AgentConn, error),
+	clock quartz.Clock,
+) fantasy.AgentTool {
+	return &computerUseTool{
+		displayWidth:     displayWidth,
+		displayHeight:    displayHeight,
+		getWorkspaceConn: getWorkspaceConn,
+		clock:            clock,
+	}
+}
+
+func (*computerUseTool) Info() fantasy.ToolInfo {
+	return fantasy.ToolInfo{
+		Name:        "computer",
+		Description: "Control the desktop: take screenshots, move the mouse, click, type, and scroll.",
+		Parameters:  map[string]any{},
+		Required:    []string{},
+	}
+}
+
+// ComputerUseProviderTool creates the provider-defined tool
+// definition for Anthropic computer use. This is passed via
+// ProviderTools so the API receives the correct wire format.
+func ComputerUseProviderTool(displayWidth, displayHeight int) fantasy.Tool {
+	return fantasyanthropic.NewComputerUseTool(
+		fantasyanthropic.ComputerUseToolOptions{
+			DisplayWidthPx:  int64(displayWidth),
+			DisplayHeightPx: int64(displayHeight),
+			ToolVersion:     fantasyanthropic.ComputerUse20251124,
+		},
+	)
+}
+
+func (t *computerUseTool) ProviderOptions() fantasy.ProviderOptions {
+	return t.providerOptions
+}
+
+func (t *computerUseTool) SetProviderOptions(opts fantasy.ProviderOptions) {
+	t.providerOptions = opts
+}
+
+func (t *computerUseTool) Run(ctx context.Context, call fantasy.ToolCall) (fantasy.ToolResponse, error) {
+	input, err := fantasyanthropic.ParseComputerUseInput(call.Input)
+	if err != nil {
+		return fantasy.NewTextErrorResponse(
+			fmt.Sprintf("invalid computer use input: %v", err),
+		), nil
+	}
+
+	conn, err := t.getWorkspaceConn(ctx)
+	if err != nil {
+		return fantasy.NewTextErrorResponse(
+			fmt.Sprintf("failed to connect to workspace: %v", err),
+		), nil
+	}
+
+	// Compute scaled screenshot size for Anthropic constraints.
+	scaledW, scaledH := computeScaledScreenshotSize(
+		t.displayWidth, t.displayHeight,
+	)
+
+	// For wait actions, sleep then return a screenshot.
+	if input.Action == fantasyanthropic.ActionWait {
+		d := input.Duration
+		if d <= 0 {
+			d = 1000
+		}
+		timer := t.clock.NewTimer(time.Duration(d)*time.Millisecond, "computeruse", "wait")
+		defer timer.Stop()
+		select {
+		case <-ctx.Done():
+		case <-timer.C:
+		}
+		screenshotAction := workspacesdk.DesktopAction{
+			Action:       "screenshot",
+			ScaledWidth:  &scaledW,
+			ScaledHeight: &scaledH,
+		}
+		screenResp, sErr := conn.ExecuteDesktopAction(ctx, screenshotAction)
+		if sErr != nil {
+			return fantasy.NewTextErrorResponse(
+				fmt.Sprintf("screenshot failed: %v", sErr),
+			), nil
+		}
+		return fantasy.NewImageResponse(
+			[]byte(screenResp.ScreenshotData), "image/png",
+		), nil
+	}
+
+	// For screenshot action, use ExecuteDesktopAction.
+	if input.Action == fantasyanthropic.ActionScreenshot {
+		screenshotAction := workspacesdk.DesktopAction{
+			Action:       "screenshot",
+			ScaledWidth:  &scaledW,
+			ScaledHeight: &scaledH,
+		}
+		screenResp, sErr := conn.ExecuteDesktopAction(ctx, screenshotAction)
+		if sErr != nil {
+			return fantasy.NewTextErrorResponse(
+				fmt.Sprintf("screenshot failed: %v", sErr),
+			), nil
+		}
+		return fantasy.NewImageResponse(
+			[]byte(screenResp.ScreenshotData), "image/png",
+		), nil
+	}
+
+	// Build the action request.
+	action := workspacesdk.DesktopAction{
+		Action:       string(input.Action),
+		ScaledWidth:  &scaledW,
+		ScaledHeight: &scaledH,
+	}
+	if input.Coordinate != ([2]int64{}) {
+		coord := [2]int{int(input.Coordinate[0]), int(input.Coordinate[1])}
+		action.Coordinate = &coord
+	}
+	if input.StartCoordinate != ([2]int64{}) {
+		coord := [2]int{int(input.StartCoordinate[0]), int(input.StartCoordinate[1])}
+		action.StartCoordinate = &coord
+	}
+	if input.Text != "" {
+		action.Text = &input.Text
+	}
+	if input.Duration > 0 {
+		d := int(input.Duration)
+		action.Duration = &d
+	}
+	if input.ScrollAmount > 0 {
+		s := int(input.ScrollAmount)
+		action.ScrollAmount = &s
+	}
+	if input.ScrollDirection != "" {
+		action.ScrollDirection = &input.ScrollDirection
+	}
+
+	// Execute the action.
+	_, err = conn.ExecuteDesktopAction(ctx, action)
+	if err != nil {
+		return fantasy.NewTextErrorResponse(
+			fmt.Sprintf("action %q failed: %v", input.Action, err),
+		), nil
+	}
+
+	// Take a screenshot after every action (Anthropic pattern).
+	screenshotAction := workspacesdk.DesktopAction{
+		Action:       "screenshot",
+		ScaledWidth:  &scaledW,
+		ScaledHeight: &scaledH,
+	}
+	screenResp, sErr := conn.ExecuteDesktopAction(ctx, screenshotAction)
+	if sErr != nil {
+		return fantasy.NewTextErrorResponse(
+			fmt.Sprintf("screenshot failed: %v", sErr),
+		), nil
+	}
+
+	return fantasy.NewImageResponse(
+		[]byte(screenResp.ScreenshotData), "image/png",
+	), nil
+}
+
+// computeScaledScreenshotSize computes the target screenshot
+// dimensions to fit within Anthropic's constraints.
+func computeScaledScreenshotSize(width, height int) (scaledWidth int, scaledHeight int) {
+	const maxLongEdge = 1568
+	const maxTotalPixels = 1_150_000
+
+	longEdge := max(width, height)
+	totalPixels := width * height
+	longEdgeScale := float64(maxLongEdge) / float64(longEdge)
+	totalPixelsScale := math.Sqrt(
+		float64(maxTotalPixels) / float64(totalPixels),
+	)
+	scale := min(1.0, longEdgeScale, totalPixelsScale)
+
+	if scale >= 1.0 {
+		return width, height
+	}
+	return max(1, int(float64(width)*scale)),
+		max(1, int(float64(height)*scale))
+}
@@ -0,0 +1,81 @@
+package chattool
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestComputeScaledScreenshotSize(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name          string
+		width, height int
+		wantW, wantH  int
+	}{
+		{
+			name:   "1920x1080_scales_down",
+			width:  1920,
+			height: 1080,
+			wantW:  1429,
+			wantH:  804,
+		},
+		{
+			name:   "1280x800_no_scaling",
+			width:  1280,
+			height: 800,
+			wantW:  1280,
+			wantH:  800,
+		},
+		{
+			name:   "3840x2160_large_display",
+			width:  3840,
+			height: 2160,
+			wantW:  1429,
+			wantH:  804,
+		},
+		{
+			name:   "1568x1000_pixel_cap_applies",
+			width:  1568,
+			height: 1000,
+			wantW:  1342,
+			wantH:  856,
+		},
+		{
+			name:   "100x100_small_display",
+			width:  100,
+			height: 100,
+			wantW:  100,
+			wantH:  100,
+		},
+		{
+			name:  "4000x3000_stays_within_limits",
+			width: 4000,
+			// Both constraints apply. The function should keep
+			// the result within maxLongEdge=1568 and
+			// totalPixels<=1,150,000.
+			height: 3000,
+			wantW:  1238,
+			wantH:  928,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+			gotW, gotH := computeScaledScreenshotSize(tt.width, tt.height)
+			assert.Equal(t, tt.wantW, gotW)
+			assert.Equal(t, tt.wantH, gotH)
+
+			// Invariant: results must respect Anthropic constraints.
+			const maxLongEdge = 1568
+			const maxTotalPixels = 1_150_000
+			longEdge := max(gotW, gotH)
+			assert.LessOrEqual(t, longEdge, maxLongEdge,
+				"long edge %d exceeds max %d", longEdge, maxLongEdge)
+			assert.LessOrEqual(t, gotW*gotH, maxTotalPixels,
+				"total pixels %d exceeds max %d", gotW*gotH, maxTotalPixels)
+		})
+	}
+}
@@ -0,0 +1,186 @@
+package chattool_test
+
+import (
+	"context"
+	"testing"
+
+	"charm.land/fantasy"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.uber.org/mock/gomock"
+	"golang.org/x/xerrors"
+
+	"github.com/coder/coder/v2/coderd/chatd/chattool"
+	"github.com/coder/coder/v2/codersdk/workspacesdk"
+	"github.com/coder/coder/v2/codersdk/workspacesdk/agentconnmock"
+	"github.com/coder/quartz"
+)
+
+func TestComputerUseTool_Info(t *testing.T) {
+	t.Parallel()
+
+	tool := chattool.NewComputerUseTool(workspacesdk.DesktopDisplayWidth, workspacesdk.DesktopDisplayHeight, nil, quartz.NewReal())
+	info := tool.Info()
+	assert.Equal(t, "computer", info.Name)
+	assert.NotEmpty(t, info.Description)
+}
+
+func TestComputerUseProviderTool(t *testing.T) {
+	t.Parallel()
+
+	def := chattool.ComputerUseProviderTool(workspacesdk.DesktopDisplayWidth, workspacesdk.DesktopDisplayHeight)
+	pdt, ok := def.(fantasy.ProviderDefinedTool)
+	require.True(t, ok, "ComputerUseProviderTool should return a ProviderDefinedTool")
+	assert.Contains(t, pdt.ID, "computer")
+	assert.Equal(t, "computer", pdt.Name)
+	// Verify display dimensions are passed through.
+	assert.Equal(t, int64(workspacesdk.DesktopDisplayWidth), pdt.Args["display_width_px"])
+	assert.Equal(t, int64(workspacesdk.DesktopDisplayHeight), pdt.Args["display_height_px"])
+}
+
+func TestComputerUseTool_Run_Screenshot(t *testing.T) {
+	t.Parallel()
+
+	ctrl := gomock.NewController(t)
+	mockConn := agentconnmock.NewMockAgentConn(ctrl)
+
+	mockConn.EXPECT().ExecuteDesktopAction(
+		gomock.Any(),
+		gomock.Any(),
+	).Return(workspacesdk.DesktopActionResponse{
+		Output:           "screenshot",
+		ScreenshotData:   "base64png",
+		ScreenshotWidth:  1024,
+		ScreenshotHeight: 768,
+	}, nil)
+
+	tool := chattool.NewComputerUseTool(workspacesdk.DesktopDisplayWidth, workspacesdk.DesktopDisplayHeight, func(_ context.Context) (workspacesdk.AgentConn, error) {
+		return mockConn, nil
+	}, quartz.NewReal())
+
+	call := fantasy.ToolCall{
+		ID:    "test-1",
+		Name:  "computer",
+		Input: `{"action":"screenshot"}`,
+	}
+
+	resp, err := tool.Run(context.Background(), call)
+	require.NoError(t, err)
+	assert.Equal(t, "image", resp.Type)
+	assert.Equal(t, "image/png", resp.MediaType)
+	assert.Equal(t, []byte("base64png"), resp.Data)
+	assert.False(t, resp.IsError)
+}
+
+func TestComputerUseTool_Run_LeftClick(t *testing.T) {
+	t.Parallel()
+
+	ctrl := gomock.NewController(t)
+	mockConn := agentconnmock.NewMockAgentConn(ctrl)
+
+	// Expect the action call first.
+	mockConn.EXPECT().ExecuteDesktopAction(
+		gomock.Any(),
+		gomock.Any(),
+	).Return(workspacesdk.DesktopActionResponse{
+		Output: "left_click performed",
+	}, nil)
+
+	// Then expect a screenshot (auto-screenshot after action).
+	mockConn.EXPECT().ExecuteDesktopAction(
+		gomock.Any(),
+		gomock.Any(),
+	).Return(workspacesdk.DesktopActionResponse{
+		Output:           "screenshot",
+		ScreenshotData:   "after-click",
+		ScreenshotWidth:  1024,
+		ScreenshotHeight: 768,
+	}, nil)
+
+	tool := chattool.NewComputerUseTool(workspacesdk.DesktopDisplayWidth, workspacesdk.DesktopDisplayHeight, func(_ context.Context) (workspacesdk.AgentConn, error) {
+		return mockConn, nil
+	}, quartz.NewReal())
+
+	call := fantasy.ToolCall{
+		ID:    "test-2",
+		Name:  "computer",
+		Input: `{"action":"left_click","coordinate":[100,200]}`,
+	}
+
+	resp, err := tool.Run(context.Background(), call)
+	require.NoError(t, err)
+	assert.Equal(t, "image", resp.Type)
+	assert.Equal(t, []byte("after-click"), resp.Data)
+}
+
+func TestComputerUseTool_Run_Wait(t *testing.T) {
+	t.Parallel()
+
+	ctrl := gomock.NewController(t)
+	mockConn := agentconnmock.NewMockAgentConn(ctrl)
+	// Expect a screenshot after the wait completes.
+	mockConn.EXPECT().ExecuteDesktopAction(
+		gomock.Any(),
+		gomock.Any(),
+	).Return(workspacesdk.DesktopActionResponse{
+		Output:           "screenshot",
+		ScreenshotData:   "after-wait",
+		ScreenshotWidth:  1024,
+		ScreenshotHeight: 768,
+	}, nil)
+
+	tool := chattool.NewComputerUseTool(workspacesdk.DesktopDisplayWidth, workspacesdk.DesktopDisplayHeight, func(_ context.Context) (workspacesdk.AgentConn, error) {
+		return mockConn, nil
+	}, quartz.NewReal())
+
+	call := fantasy.ToolCall{
+		ID:    "test-3",
+		Name:  "computer",
+		Input: `{"action":"wait","duration":10}`,
+	}
+
+	resp, err := tool.Run(context.Background(), call)
+	require.NoError(t, err)
+	assert.Equal(t, "image", resp.Type)
+	assert.Equal(t, "image/png", resp.MediaType)
+	assert.Equal(t, []byte("after-wait"), resp.Data)
+	assert.False(t, resp.IsError)
+}
+
+func TestComputerUseTool_Run_ConnError(t *testing.T) {
+	t.Parallel()
+
+	tool := chattool.NewComputerUseTool(workspacesdk.DesktopDisplayWidth, workspacesdk.DesktopDisplayHeight, func(_ context.Context) (workspacesdk.AgentConn, error) {
+		return nil, xerrors.New("workspace not available")
+	}, quartz.NewReal())
+
+	call := fantasy.ToolCall{
+		ID:    "test-4",
+		Name:  "computer",
+		Input: `{"action":"screenshot"}`,
+	}
+
+	resp, err := tool.Run(context.Background(), call)
+	require.NoError(t, err)
+	assert.True(t, resp.IsError)
+	assert.Contains(t, resp.Content, "workspace not available")
+}
+
+func TestComputerUseTool_Run_InvalidInput(t *testing.T) {
+	t.Parallel()
+
+	tool := chattool.NewComputerUseTool(workspacesdk.DesktopDisplayWidth, workspacesdk.DesktopDisplayHeight, func(_ context.Context) (workspacesdk.AgentConn, error) {
+		return nil, xerrors.New("should not be called")
+	}, quartz.NewReal())
+
+	call := fantasy.ToolCall{
+		ID:    "test-5",
+		Name:  "computer",
+		Input: `{invalid json`,
+	}
+
+	resp, err := tool.Run(context.Background(), call)
+	require.NoError(t, err)
+	assert.True(t, resp.IsError)
+	assert.Contains(t, resp.Content, "invalid computer use input")
+}