opensource/coder
3
0
Fork 0
mirror of https://github.com/coder/coder.git synced 2026-09-24 15:04:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: bump agents desktop resolution to 1920x1080 (#23425)

Browse Source 
This PR changes agents desktop resolution from 1366x768 to 1920x1080.
Anthropic requires the that the resolution of desktop screenshots fits
in 1,150,000 total pixels, so we downscale screenshots to 1280x720
before sending them to the LLM provider.

Resolution scaling was already implemented, but our code didn't exercise
it. The resolution bump showed that there were some bugs in the scaling
logic - this PR fixes these bugs too.
This commit is contained in:
Hugo Dutka
2026-03-23 11:51:10 +01:00
committed by GitHub
parent eca2257c26
commit 3163e74b77
10 changed files with 678 additions and 174 deletions
Download Patch File Download Diff File Expand all files Collapse all files
coderd/chatd/chatd.go
+8 -5
View File
@@ -3272,14 +3272,17 @@ func (p *Server) runChat(
}
if isComputerUse {
desktopGeometry := workspacesdk.DefaultDesktopGeometry()
providerTools = append(providerTools, chatloop.ProviderTool{
Definition: chattool.ComputerUseProviderTool(
workspacesdk.DesktopDisplayWidth,
workspacesdk.DesktopDisplayHeight),
desktopGeometry.DeclaredWidth,
desktopGeometry.DeclaredHeight,
),
Runner: chattool.NewComputerUseTool(
workspacesdk.DesktopDisplayWidth,
workspacesdk.DesktopDisplayHeight,
workspaceCtx.getWorkspaceConn, quartz.NewReal(),
desktopGeometry.DeclaredWidth,
desktopGeometry.DeclaredHeight,
workspaceCtx.getWorkspaceConn,
quartz.NewReal(),
),
})
}
coderd/chatd/chattool/computeruse.go
+35 -44
View File
@@ -3,7 +3,6 @@ package chattool
import (
"context"
"fmt"
"math"
"time"
"charm.land/fantasy"
@@ -25,23 +24,25 @@ const (
// computerUseTool implements fantasy.AgentTool and
// chatloop.ToolDefiner for Anthropic computer use.
type computerUseTool struct {
displayWidth int
displayHeight int
declaredWidth int
declaredHeight int
getWorkspaceConn func(ctx context.Context) (workspacesdk.AgentConn, error)
providerOptions fantasy.ProviderOptions
clock quartz.Clock
}
// NewComputerUseTool creates a computer use AgentTool that
// delegates to the agent's desktop endpoints.
// NewComputerUseTool creates a computer use AgentTool that delegates to the
// agent's desktop endpoints. declaredWidth and declaredHeight are the
// model-facing desktop dimensions advertised to Anthropic and requested for
// screenshots.
func NewComputerUseTool(
displayWidth, displayHeight int,
declaredWidth, declaredHeight int,
getWorkspaceConn func(ctx context.Context) (workspacesdk.AgentConn, error),
clock quartz.Clock,
) fantasy.AgentTool {
return &computerUseTool{
displayWidth: displayWidth,
displayHeight: displayHeight,
declaredWidth: declaredWidth,
declaredHeight: declaredHeight,
getWorkspaceConn: getWorkspaceConn,
clock: clock,
}
@@ -56,14 +57,13 @@ func (*computerUseTool) Info() fantasy.ToolInfo {
}
}
// ComputerUseProviderTool creates the provider-defined tool
// definition for Anthropic computer use. This is passed via
// ProviderTools so the API receives the correct wire format.
func ComputerUseProviderTool(displayWidth, displayHeight int) fantasy.Tool {
// ComputerUseProviderTool creates the provider-defined Anthropic computer-use
// tool definition using the declared model-facing desktop geometry.
func ComputerUseProviderTool(declaredWidth, declaredHeight int) fantasy.Tool {
return fantasyanthropic.NewComputerUseTool(
fantasyanthropic.ComputerUseToolOptions{
DisplayWidthPx: int64(displayWidth),
DisplayHeightPx: int64(displayHeight),
DisplayWidthPx: int64(declaredWidth),
DisplayHeightPx: int64(declaredHeight),
ToolVersion: fantasyanthropic.ComputerUse20251124,
},
)
@@ -92,10 +92,7 @@ func (t *computerUseTool) Run(ctx context.Context, call fantasy.ToolCall) (fanta
), nil
}
// Compute scaled screenshot size for Anthropic constraints.
scaledW, scaledH := computeScaledScreenshotSize(
t.displayWidth, t.displayHeight,
)
declaredWidth, declaredHeight := t.declaredActionDimensions()
// For wait actions, sleep then return a screenshot.
if input.Action == fantasyanthropic.ActionWait {
@@ -111,8 +108,8 @@ func (t *computerUseTool) Run(ctx context.Context, call fantasy.ToolCall) (fanta
}
screenshotAction := workspacesdk.DesktopAction{
Action: "screenshot",
ScaledWidth: &scaledW,
ScaledHeight: &scaledH,
ScaledWidth: &declaredWidth,
ScaledHeight: &declaredHeight,
}
screenResp, sErr := conn.ExecuteDesktopAction(ctx, screenshotAction)
if sErr != nil {
@@ -129,8 +126,8 @@ func (t *computerUseTool) Run(ctx context.Context, call fantasy.ToolCall) (fanta
if input.Action == fantasyanthropic.ActionScreenshot {
screenshotAction := workspacesdk.DesktopAction{
Action: "screenshot",
ScaledWidth: &scaledW,
ScaledHeight: &scaledH,
ScaledWidth: &declaredWidth,
ScaledHeight: &declaredHeight,
}
screenResp, sErr := conn.ExecuteDesktopAction(ctx, screenshotAction)
if sErr != nil {
@@ -146,8 +143,8 @@ func (t *computerUseTool) Run(ctx context.Context, call fantasy.ToolCall) (fanta
// Build the action request.
action := workspacesdk.DesktopAction{
Action: string(input.Action),
ScaledWidth: &scaledW,
ScaledHeight: &scaledH,
ScaledWidth: &declaredWidth,
ScaledHeight: &declaredHeight,
}
if input.Coordinate != ([2]int64{}) {
coord := [2]int{int(input.Coordinate[0]), int(input.Coordinate[1])}
@@ -183,8 +180,8 @@ func (t *computerUseTool) Run(ctx context.Context, call fantasy.ToolCall) (fanta
// Take a screenshot after every action (Anthropic pattern).
screenshotAction := workspacesdk.DesktopAction{
Action: "screenshot",
ScaledWidth: &scaledW,
ScaledHeight: &scaledH,
ScaledWidth: &declaredWidth,
ScaledHeight: &declaredHeight,
}
screenResp, sErr := conn.ExecuteDesktopAction(ctx, screenshotAction)
if sErr != nil {
@@ -198,23 +195,17 @@ func (t *computerUseTool) Run(ctx context.Context, call fantasy.ToolCall) (fanta
), nil
}
// computeScaledScreenshotSize computes the target screenshot
// dimensions to fit within Anthropic's constraints.
func computeScaledScreenshotSize(width, height int) (scaledWidth int, scaledHeight int) {
const maxLongEdge = 1568
const maxTotalPixels = 1_150_000
longEdge := max(width, height)
totalPixels := width * height
longEdgeScale := float64(maxLongEdge) / float64(longEdge)
totalPixelsScale := math.Sqrt(
float64(maxTotalPixels) / float64(totalPixels),
)
scale := min(1.0, longEdgeScale, totalPixelsScale)
if scale >= 1.0 {
return width, height
func (t *computerUseTool) declaredActionDimensions() (declaredWidth, declaredHeight int) {
if t.declaredWidth <= 0 || t.declaredHeight <= 0 {
geometry := workspacesdk.DefaultDesktopGeometry()
return geometry.DeclaredWidth, geometry.DeclaredHeight
}
return max(1, int(float64(width)*scale)),
max(1, int(float64(height)*scale))
return t.declaredWidth, t.declaredHeight
}
// computeScaledScreenshotSize preserves the historical helper name while using
// the shared declared-geometry selection logic.
func computeScaledScreenshotSize(width, height int) (scaledWidth int, scaledHeight int) {
geometry := workspacesdk.NewDesktopGeometry(width, height)
return geometry.DeclaredWidth, geometry.DeclaredHeight
}
coderd/chatd/chattool/computeruse_internal_test.go
+20 -16
View File
@@ -15,11 +15,11 @@ func TestComputeScaledScreenshotSize(t *testing.T) {
wantW, wantH int
}{
{
name: "1920x1080_scales_down",
name: "1920x1080_prefers_standard_1280x720",
width: 1920,
height: 1080,
wantW: 1429,
wantH: 804,
wantW: 1280,
wantH: 720,
},
{
name: "1280x800_no_scaling",
@@ -29,18 +29,18 @@ func TestComputeScaledScreenshotSize(t *testing.T) {
wantH: 800,
},
{
name: "3840x2160_large_display",
name: "3840x2160_prefers_standard_1280x720",
width: 3840,
height: 2160,
wantW: 1429,
wantH: 804,
wantW: 1280,
wantH: 720,
},
{
name: "1568x1000_pixel_cap_applies",
name: "1568x1000_prefers_standard_1280x816",
width: 1568,
height: 1000,
wantW: 1342,
wantH: 856,
wantW: 1280,
wantH: 816,
},
{
name: "100x100_small_display",
@@ -50,14 +50,18 @@ func TestComputeScaledScreenshotSize(t *testing.T) {
wantH: 100,
},
{
name: "4000x3000_stays_within_limits",
width: 4000,
// Both constraints apply. The function should keep
// the result within maxLongEdge=1568 and
// totalPixels<=1,150,000.
name: "4000x3000_prefers_standard_1024x768",
width: 4000,
height: 3000,
wantW: 1238,
wantH: 928,
wantW: 1024,
wantH: 768,
},
{
name: "1920x1200_prefers_standard_1280x800",
width: 1920,
height: 1200,
wantW: 1280,
wantH: 800,
},
}
coderd/chatd/chattool/computeruse_test.go
+78 -38
View File
@@ -19,7 +19,8 @@ import (
func TestComputerUseTool_Info(t *testing.T) {
t.Parallel()
tool := chattool.NewComputerUseTool(workspacesdk.DesktopDisplayWidth, workspacesdk.DesktopDisplayHeight, nil, quartz.NewReal())
geometry := workspacesdk.DefaultDesktopGeometry()
tool := chattool.NewComputerUseTool(geometry.DeclaredWidth, geometry.DeclaredHeight, nil, quartz.NewReal())
info := tool.Info()
assert.Equal(t, "computer", info.Name)
assert.NotEmpty(t, info.Description)
@@ -28,14 +29,25 @@ func TestComputerUseTool_Info(t *testing.T) {
func TestComputerUseProviderTool(t *testing.T) {
t.Parallel()
def := chattool.ComputerUseProviderTool(workspacesdk.DesktopDisplayWidth, workspacesdk.DesktopDisplayHeight)
geometry := workspacesdk.DefaultDesktopGeometry()
def := chattool.ComputerUseProviderTool(geometry.DeclaredWidth, geometry.DeclaredHeight)
pdt, ok := def.(fantasy.ProviderDefinedTool)
require.True(t, ok, "ComputerUseProviderTool should return a ProviderDefinedTool")
assert.Contains(t, pdt.ID, "computer")
assert.Equal(t, "computer", pdt.Name)
// Verify display dimensions are passed through.
assert.Equal(t, int64(workspacesdk.DesktopDisplayWidth), pdt.Args["display_width_px"])
assert.Equal(t, int64(workspacesdk.DesktopDisplayHeight), pdt.Args["display_height_px"])
assert.Equal(t, int64(geometry.DeclaredWidth), pdt.Args["display_width_px"])
assert.Equal(t, int64(geometry.DeclaredHeight), pdt.Args["display_height_px"])
}
func TestComputerUseProviderTool_PrefersDeclaredGeometry(t *testing.T) {
t.Parallel()
geometry := workspacesdk.NewDesktopGeometry(1920, 1080)
def := chattool.ComputerUseProviderTool(geometry.DeclaredWidth, geometry.DeclaredHeight)
pdt, ok := def.(fantasy.ProviderDefinedTool)
require.True(t, ok, "ComputerUseProviderTool should return a ProviderDefinedTool")
assert.Equal(t, int64(1280), pdt.Args["display_width_px"])
assert.Equal(t, int64(720), pdt.Args["display_height_px"])
}
func TestComputerUseTool_Run_Screenshot(t *testing.T) {
@@ -43,18 +55,25 @@ func TestComputerUseTool_Run_Screenshot(t *testing.T) {
ctrl := gomock.NewController(t)
mockConn := agentconnmock.NewMockAgentConn(ctrl)
geometry := workspacesdk.DefaultDesktopGeometry()
mockConn.EXPECT().ExecuteDesktopAction(
gomock.Any(),
gomock.Any(),
).Return(workspacesdk.DesktopActionResponse{
Output: "screenshot",
ScreenshotData: "base64png",
ScreenshotWidth: 1024,
ScreenshotHeight: 768,
}, nil)
gomock.AssignableToTypeOf(workspacesdk.DesktopAction{}),
).DoAndReturn(func(_ context.Context, action workspacesdk.DesktopAction) (workspacesdk.DesktopActionResponse, error) {
require.NotNil(t, action.ScaledWidth)
require.NotNil(t, action.ScaledHeight)
assert.Equal(t, geometry.DeclaredWidth, *action.ScaledWidth)
assert.Equal(t, geometry.DeclaredHeight, *action.ScaledHeight)
return workspacesdk.DesktopActionResponse{
Output: "screenshot",
ScreenshotData: "base64png",
ScreenshotWidth: geometry.DeclaredWidth,
ScreenshotHeight: geometry.DeclaredHeight,
}, nil
})
tool := chattool.NewComputerUseTool(workspacesdk.DesktopDisplayWidth, workspacesdk.DesktopDisplayHeight, func(_ context.Context) (workspacesdk.AgentConn, error) {
tool := chattool.NewComputerUseTool(geometry.DeclaredWidth, geometry.DeclaredHeight, func(_ context.Context) (workspacesdk.AgentConn, error) {
return mockConn, nil
}, quartz.NewReal())
@@ -77,27 +96,39 @@ func TestComputerUseTool_Run_LeftClick(t *testing.T) {
ctrl := gomock.NewController(t)
mockConn := agentconnmock.NewMockAgentConn(ctrl)
geometry := workspacesdk.DefaultDesktopGeometry()
// Expect the action call first.
mockConn.EXPECT().ExecuteDesktopAction(
gomock.Any(),
gomock.Any(),
).Return(workspacesdk.DesktopActionResponse{
Output: "left_click performed",
}, nil)
gomock.AssignableToTypeOf(workspacesdk.DesktopAction{}),
).DoAndReturn(func(_ context.Context, action workspacesdk.DesktopAction) (workspacesdk.DesktopActionResponse, error) {
require.NotNil(t, action.Coordinate)
assert.Equal(t, [2]int{100, 200}, *action.Coordinate)
require.NotNil(t, action.ScaledWidth)
require.NotNil(t, action.ScaledHeight)
assert.Equal(t, geometry.DeclaredWidth, *action.ScaledWidth)
assert.Equal(t, geometry.DeclaredHeight, *action.ScaledHeight)
return workspacesdk.DesktopActionResponse{Output: "left_click performed"}, nil
})
// Then expect a screenshot (auto-screenshot after action).
mockConn.EXPECT().ExecuteDesktopAction(
gomock.Any(),
gomock.Any(),
).Return(workspacesdk.DesktopActionResponse{
Output: "screenshot",
ScreenshotData: "after-click",
ScreenshotWidth: 1024,
ScreenshotHeight: 768,
}, nil)
gomock.AssignableToTypeOf(workspacesdk.DesktopAction{}),
).DoAndReturn(func(_ context.Context, action workspacesdk.DesktopAction) (workspacesdk.DesktopActionResponse, error) {
assert.Equal(t, "screenshot", action.Action)
require.NotNil(t, action.ScaledWidth)
require.NotNil(t, action.ScaledHeight)
assert.Equal(t, geometry.DeclaredWidth, *action.ScaledWidth)
assert.Equal(t, geometry.DeclaredHeight, *action.ScaledHeight)
return workspacesdk.DesktopActionResponse{
Output: "screenshot",
ScreenshotData: "after-click",
ScreenshotWidth: geometry.DeclaredWidth,
ScreenshotHeight: geometry.DeclaredHeight,
}, nil
})
tool := chattool.NewComputerUseTool(workspacesdk.DesktopDisplayWidth, workspacesdk.DesktopDisplayHeight, func(_ context.Context) (workspacesdk.AgentConn, error) {
tool := chattool.NewComputerUseTool(geometry.DeclaredWidth, geometry.DeclaredHeight, func(_ context.Context) (workspacesdk.AgentConn, error) {
return mockConn, nil
}, quartz.NewReal())
@@ -118,18 +149,25 @@ func TestComputerUseTool_Run_Wait(t *testing.T) {
ctrl := gomock.NewController(t)
mockConn := agentconnmock.NewMockAgentConn(ctrl)
// Expect a screenshot after the wait completes.
geometry := workspacesdk.DefaultDesktopGeometry()
mockConn.EXPECT().ExecuteDesktopAction(
gomock.Any(),
gomock.Any(),
).Return(workspacesdk.DesktopActionResponse{
Output: "screenshot",
ScreenshotData: "after-wait",
ScreenshotWidth: 1024,
ScreenshotHeight: 768,
}, nil)
gomock.AssignableToTypeOf(workspacesdk.DesktopAction{}),
).DoAndReturn(func(_ context.Context, action workspacesdk.DesktopAction) (workspacesdk.DesktopActionResponse, error) {
require.NotNil(t, action.ScaledWidth)
require.NotNil(t, action.ScaledHeight)
assert.Equal(t, geometry.DeclaredWidth, *action.ScaledWidth)
assert.Equal(t, geometry.DeclaredHeight, *action.ScaledHeight)
return workspacesdk.DesktopActionResponse{
Output: "screenshot",
ScreenshotData: "after-wait",
ScreenshotWidth: geometry.DeclaredWidth,
ScreenshotHeight: geometry.DeclaredHeight,
}, nil
})
tool := chattool.NewComputerUseTool(workspacesdk.DesktopDisplayWidth, workspacesdk.DesktopDisplayHeight, func(_ context.Context) (workspacesdk.AgentConn, error) {
tool := chattool.NewComputerUseTool(geometry.DeclaredWidth, geometry.DeclaredHeight, func(_ context.Context) (workspacesdk.AgentConn, error) {
return mockConn, nil
}, quartz.NewReal())
@@ -150,7 +188,8 @@ func TestComputerUseTool_Run_Wait(t *testing.T) {
func TestComputerUseTool_Run_ConnError(t *testing.T) {
t.Parallel()
tool := chattool.NewComputerUseTool(workspacesdk.DesktopDisplayWidth, workspacesdk.DesktopDisplayHeight, func(_ context.Context) (workspacesdk.AgentConn, error) {
geometry := workspacesdk.DefaultDesktopGeometry()
tool := chattool.NewComputerUseTool(geometry.DeclaredWidth, geometry.DeclaredHeight, func(_ context.Context) (workspacesdk.AgentConn, error) {
return nil, xerrors.New("workspace not available")
}, quartz.NewReal())
@@ -169,7 +208,8 @@ func TestComputerUseTool_Run_ConnError(t *testing.T) {
func TestComputerUseTool_Run_InvalidInput(t *testing.T) {
t.Parallel()
tool := chattool.NewComputerUseTool(workspacesdk.DesktopDisplayWidth, workspacesdk.DesktopDisplayHeight, func(_ context.Context) (workspacesdk.AgentConn, error) {
geometry := workspacesdk.DefaultDesktopGeometry()
tool := chattool.NewComputerUseTool(geometry.DeclaredWidth, geometry.DeclaredHeight, func(_ context.Context) (workspacesdk.AgentConn, error) {
return nil, xerrors.New("should not be called")
}, quartz.NewReal())