fix: bump agents desktop resolution to 1920x1080 (#23425)

This PR changes agents desktop resolution from 1366x768 to 1920x1080. Anthropic requires the that the resolution of desktop screenshots fits in 1,150,000 total pixels, so we downscale screenshots to 1280x720 before sending them to the LLM provider. Resolution scaling was already implemented, but our code didn't exercise it. The resolution bump showed that there were some bugs in the scaling logic - this PR fixes these bugs too.
2026-09-24 15:04:27 +08:00 · 2026-03-23 11:51:10 +01:00
parent eca2257c26
commit 3163e74b77
10 changed files with 678 additions and 174 deletions
@@ -578,8 +578,10 @@ type DesktopAction struct {
 	Duration        *int    `json:"duration,omitempty"`
 	ScrollAmount    *int    `json:"scroll_amount,omitempty"`
 	ScrollDirection *string `json:"scroll_direction,omitempty"`
-	ScaledWidth     *int    `json:"scaled_width,omitempty"`
-	ScaledHeight    *int    `json:"scaled_height,omitempty"`
+	// ScaledWidth and ScaledHeight carry the declared model-facing desktop
+	// geometry used for screenshot sizing and coordinate mapping.
+	ScaledWidth  *int `json:"scaled_width,omitempty"`
+	ScaledHeight *int `json:"scaled_height,omitempty"`
 }

 // DesktopActionResponse is the response from the desktop action
@@ -1,10 +1,167 @@
 package workspacesdk

+import "math"
+
 const (
-	// DesktopDisplayWidth is the default display width in pixels
-	// used for computer-use desktop sessions.
-	DesktopDisplayWidth = 1366
-	// DesktopDisplayHeight is the default display height in pixels
-	// used for computer-use desktop sessions.
-	DesktopDisplayHeight = 768
+	// DesktopNativeWidth is the default native desktop width in pixels used for
+	// computer-use desktop sessions.
+	DesktopNativeWidth = 1920
+	// DesktopNativeHeight is the default native desktop height in pixels used for
+	// computer-use desktop sessions.
+	DesktopNativeHeight = 1080
+
+	desktopDeclaredMaxLongEdge    = 1568
+	desktopDeclaredMaxTotalPixels = 1_150_000
 )
+
+var preferredDeclaredDesktopWidths = []int{1280, 1024}
+
+// DesktopGeometry describes the native workspace desktop and the declared
+// model-facing geometry used for screenshots and coordinates.
+type DesktopGeometry struct {
+	NativeWidth    int
+	NativeHeight   int
+	DeclaredWidth  int
+	DeclaredHeight int
+}
+
+// DefaultDesktopGeometry returns the default native desktop geometry together
+// with the declared model-facing geometry derived from it.
+func DefaultDesktopGeometry() DesktopGeometry {
+	return NewDesktopGeometry(DesktopNativeWidth, DesktopNativeHeight)
+}
+
+// NewDesktopGeometry derives a declared model-facing geometry from the native
+// desktop size.
+func NewDesktopGeometry(nativeWidth, nativeHeight int) DesktopGeometry {
+	nativeWidth = sanitizeDesktopDimension(nativeWidth)
+	nativeHeight = sanitizeDesktopDimension(nativeHeight)
+
+	declaredWidth, declaredHeight := computeDeclaredDesktopSize(
+		nativeWidth,
+		nativeHeight,
+	)
+
+	return DesktopGeometry{
+		NativeWidth:    nativeWidth,
+		NativeHeight:   nativeHeight,
+		DeclaredWidth:  declaredWidth,
+		DeclaredHeight: declaredHeight,
+	}
+}
+
+// NewDesktopGeometryWithDeclared returns a geometry that preserves the native
+// desktop size while using the provided declared model-facing dimensions.
+func NewDesktopGeometryWithDeclared(
+	nativeWidth,
+	nativeHeight,
+	declaredWidth,
+	declaredHeight int,
+) DesktopGeometry {
+	nativeWidth = sanitizeDesktopDimension(nativeWidth)
+	nativeHeight = sanitizeDesktopDimension(nativeHeight)
+	if declaredWidth <= 0 {
+		declaredWidth = nativeWidth
+	}
+	if declaredHeight <= 0 {
+		declaredHeight = nativeHeight
+	}
+
+	return DesktopGeometry{
+		NativeWidth:    nativeWidth,
+		NativeHeight:   nativeHeight,
+		DeclaredWidth:  sanitizeDesktopDimension(declaredWidth),
+		DeclaredHeight: sanitizeDesktopDimension(declaredHeight),
+	}
+}
+
+// DeclaredPointToNative maps a point from declared model-facing coordinates to
+// native desktop coordinates using the existing pixel-center truncation rule.
+func (g DesktopGeometry) DeclaredPointToNative(x, y int) (nativeX, nativeY int) {
+	return scaleDesktopCoordinate(x, g.DeclaredWidth, g.NativeWidth),
+		scaleDesktopCoordinate(y, g.DeclaredHeight, g.NativeHeight)
+}
+
+// NativePointToDeclared maps a point from native desktop coordinates to the
+// declared model-facing coordinate space using the same truncating transform.
+func (g DesktopGeometry) NativePointToDeclared(x, y int) (declaredX, declaredY int) {
+	return scaleDesktopCoordinate(x, g.NativeWidth, g.DeclaredWidth),
+		scaleDesktopCoordinate(y, g.NativeHeight, g.DeclaredHeight)
+}
+
+func computeDeclaredDesktopSize(nativeWidth, nativeHeight int) (declaredWidth, declaredHeight int) {
+	if desktopSizeFitsDeclaredLimits(nativeWidth, nativeHeight) {
+		return nativeWidth, nativeHeight
+	}
+
+	if nativeWidth >= nativeHeight {
+		for _, declaredWidth := range preferredDeclaredDesktopWidths {
+			if declaredWidth > nativeWidth {
+				continue
+			}
+
+			declaredHeight := max(1, declaredWidth*nativeHeight/nativeWidth)
+			if desktopSizeFitsDeclaredLimits(declaredWidth, declaredHeight) {
+				return declaredWidth, declaredHeight
+			}
+		}
+	}
+
+	return computeGenericDeclaredDesktopSize(nativeWidth, nativeHeight)
+}
+
+func desktopSizeFitsDeclaredLimits(width, height int) bool {
+	return max(width, height) <= desktopDeclaredMaxLongEdge &&
+		width*height <= desktopDeclaredMaxTotalPixels
+}
+
+func computeGenericDeclaredDesktopSize(width, height int) (scaledWidth, scaledHeight int) {
+	longEdge := max(width, height)
+	totalPixels := width * height
+	longEdgeScale := float64(desktopDeclaredMaxLongEdge) / float64(longEdge)
+	totalPixelsScale := math.Sqrt(
+		float64(desktopDeclaredMaxTotalPixels) / float64(totalPixels),
+	)
+	scale := min(1.0, longEdgeScale, totalPixelsScale)
+
+	if scale >= 1.0 {
+		return width, height
+	}
+
+	return max(1, int(float64(width)*scale)),
+		max(1, int(float64(height)*scale))
+}
+
+func scaleDesktopCoordinate(coord, fromDim, toDim int) int {
+	if toDim <= 0 {
+		return 0
+	}
+	if fromDim <= 0 || fromDim == toDim {
+		return clampDesktopCoordinate(coord, toDim)
+	}
+
+	scaled := (float64(coord)+0.5)*float64(toDim)/float64(fromDim) - 0.5
+	scaled = math.Max(scaled, 0)
+	scaled = math.Min(scaled, float64(toDim-1))
+	return int(math.Round(scaled))
+}
+
+func clampDesktopCoordinate(coord, dim int) int {
+	if dim <= 0 {
+		return 0
+	}
+	if coord < 0 {
+		return 0
+	}
+	if coord >= dim {
+		return dim - 1
+	}
+	return coord
+}
+
+func sanitizeDesktopDimension(dim int) int {
+	if dim <= 0 {
+		return 1
+	}
+	return dim
+}
@@ -0,0 +1,213 @@
+package workspacesdk_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/coder/coder/v2/codersdk/workspacesdk"
+)
+
+func TestNewDesktopGeometry(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name           string
+		nativeWidth    int
+		nativeHeight   int
+		declaredWidth  int
+		declaredHeight int
+	}{
+		{
+			name:           "1366x768_keeps_native_geometry",
+			nativeWidth:    1366,
+			nativeHeight:   768,
+			declaredWidth:  1366,
+			declaredHeight: 768,
+		},
+		{
+			name:           "1920x1080_prefers_1280x720",
+			nativeWidth:    1920,
+			nativeHeight:   1080,
+			declaredWidth:  1280,
+			declaredHeight: 720,
+		},
+		{
+			name:           "1920x1200_prefers_1280x800",
+			nativeWidth:    1920,
+			nativeHeight:   1200,
+			declaredWidth:  1280,
+			declaredHeight: 800,
+		},
+		{
+			name:           "2048x1536_prefers_1024x768",
+			nativeWidth:    2048,
+			nativeHeight:   1536,
+			declaredWidth:  1024,
+			declaredHeight: 768,
+		},
+		{
+			name:           "3840x2160_prefers_1280x720",
+			nativeWidth:    3840,
+			nativeHeight:   2160,
+			declaredWidth:  1280,
+			declaredHeight: 720,
+		},
+		{
+			name:           "1568x1000_prefers_1280x816",
+			nativeWidth:    1568,
+			nativeHeight:   1000,
+			declaredWidth:  1280,
+			declaredHeight: 816,
+		},
+		{
+			name:           "portrait_falls_back_to_generic_scaling",
+			nativeWidth:    1000,
+			nativeHeight:   2000,
+			declaredWidth:  758,
+			declaredHeight: 1516,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			geometry := workspacesdk.NewDesktopGeometry(
+				tt.nativeWidth,
+				tt.nativeHeight,
+			)
+
+			assert.Equal(t, tt.nativeWidth, geometry.NativeWidth)
+			assert.Equal(t, tt.nativeHeight, geometry.NativeHeight)
+			assert.Equal(t, tt.declaredWidth, geometry.DeclaredWidth)
+			assert.Equal(t, tt.declaredHeight, geometry.DeclaredHeight)
+			assert.LessOrEqual(t, max(geometry.DeclaredWidth, geometry.DeclaredHeight), 1568)
+			assert.LessOrEqual(t, geometry.DeclaredWidth*geometry.DeclaredHeight, 1_150_000)
+		})
+	}
+}
+
+func TestDefaultDesktopGeometry(t *testing.T) {
+	t.Parallel()
+
+	geometry := workspacesdk.DefaultDesktopGeometry()
+
+	assert.Equal(t, workspacesdk.DesktopNativeWidth, geometry.NativeWidth)
+	assert.Equal(t, workspacesdk.DesktopNativeHeight, geometry.NativeHeight)
+	assert.Equal(t, 1280, geometry.DeclaredWidth)
+	assert.Equal(t, 720, geometry.DeclaredHeight)
+}
+
+func TestDesktopGeometryDeclaredPointToNative(t *testing.T) {
+	t.Parallel()
+
+	geometry := workspacesdk.NewDesktopGeometryWithDeclared(1920, 1080, 1280, 720)
+
+	tests := []struct {
+		name  string
+		x     int
+		y     int
+		wantX int
+		wantY int
+	}{
+		{
+			name:  "origin",
+			x:     0,
+			y:     0,
+			wantX: 0,
+			wantY: 0,
+		},
+		{
+			name:  "center",
+			x:     640,
+			y:     360,
+			wantX: 960,
+			wantY: 540,
+		},
+		{
+			name:  "max_coordinate_maps_to_last_native_pixel",
+			x:     1279,
+			y:     719,
+			wantX: 1919,
+			wantY: 1079,
+		},
+		{
+			name:  "out_of_bounds_values_are_clamped",
+			x:     5000,
+			y:     -5,
+			wantX: 1919,
+			wantY: 0,
+		},
+		{
+			name:  "rounding_applies",
+			x:     853,
+			y:     402,
+			wantX: 1280,
+			wantY: 603,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			gotX, gotY := geometry.DeclaredPointToNative(tt.x, tt.y)
+			assert.Equal(t, tt.wantX, gotX)
+			assert.Equal(t, tt.wantY, gotY)
+		})
+	}
+}
+
+func TestDesktopGeometryNativePointToDeclared(t *testing.T) {
+	t.Parallel()
+
+	geometry := workspacesdk.NewDesktopGeometryWithDeclared(1920, 1080, 1366, 768)
+
+	tests := []struct {
+		name  string
+		x     int
+		y     int
+		wantX int
+		wantY int
+	}{
+		{
+			name:  "origin",
+			x:     0,
+			y:     0,
+			wantX: 0,
+			wantY: 0,
+		},
+		{
+			name:  "center",
+			x:     960,
+			y:     540,
+			wantX: 683,
+			wantY: 384,
+		},
+		{
+			name:  "bottom_right_maps_to_last_pixel",
+			x:     1919,
+			y:     1079,
+			wantX: 1365,
+			wantY: 767,
+		},
+		{
+			name:  "out_of_bounds_values_are_clamped",
+			x:     -10,
+			y:     5000,
+			wantX: 0,
+			wantY: 767,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			gotX, gotY := geometry.NativePointToDeclared(tt.x, tt.y)
+			assert.Equal(t, tt.wantX, gotX)
+			assert.Equal(t, tt.wantY, gotY)
+		})
+	}
+}