feat: agents desktop recordings backend (#23894)

This PR introduces screen recording of the computer use agent using the
virtual desktop.

- Screen recording is triggered by a `wait_agent` tool call. Recording
is stopped by a successful `wait_agent` tool call or when there hasn't
been any desktop activity for 10 minutes.
- Recordings are handled by the `portabledesktop` cli via the `record`
command. The videos are sped up in periods of inactivity.
- Recordings are saved to the database to the `chat_files` table.
There's a hard limit of 100MB per recording. Larger recordings are
dropped.
- A successful `wait_agent` on a computer use subagent tool call returns
a `recording_file_id`, later allowing the frontend to display the
corresponding video.
This commit is contained in:
Hugo Dutka
2026-04-02 17:23:27 +00:00
committed by GitHub
parent f796f3645f
commit 17dec2a70f
14 changed files with 2480 additions and 29 deletions
+56
View File
@@ -94,6 +94,8 @@ type AgentConn interface {
WatchGit(ctx context.Context, logger slog.Logger, chatID uuid.UUID) (*wsjson.Stream[codersdk.WorkspaceAgentGitServerMessage, codersdk.WorkspaceAgentGitClientMessage], error)
ConnectDesktopVNC(ctx context.Context) (net.Conn, error)
ExecuteDesktopAction(ctx context.Context, action DesktopAction) (DesktopActionResponse, error)
StartDesktopRecording(ctx context.Context, req StartDesktopRecordingRequest) error
StopDesktopRecording(ctx context.Context, req StopDesktopRecordingRequest) (io.ReadCloser, error)
}
// AgentConn represents a connection to a workspace agent.
@@ -596,6 +598,23 @@ type DesktopActionResponse struct {
ScreenshotHeight int `json:"screenshot_height,omitempty"`
}
// StartDesktopRecordingRequest is the request body for starting a
// desktop recording session.
type StartDesktopRecordingRequest struct {
RecordingID string `json:"recording_id"`
}
// StopDesktopRecordingRequest is the request body for stopping a
// desktop recording session.
type StopDesktopRecordingRequest struct {
RecordingID string `json:"recording_id"`
}
// MaxRecordingSize is the largest desktop recording (in bytes)
// that will be accepted. Used by both the agent-side stop handler
// and the server-side storage pipeline.
const MaxRecordingSize = 100 << 20 // 100 MB
// ExecuteDesktopAction executes a mouse/keyboard/scroll action on the
// agent's desktop.
func (c *agentConn) ExecuteDesktopAction(ctx context.Context, action DesktopAction) (DesktopActionResponse, error) {
@@ -643,6 +662,43 @@ func (c *agentConn) ExecuteDesktopAction(ctx context.Context, action DesktopActi
return result, nil
}
// StartDesktopRecording starts a desktop recording session on the
// agent with the given recording ID. The recording ID is
// caller-provided and must be unique. Idempotent — if the ID is
// already recording, returns success.
func (c *agentConn) StartDesktopRecording(ctx context.Context, req StartDesktopRecordingRequest) error {
ctx, span := tracing.StartSpan(ctx)
defer span.End()
res, err := c.apiRequest(ctx, http.MethodPost, "/api/v0/desktop/recording/start", req)
if err != nil {
return xerrors.Errorf("start recording request: %w", err)
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
return codersdk.ReadBodyAsError(res)
}
return nil
}
// StopDesktopRecording stops a desktop recording session on the
// agent and returns the MP4 data as an io.ReadCloser. The caller
// is responsible for closing the returned reader. Idempotent —
// safe to call on an already-stopped recording.
func (c *agentConn) StopDesktopRecording(ctx context.Context, req StopDesktopRecordingRequest) (io.ReadCloser, error) {
ctx, span := tracing.StartSpan(ctx)
defer span.End()
res, err := c.apiRequest(ctx, http.MethodPost, "/api/v0/desktop/recording/stop", req)
if err != nil {
return nil, xerrors.Errorf("stop recording request: %w", err)
}
if res.StatusCode != http.StatusOK {
defer res.Body.Close()
return nil, codersdk.ReadBodyAsError(res)
}
// Caller is responsible for closing res.Body.
return res.Body, nil
}
// DeleteDevcontainer deletes the provided devcontainer.
// This is a blocking call and will wait for the container to be deleted.
func (c *agentConn) DeleteDevcontainer(ctx context.Context, devcontainerID string) error {
@@ -550,6 +550,20 @@ func (mr *MockAgentConnMockRecorder) Speedtest(ctx, direction, duration any) *go
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Speedtest", reflect.TypeOf((*MockAgentConn)(nil).Speedtest), ctx, direction, duration)
}
// StartDesktopRecording mocks base method.
func (m *MockAgentConn) StartDesktopRecording(ctx context.Context, req workspacesdk.StartDesktopRecordingRequest) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "StartDesktopRecording", ctx, req)
ret0, _ := ret[0].(error)
return ret0
}
// StartDesktopRecording indicates an expected call of StartDesktopRecording.
func (mr *MockAgentConnMockRecorder) StartDesktopRecording(ctx, req any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StartDesktopRecording", reflect.TypeOf((*MockAgentConn)(nil).StartDesktopRecording), ctx, req)
}
// StartProcess mocks base method.
func (m *MockAgentConn) StartProcess(ctx context.Context, req workspacesdk.StartProcessRequest) (workspacesdk.StartProcessResponse, error) {
m.ctrl.T.Helper()
@@ -565,6 +579,21 @@ func (mr *MockAgentConnMockRecorder) StartProcess(ctx, req any) *gomock.Call {
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StartProcess", reflect.TypeOf((*MockAgentConn)(nil).StartProcess), ctx, req)
}
// StopDesktopRecording mocks base method.
func (m *MockAgentConn) StopDesktopRecording(ctx context.Context, req workspacesdk.StopDesktopRecordingRequest) (io.ReadCloser, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "StopDesktopRecording", ctx, req)
ret0, _ := ret[0].(io.ReadCloser)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// StopDesktopRecording indicates an expected call of StopDesktopRecording.
func (mr *MockAgentConnMockRecorder) StopDesktopRecording(ctx, req any) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StopDesktopRecording", reflect.TypeOf((*MockAgentConn)(nil).StopDesktopRecording), ctx, req)
}
// TailnetConn mocks base method.
func (m *MockAgentConn) TailnetConn() *tailnet.Conn {
m.ctrl.T.Helper()