mirror of
https://github.com/coder/coder.git
synced 2026-09-24 15:04:27 +08:00
feat: agents desktop recordings backend (#23894)
This PR introduces screen recording of the computer use agent using the virtual desktop. - Screen recording is triggered by a `wait_agent` tool call. Recording is stopped by a successful `wait_agent` tool call or when there hasn't been any desktop activity for 10 minutes. - Recordings are handled by the `portabledesktop` cli via the `record` command. The videos are sped up in periods of inactivity. - Recordings are saved to the database to the `chat_files` table. There's a hard limit of 100MB per recording. Larger recordings are dropped. - A successful `wait_agent` on a computer use subagent tool call returns a `recording_file_id`, later allowing the frontend to display the corresponding video.
This commit is contained in:
@@ -94,6 +94,8 @@ type AgentConn interface {
|
||||
WatchGit(ctx context.Context, logger slog.Logger, chatID uuid.UUID) (*wsjson.Stream[codersdk.WorkspaceAgentGitServerMessage, codersdk.WorkspaceAgentGitClientMessage], error)
|
||||
ConnectDesktopVNC(ctx context.Context) (net.Conn, error)
|
||||
ExecuteDesktopAction(ctx context.Context, action DesktopAction) (DesktopActionResponse, error)
|
||||
StartDesktopRecording(ctx context.Context, req StartDesktopRecordingRequest) error
|
||||
StopDesktopRecording(ctx context.Context, req StopDesktopRecordingRequest) (io.ReadCloser, error)
|
||||
}
|
||||
|
||||
// AgentConn represents a connection to a workspace agent.
|
||||
@@ -596,6 +598,23 @@ type DesktopActionResponse struct {
|
||||
ScreenshotHeight int `json:"screenshot_height,omitempty"`
|
||||
}
|
||||
|
||||
// StartDesktopRecordingRequest is the request body for starting a
|
||||
// desktop recording session.
|
||||
type StartDesktopRecordingRequest struct {
|
||||
RecordingID string `json:"recording_id"`
|
||||
}
|
||||
|
||||
// StopDesktopRecordingRequest is the request body for stopping a
|
||||
// desktop recording session.
|
||||
type StopDesktopRecordingRequest struct {
|
||||
RecordingID string `json:"recording_id"`
|
||||
}
|
||||
|
||||
// MaxRecordingSize is the largest desktop recording (in bytes)
|
||||
// that will be accepted. Used by both the agent-side stop handler
|
||||
// and the server-side storage pipeline.
|
||||
const MaxRecordingSize = 100 << 20 // 100 MB
|
||||
|
||||
// ExecuteDesktopAction executes a mouse/keyboard/scroll action on the
|
||||
// agent's desktop.
|
||||
func (c *agentConn) ExecuteDesktopAction(ctx context.Context, action DesktopAction) (DesktopActionResponse, error) {
|
||||
@@ -643,6 +662,43 @@ func (c *agentConn) ExecuteDesktopAction(ctx context.Context, action DesktopActi
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// StartDesktopRecording starts a desktop recording session on the
|
||||
// agent with the given recording ID. The recording ID is
|
||||
// caller-provided and must be unique. Idempotent — if the ID is
|
||||
// already recording, returns success.
|
||||
func (c *agentConn) StartDesktopRecording(ctx context.Context, req StartDesktopRecordingRequest) error {
|
||||
ctx, span := tracing.StartSpan(ctx)
|
||||
defer span.End()
|
||||
res, err := c.apiRequest(ctx, http.MethodPost, "/api/v0/desktop/recording/start", req)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("start recording request: %w", err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode != http.StatusOK {
|
||||
return codersdk.ReadBodyAsError(res)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// StopDesktopRecording stops a desktop recording session on the
|
||||
// agent and returns the MP4 data as an io.ReadCloser. The caller
|
||||
// is responsible for closing the returned reader. Idempotent —
|
||||
// safe to call on an already-stopped recording.
|
||||
func (c *agentConn) StopDesktopRecording(ctx context.Context, req StopDesktopRecordingRequest) (io.ReadCloser, error) {
|
||||
ctx, span := tracing.StartSpan(ctx)
|
||||
defer span.End()
|
||||
res, err := c.apiRequest(ctx, http.MethodPost, "/api/v0/desktop/recording/stop", req)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("stop recording request: %w", err)
|
||||
}
|
||||
if res.StatusCode != http.StatusOK {
|
||||
defer res.Body.Close()
|
||||
return nil, codersdk.ReadBodyAsError(res)
|
||||
}
|
||||
// Caller is responsible for closing res.Body.
|
||||
return res.Body, nil
|
||||
}
|
||||
|
||||
// DeleteDevcontainer deletes the provided devcontainer.
|
||||
// This is a blocking call and will wait for the container to be deleted.
|
||||
func (c *agentConn) DeleteDevcontainer(ctx context.Context, devcontainerID string) error {
|
||||
|
||||
@@ -550,6 +550,20 @@ func (mr *MockAgentConnMockRecorder) Speedtest(ctx, direction, duration any) *go
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Speedtest", reflect.TypeOf((*MockAgentConn)(nil).Speedtest), ctx, direction, duration)
|
||||
}
|
||||
|
||||
// StartDesktopRecording mocks base method.
|
||||
func (m *MockAgentConn) StartDesktopRecording(ctx context.Context, req workspacesdk.StartDesktopRecordingRequest) error {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "StartDesktopRecording", ctx, req)
|
||||
ret0, _ := ret[0].(error)
|
||||
return ret0
|
||||
}
|
||||
|
||||
// StartDesktopRecording indicates an expected call of StartDesktopRecording.
|
||||
func (mr *MockAgentConnMockRecorder) StartDesktopRecording(ctx, req any) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StartDesktopRecording", reflect.TypeOf((*MockAgentConn)(nil).StartDesktopRecording), ctx, req)
|
||||
}
|
||||
|
||||
// StartProcess mocks base method.
|
||||
func (m *MockAgentConn) StartProcess(ctx context.Context, req workspacesdk.StartProcessRequest) (workspacesdk.StartProcessResponse, error) {
|
||||
m.ctrl.T.Helper()
|
||||
@@ -565,6 +579,21 @@ func (mr *MockAgentConnMockRecorder) StartProcess(ctx, req any) *gomock.Call {
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StartProcess", reflect.TypeOf((*MockAgentConn)(nil).StartProcess), ctx, req)
|
||||
}
|
||||
|
||||
// StopDesktopRecording mocks base method.
|
||||
func (m *MockAgentConn) StopDesktopRecording(ctx context.Context, req workspacesdk.StopDesktopRecordingRequest) (io.ReadCloser, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "StopDesktopRecording", ctx, req)
|
||||
ret0, _ := ret[0].(io.ReadCloser)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
|
||||
// StopDesktopRecording indicates an expected call of StopDesktopRecording.
|
||||
func (mr *MockAgentConnMockRecorder) StopDesktopRecording(ctx, req any) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StopDesktopRecording", reflect.TypeOf((*MockAgentConn)(nil).StopDesktopRecording), ctx, req)
|
||||
}
|
||||
|
||||
// TailnetConn mocks base method.
|
||||
func (m *MockAgentConn) TailnetConn() *tailnet.Conn {
|
||||
m.ctrl.T.Helper()
|
||||
|
||||
Reference in New Issue
Block a user