feat: agents desktop recording thumbnail backend (#24022)

The agents chat interface displays thumbnails for videos recorded by the
computer use agent. Currently, to display a thumbnail, the frontend
downloads the entire video and shows the first frame. This PR starts
storing a new thumbnail file in the database for every recorded video,
and exposes the file id in the `wait_agent` tool result alongside the
recording file id, so the frontend can fetch just the thumbnail.
This commit is contained in:
Hugo Dutka
2026-04-09 13:47:54 +02:00
committed by GitHub
parent 2c499484b7
commit efb19eb748
11 changed files with 1072 additions and 126 deletions
+27 -8
View File
@@ -95,7 +95,7 @@ type AgentConn interface {
ConnectDesktopVNC(ctx context.Context) (net.Conn, error)
ExecuteDesktopAction(ctx context.Context, action DesktopAction) (DesktopActionResponse, error)
StartDesktopRecording(ctx context.Context, req StartDesktopRecordingRequest) error
StopDesktopRecording(ctx context.Context, req StopDesktopRecordingRequest) (io.ReadCloser, error)
StopDesktopRecording(ctx context.Context, req StopDesktopRecordingRequest) (StopDesktopRecordingResponse, error)
}
// AgentConn represents a connection to a workspace agent.
@@ -610,11 +610,25 @@ type StopDesktopRecordingRequest struct {
RecordingID string `json:"recording_id"`
}
// StopDesktopRecordingResponse wraps the response from stopping a
// desktop recording. Body contains the recording data as a
// multipart/mixed stream. ContentType holds the Content-Type
// header (including boundary) so callers can parse the body.
type StopDesktopRecordingResponse struct {
Body io.ReadCloser
ContentType string
}
// MaxRecordingSize is the largest desktop recording (in bytes)
// that will be accepted. Used by both the agent-side stop handler
// and the server-side storage pipeline.
const MaxRecordingSize = 100 << 20 // 100 MB
// MaxThumbnailSize is the largest thumbnail (in bytes) that will
// be accepted. Applied both agent-side (before streaming) and
// server-side (when parsing multipart parts).
const MaxThumbnailSize = 10 << 20 // 10 MB
// ExecuteDesktopAction executes a mouse/keyboard/scroll action on the
// agent's desktop.
func (c *agentConn) ExecuteDesktopAction(ctx context.Context, action DesktopAction) (DesktopActionResponse, error) {
@@ -681,22 +695,27 @@ func (c *agentConn) StartDesktopRecording(ctx context.Context, req StartDesktopR
}
// StopDesktopRecording stops a desktop recording session on the
// agent and returns the MP4 data as an io.ReadCloser. The caller
// is responsible for closing the returned reader. Idempotent —
// safe to call on an already-stopped recording.
func (c *agentConn) StopDesktopRecording(ctx context.Context, req StopDesktopRecordingRequest) (io.ReadCloser, error) {
// agent and returns the recording as a StopDesktopRecordingResponse.
// The response body is a multipart/mixed stream containing the
// video (and optionally a JPEG thumbnail). The caller is
// responsible for closing the returned Body. Idempotent — safe
// to call on an already-stopped recording.
func (c *agentConn) StopDesktopRecording(ctx context.Context, req StopDesktopRecordingRequest) (StopDesktopRecordingResponse, error) {
ctx, span := tracing.StartSpan(ctx)
defer span.End()
res, err := c.apiRequest(ctx, http.MethodPost, "/api/v0/desktop/recording/stop", req)
if err != nil {
return nil, xerrors.Errorf("stop recording request: %w", err)
return StopDesktopRecordingResponse{}, xerrors.Errorf("stop recording request: %w", err)
}
if res.StatusCode != http.StatusOK {
defer res.Body.Close()
return nil, codersdk.ReadBodyAsError(res)
return StopDesktopRecordingResponse{}, codersdk.ReadBodyAsError(res)
}
// Caller is responsible for closing res.Body.
return res.Body, nil
return StopDesktopRecordingResponse{
Body: res.Body,
ContentType: res.Header.Get("Content-Type"),
}, nil
}
// DeleteDevcontainer deletes the provided devcontainer.
@@ -580,10 +580,10 @@ func (mr *MockAgentConnMockRecorder) StartProcess(ctx, req any) *gomock.Call {
}
// StopDesktopRecording mocks base method.
func (m *MockAgentConn) StopDesktopRecording(ctx context.Context, req workspacesdk.StopDesktopRecordingRequest) (io.ReadCloser, error) {
func (m *MockAgentConn) StopDesktopRecording(ctx context.Context, req workspacesdk.StopDesktopRecordingRequest) (workspacesdk.StopDesktopRecordingResponse, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "StopDesktopRecording", ctx, req)
ret0, _ := ret[0].(io.ReadCloser)
ret0, _ := ret[0].(workspacesdk.StopDesktopRecordingResponse)
ret1, _ := ret[1].(error)
return ret0, ret1
}