mirror of
https://github.com/coder/coder.git
synced 2026-09-24 15:04:27 +08:00
feat: agents desktop recording thumbnail backend (#24022)
The agents chat interface displays thumbnails for videos recorded by the computer use agent. Currently, to display a thumbnail, the frontend downloads the entire video and shows the first frame. This PR starts storing a new thumbnail file in the database for every recorded video, and exposes the file id in the `wait_agent` tool result alongside the recording file id, so the frontend can fetch just the thumbnail.
This commit is contained in:
@@ -95,7 +95,7 @@ type AgentConn interface {
|
||||
ConnectDesktopVNC(ctx context.Context) (net.Conn, error)
|
||||
ExecuteDesktopAction(ctx context.Context, action DesktopAction) (DesktopActionResponse, error)
|
||||
StartDesktopRecording(ctx context.Context, req StartDesktopRecordingRequest) error
|
||||
StopDesktopRecording(ctx context.Context, req StopDesktopRecordingRequest) (io.ReadCloser, error)
|
||||
StopDesktopRecording(ctx context.Context, req StopDesktopRecordingRequest) (StopDesktopRecordingResponse, error)
|
||||
}
|
||||
|
||||
// AgentConn represents a connection to a workspace agent.
|
||||
@@ -610,11 +610,25 @@ type StopDesktopRecordingRequest struct {
|
||||
RecordingID string `json:"recording_id"`
|
||||
}
|
||||
|
||||
// StopDesktopRecordingResponse wraps the response from stopping a
|
||||
// desktop recording. Body contains the recording data as a
|
||||
// multipart/mixed stream. ContentType holds the Content-Type
|
||||
// header (including boundary) so callers can parse the body.
|
||||
type StopDesktopRecordingResponse struct {
|
||||
Body io.ReadCloser
|
||||
ContentType string
|
||||
}
|
||||
|
||||
// MaxRecordingSize is the largest desktop recording (in bytes)
|
||||
// that will be accepted. Used by both the agent-side stop handler
|
||||
// and the server-side storage pipeline.
|
||||
const MaxRecordingSize = 100 << 20 // 100 MB
|
||||
|
||||
// MaxThumbnailSize is the largest thumbnail (in bytes) that will
|
||||
// be accepted. Applied both agent-side (before streaming) and
|
||||
// server-side (when parsing multipart parts).
|
||||
const MaxThumbnailSize = 10 << 20 // 10 MB
|
||||
|
||||
// ExecuteDesktopAction executes a mouse/keyboard/scroll action on the
|
||||
// agent's desktop.
|
||||
func (c *agentConn) ExecuteDesktopAction(ctx context.Context, action DesktopAction) (DesktopActionResponse, error) {
|
||||
@@ -681,22 +695,27 @@ func (c *agentConn) StartDesktopRecording(ctx context.Context, req StartDesktopR
|
||||
}
|
||||
|
||||
// StopDesktopRecording stops a desktop recording session on the
|
||||
// agent and returns the MP4 data as an io.ReadCloser. The caller
|
||||
// is responsible for closing the returned reader. Idempotent —
|
||||
// safe to call on an already-stopped recording.
|
||||
func (c *agentConn) StopDesktopRecording(ctx context.Context, req StopDesktopRecordingRequest) (io.ReadCloser, error) {
|
||||
// agent and returns the recording as a StopDesktopRecordingResponse.
|
||||
// The response body is a multipart/mixed stream containing the
|
||||
// video (and optionally a JPEG thumbnail). The caller is
|
||||
// responsible for closing the returned Body. Idempotent — safe
|
||||
// to call on an already-stopped recording.
|
||||
func (c *agentConn) StopDesktopRecording(ctx context.Context, req StopDesktopRecordingRequest) (StopDesktopRecordingResponse, error) {
|
||||
ctx, span := tracing.StartSpan(ctx)
|
||||
defer span.End()
|
||||
res, err := c.apiRequest(ctx, http.MethodPost, "/api/v0/desktop/recording/stop", req)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("stop recording request: %w", err)
|
||||
return StopDesktopRecordingResponse{}, xerrors.Errorf("stop recording request: %w", err)
|
||||
}
|
||||
if res.StatusCode != http.StatusOK {
|
||||
defer res.Body.Close()
|
||||
return nil, codersdk.ReadBodyAsError(res)
|
||||
return StopDesktopRecordingResponse{}, codersdk.ReadBodyAsError(res)
|
||||
}
|
||||
// Caller is responsible for closing res.Body.
|
||||
return res.Body, nil
|
||||
return StopDesktopRecordingResponse{
|
||||
Body: res.Body,
|
||||
ContentType: res.Header.Get("Content-Type"),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// DeleteDevcontainer deletes the provided devcontainer.
|
||||
|
||||
@@ -580,10 +580,10 @@ func (mr *MockAgentConnMockRecorder) StartProcess(ctx, req any) *gomock.Call {
|
||||
}
|
||||
|
||||
// StopDesktopRecording mocks base method.
|
||||
func (m *MockAgentConn) StopDesktopRecording(ctx context.Context, req workspacesdk.StopDesktopRecordingRequest) (io.ReadCloser, error) {
|
||||
func (m *MockAgentConn) StopDesktopRecording(ctx context.Context, req workspacesdk.StopDesktopRecordingRequest) (workspacesdk.StopDesktopRecordingResponse, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "StopDesktopRecording", ctx, req)
|
||||
ret0, _ := ret[0].(io.ReadCloser)
|
||||
ret0, _ := ret[0].(workspacesdk.StopDesktopRecordingResponse)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user