opensource/coder
3
0
Fork 0
mirror of https://github.com/coder/coder.git synced 2026-09-24 15:04:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: chat desktop backend (#23005)

Browse Source 
Implement the backend for the desktop feature for agents.

- Adds a new `/api/experimental/chats/$id/desktop` endpoint to coderd
which exposes a VNC stream from a
[portabledesktop](https://github.com/coder/portabledesktop) process
running inside the workspace
- Adds a new `spawn_computer_use_agent` tool to chatd, which spawns a
subagent that has access to the `computer` tool which lets it interact
with the `portabledesktop` process running inside the workspace
- Adds the plumbing to make the above possible

There's a follow up frontend PR here:
https://github.com/coder/coder/pull/23006
This commit is contained in:
Hugo Dutka
2026-03-13 19:49:34 +01:00
committed by GitHub
parent 67f5494665
commit 84527390c6
34 changed files with 4465 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files
coderd/database/dump.sql
Generated
+6 -1
View File
@@ -278,6 +278,10 @@ CREATE TYPE chat_message_visibility AS ENUM (
'both'
);
CREATE TYPE chat_mode AS ENUM (
'computer_use'
);
CREATE TYPE chat_status AS ENUM (
'waiting',
'pending',
@@ -1306,7 +1310,8 @@ CREATE TABLE chats (
root_chat_id uuid,
last_model_config_id uuid NOT NULL,
archived boolean DEFAULT false NOT NULL,
last_error text
last_error text,
mode chat_mode
);
CREATE TABLE connection_logs (
coderd/database/migrations/000436_add_chat_mode.down.sql
+2
View File
@@ -0,0 +1,2 @@
ALTER TABLE chats DROP COLUMN mode;
DROP TYPE IF EXISTS chat_mode;
coderd/database/migrations/000436_add_chat_mode.up.sql
+3
View File
@@ -0,0 +1,3 @@
CREATE TYPE chat_mode AS ENUM ('computer_use');
ALTER TABLE chats ADD COLUMN mode chat_mode;
coderd/database/models.go
+56
View File
@@ -1174,6 +1174,61 @@ func AllChatMessageVisibilityValues() []ChatMessageVisibility {
}
}
type ChatMode string
const (
ChatModeComputerUse ChatMode = "computer_use"
)
func (e *ChatMode) Scan(src interface{}) error {
switch s := src.(type) {
case []byte:
*e = ChatMode(s)
case string:
*e = ChatMode(s)
default:
return fmt.Errorf("unsupported scan type for ChatMode: %T", src)
}
return nil
}
type NullChatMode struct {
ChatMode ChatMode `json:"chat_mode"`
Valid bool `json:"valid"` // Valid is true if ChatMode is not NULL
}
// Scan implements the Scanner interface.
func (ns *NullChatMode) Scan(value interface{}) error {
if value == nil {
ns.ChatMode, ns.Valid = "", false
return nil
}
ns.Valid = true
return ns.ChatMode.Scan(value)
}
// Value implements the driver Valuer interface.
func (ns NullChatMode) Value() (driver.Value, error) {
if !ns.Valid {
return nil, nil
}
return string(ns.ChatMode), nil
}
func (e ChatMode) Valid() bool {
switch e {
case ChatModeComputerUse:
return true
}
return false
}
func AllChatModeValues() []ChatMode {
return []ChatMode{
ChatModeComputerUse,
}
}
type ChatStatus string
const (
@@ -3972,6 +4027,7 @@ type Chat struct {
LastModelConfigID uuid.UUID `db:"last_model_config_id" json:"last_model_config_id"`
Archived bool `db:"archived" json:"archived"`
LastError sql.NullString `db:"last_error" json:"last_error"`
Mode NullChatMode `db:"mode" json:"mode"`
}
type ChatDiffStatus struct {
coderd/database/queries.sql.go
+24 -11
View File
@@ -2949,7 +2949,7 @@ WHERE
$3::int
)
RETURNING
id, owner_id, workspace_id, title, status, worker_id, started_at, heartbeat_at, created_at, updated_at, parent_chat_id, root_chat_id, last_model_config_id, archived, last_error
id, owner_id, workspace_id, title, status, worker_id, started_at, heartbeat_at, created_at, updated_at, parent_chat_id, root_chat_id, last_model_config_id, archived, last_error, mode
`
type AcquireChatsParams struct {
@@ -2985,6 +2985,7 @@ func (q *sqlQuerier) AcquireChats(ctx context.Context, arg AcquireChatsParams) (
&i.LastModelConfigID,
&i.Archived,
&i.LastError,
&i.Mode,
); err != nil {
return nil, err
}
@@ -3172,7 +3173,7 @@ func (q *sqlQuerier) DeleteChatQueuedMessage(ctx context.Context, arg DeleteChat
const getChatByID = `-- name: GetChatByID :one
SELECT
id, owner_id, workspace_id, title, status, worker_id, started_at, heartbeat_at, created_at, updated_at, parent_chat_id, root_chat_id, last_model_config_id, archived, last_error
id, owner_id, workspace_id, title, status, worker_id, started_at, heartbeat_at, created_at, updated_at, parent_chat_id, root_chat_id, last_model_config_id, archived, last_error, mode
FROM
chats
WHERE
@@ -3198,12 +3199,13 @@ func (q *sqlQuerier) GetChatByID(ctx context.Context, id uuid.UUID) (Chat, error
&i.LastModelConfigID,
&i.Archived,
&i.LastError,
&i.Mode,
)
return i, err
}
const getChatByIDForUpdate = `-- name: GetChatByIDForUpdate :one
SELECT id, owner_id, workspace_id, title, status, worker_id, started_at, heartbeat_at, created_at, updated_at, parent_chat_id, root_chat_id, last_model_config_id, archived, last_error FROM chats WHERE id = $1::uuid FOR UPDATE
SELECT id, owner_id, workspace_id, title, status, worker_id, started_at, heartbeat_at, created_at, updated_at, parent_chat_id, root_chat_id, last_model_config_id, archived, last_error, mode FROM chats WHERE id = $1::uuid FOR UPDATE
`
func (q *sqlQuerier) GetChatByIDForUpdate(ctx context.Context, id uuid.UUID) (Chat, error) {
@@ -3225,6 +3227,7 @@ func (q *sqlQuerier) GetChatByIDForUpdate(ctx context.Context, id uuid.UUID) (Ch
&i.LastModelConfigID,
&i.Archived,
&i.LastError,
&i.Mode,
)
return i, err
}
@@ -3886,7 +3889,7 @@ func (q *sqlQuerier) GetChatQueuedMessages(ctx context.Context, chatID uuid.UUID
const getChatsByOwnerID = `-- name: GetChatsByOwnerID :many
SELECT
id, owner_id, workspace_id, title, status, worker_id, started_at, heartbeat_at, created_at, updated_at, parent_chat_id, root_chat_id, last_model_config_id, archived, last_error
id, owner_id, workspace_id, title, status, worker_id, started_at, heartbeat_at, created_at, updated_at, parent_chat_id, root_chat_id, last_model_config_id, archived, last_error, mode
FROM
chats
WHERE
@@ -3963,6 +3966,7 @@ func (q *sqlQuerier) GetChatsByOwnerID(ctx context.Context, arg GetChatsByOwnerI
&i.LastModelConfigID,
&i.Archived,
&i.LastError,
&i.Mode,
); err != nil {
return nil, err
}
@@ -4024,7 +4028,7 @@ func (q *sqlQuerier) GetLastChatMessageByRole(ctx context.Context, arg GetLastCh
const getStaleChats = `-- name: GetStaleChats :many
SELECT
id, owner_id, workspace_id, title, status, worker_id, started_at, heartbeat_at, created_at, updated_at, parent_chat_id, root_chat_id, last_model_config_id, archived, last_error
id, owner_id, workspace_id, title, status, worker_id, started_at, heartbeat_at, created_at, updated_at, parent_chat_id, root_chat_id, last_model_config_id, archived, last_error, mode
FROM
chats
WHERE
@@ -4059,6 +4063,7 @@ func (q *sqlQuerier) GetStaleChats(ctx context.Context, staleThreshold time.Time
&i.LastModelConfigID,
&i.Archived,
&i.LastError,
&i.Mode,
); err != nil {
return nil, err
}
@@ -4080,17 +4085,19 @@ INSERT INTO chats (
parent_chat_id,
root_chat_id,
last_model_config_id,
title
title,
mode
) VALUES (
$1::uuid,
$2::uuid,
$3::uuid,
$4::uuid,
$5::uuid,
$6::text
$6::text,
$7::chat_mode
)
RETURNING
id, owner_id, workspace_id, title, status, worker_id, started_at, heartbeat_at, created_at, updated_at, parent_chat_id, root_chat_id, last_model_config_id, archived, last_error
id, owner_id, workspace_id, title, status, worker_id, started_at, heartbeat_at, created_at, updated_at, parent_chat_id, root_chat_id, last_model_config_id, archived, last_error, mode
`
type InsertChatParams struct {
@@ -4100,6 +4107,7 @@ type InsertChatParams struct {
RootChatID uuid.NullUUID `db:"root_chat_id" json:"root_chat_id"`
LastModelConfigID uuid.UUID `db:"last_model_config_id" json:"last_model_config_id"`
Title string `db:"title" json:"title"`
Mode NullChatMode `db:"mode" json:"mode"`
}
func (q *sqlQuerier) InsertChat(ctx context.Context, arg InsertChatParams) (Chat, error) {
@@ -4110,6 +4118,7 @@ func (q *sqlQuerier) InsertChat(ctx context.Context, arg InsertChatParams) (Chat
arg.RootChatID,
arg.LastModelConfigID,
arg.Title,
arg.Mode,
)
var i Chat
err := row.Scan(
@@ -4128,6 +4137,7 @@ func (q *sqlQuerier) InsertChat(ctx context.Context, arg InsertChatParams) (Chat
&i.LastModelConfigID,
&i.Archived,
&i.LastError,
&i.Mode,
)
return i, err
}
@@ -4307,7 +4317,7 @@ SET
WHERE
id = $2::uuid
RETURNING
id, owner_id, workspace_id, title, status, worker_id, started_at, heartbeat_at, created_at, updated_at, parent_chat_id, root_chat_id, last_model_config_id, archived, last_error
id, owner_id, workspace_id, title, status, worker_id, started_at, heartbeat_at, created_at, updated_at, parent_chat_id, root_chat_id, last_model_config_id, archived, last_error, mode
`
type UpdateChatByIDParams struct {
@@ -4334,6 +4344,7 @@ func (q *sqlQuerier) UpdateChatByID(ctx context.Context, arg UpdateChatByIDParam
&i.LastModelConfigID,
&i.Archived,
&i.LastError,
&i.Mode,
)
return i, err
}
@@ -4421,7 +4432,7 @@ SET
WHERE
id = $6::uuid
RETURNING
id, owner_id, workspace_id, title, status, worker_id, started_at, heartbeat_at, created_at, updated_at, parent_chat_id, root_chat_id, last_model_config_id, archived, last_error
id, owner_id, workspace_id, title, status, worker_id, started_at, heartbeat_at, created_at, updated_at, parent_chat_id, root_chat_id, last_model_config_id, archived, last_error, mode
`
type UpdateChatStatusParams struct {
@@ -4459,6 +4470,7 @@ func (q *sqlQuerier) UpdateChatStatus(ctx context.Context, arg UpdateChatStatusP
&i.LastModelConfigID,
&i.Archived,
&i.LastError,
&i.Mode,
)
return i, err
}
@@ -4472,7 +4484,7 @@ SET
WHERE
id = $2::uuid
RETURNING
id, owner_id, workspace_id, title, status, worker_id, started_at, heartbeat_at, created_at, updated_at, parent_chat_id, root_chat_id, last_model_config_id, archived, last_error
id, owner_id, workspace_id, title, status, worker_id, started_at, heartbeat_at, created_at, updated_at, parent_chat_id, root_chat_id, last_model_config_id, archived, last_error, mode
`
type UpdateChatWorkspaceParams struct {
@@ -4499,6 +4511,7 @@ func (q *sqlQuerier) UpdateChatWorkspace(ctx context.Context, arg UpdateChatWork
&i.LastModelConfigID,
&i.Archived,
&i.LastError,
&i.Mode,
)
return i, err
}
coderd/database/queries/chats.sql
+4 -2
View File
@@ -142,14 +142,16 @@ INSERT INTO chats (
parent_chat_id,
root_chat_id,
last_model_config_id,
title
title,
mode
) VALUES (
@owner_id::uuid,
sqlc.narg('workspace_id')::uuid,
sqlc.narg('parent_chat_id')::uuid,
sqlc.narg('root_chat_id')::uuid,
@last_model_config_id::uuid,
@title::text
@title::text,
sqlc.narg('mode')::chat_mode
)
RETURNING
*;