feat: add computer-use provider selection for AI agents (#24772)

Adds a deployment-wide setting to select the computer-use provider
(Anthropic or OpenAI) for AI agents, plus the OpenAI computer-use runner
needed to honor that selection.

The setting is stored in `site_configs` under
`agents_computer_use_provider`, defaults to Anthropic when unset, and is
exposed via experimental GET/PUT endpoints under
`/api/experimental/chats/config/computer-use-provider`. The chatd
computer-use tool now dispatches to either `runAnthropicComputerUse` or
`runOpenAIComputerUse` based on the resolved provider, with
provider-specific result metadata for OpenAI screenshots.

Frontend adds a provider dropdown to the Agents Experiments settings
page nested under the virtual desktop toggle, with disabled state
handling while virtual desktop is off and skeleton loaders while config
queries are in flight.

Hugo and Codex review follow-up:
- Uses shared provider validation and clearer computer-use constant
names.
- Removes stale OpenAI pending-safety-checks commentary.
- Documents why provider result metadata is needed for OpenAI
screenshots.
- Keeps the computer-use subagent visible when provider credentials are
missing, then returns a clear spawn-time configuration error.
- Uses OpenAI's recommended 1600x900 screenshot geometry to preserve the
native 16:9 aspect ratio.
- Moves OpenAI-specific computer-use helpers into
`coderd/x/chatd/chatopenai/computeruse` after rebasing onto the provider
package refactor in `main`.
- Converts OpenAI pixel scroll deltas to Coder desktop wheel-click
amounts.
- Preserves OpenAI pointer modifiers with key down/up desktop actions
and rejects unsupported non-left double-click buttons explicitly.
- Maps OpenAI back/forward side-button clicks to browser navigation key
actions.
- Defaults omitted OpenAI click buttons to left-click.
- Retries mouse release cleanup if the final OpenAI drag release fails.
- Keeps computer-use subagent availability messages stable when provider
config cannot be loaded, while logging the backend error.
- Releases remaining OpenAI modifier keys if a synthetic key-up cleanup
action fails.
- Updates Storybook interaction stories so provider snapshots show the
selected final provider.

> Mux updated this PR description on behalf of Mike.
This commit is contained in:
Michael Suchacz
2026-05-04 20:30:50 +02:00
committed by GitHub
parent da6e708bd2
commit 0bb09935bc
38 changed files with 3181 additions and 141 deletions
+18
View File
@@ -3360,6 +3360,24 @@ class ExperimentalApiMethods {
await this.axios.put("/api/experimental/chats/config/advisor", req);
};
getChatComputerUseProvider =
async (): Promise<TypesGen.ChatComputerUseProviderResponse> => {
const response =
await this.axios.get<TypesGen.ChatComputerUseProviderResponse>(
"/api/experimental/chats/config/computer-use-provider",
);
return response.data;
};
updateChatComputerUseProvider = async (
req: TypesGen.UpdateChatComputerUseProviderRequest,
): Promise<void> => {
await this.axios.put(
"/api/experimental/chats/config/computer-use-provider",
req,
);
};
getChatWorkspaceTTL =
async (): Promise<TypesGen.ChatWorkspaceTTLResponse> => {
const response = await this.axios.get<TypesGen.ChatWorkspaceTTLResponse>(
+16
View File
@@ -1348,6 +1348,22 @@ export const updateChatAdvisorConfig = (queryClient: QueryClient) => ({
},
});
const chatComputerUseProviderKey = ["chat-computer-use-provider"] as const;
export const chatComputerUseProvider = () => ({
queryKey: chatComputerUseProviderKey,
queryFn: () => API.experimental.getChatComputerUseProvider(),
});
export const updateChatComputerUseProvider = (queryClient: QueryClient) => ({
mutationFn: API.experimental.updateChatComputerUseProvider,
onSuccess: async () => {
await queryClient.invalidateQueries({
queryKey: chatComputerUseProviderKey,
});
},
});
const chatWorkspaceTTLKey = ["chat-workspace-ttl"] as const;
export const chatWorkspaceTTL = () => ({
+18
View File
@@ -1346,6 +1346,15 @@ export const ChatClientTypes: ChatClientType[] = ["api", "ui"];
export const ChatCompactionThresholdKeyPrefix =
"chat_compaction_threshold_pct:";
// From codersdk/chats.go
/**
* ChatComputerUseProviderResponse is the response for getting the computer use
* provider setting.
*/
export interface ChatComputerUseProviderResponse {
readonly provider: string;
}
// From codersdk/deployment.go
export interface ChatConfig {
readonly acquire_batch_size: number;
@@ -7817,6 +7826,15 @@ export interface UpdateChatAutoArchiveDaysRequest {
readonly auto_archive_days: number;
}
// From codersdk/chats.go
/**
* UpdateChatComputerUseProviderRequest is the request to update the computer use
* provider setting.
*/
export interface UpdateChatComputerUseProviderRequest {
readonly provider: string;
}
// From codersdk/chats.go
/**
* UpdateChatDebugLoggingAllowUsersRequest is the admin request to
@@ -2,10 +2,12 @@ import type { FC } from "react";
import { useMutation, useQuery, useQueryClient } from "react-query";
import {
chatAdvisorConfig,
chatComputerUseProvider,
chatDebugLogging,
chatDesktopEnabled,
chatModelConfigs,
updateChatAdvisorConfig,
updateChatComputerUseProvider,
updateChatDebugLogging,
updateChatDesktopEnabled,
} from "#/api/queries/chats";
@@ -20,6 +22,10 @@ const AgentSettingsExperimentsPage: FC = () => {
...chatDesktopEnabled(),
enabled: permissions.editDeploymentConfig,
});
const computerUseProviderQuery = useQuery({
...chatComputerUseProvider(),
enabled: permissions.editDeploymentConfig,
});
const debugLoggingQuery = useQuery({
...chatDebugLogging(),
enabled: permissions.editDeploymentConfig,
@@ -35,6 +41,9 @@ const AgentSettingsExperimentsPage: FC = () => {
const saveDesktopEnabledMutation = useMutation(
updateChatDesktopEnabled(queryClient),
);
const saveComputerUseProviderMutation = useMutation(
updateChatComputerUseProvider(queryClient),
);
const saveDebugLoggingMutation = useMutation(
updateChatDebugLogging(queryClient),
);
@@ -46,10 +55,17 @@ const AgentSettingsExperimentsPage: FC = () => {
<RequirePermission isFeatureVisible={permissions.editDeploymentConfig}>
<AgentSettingsExperimentsPageView
desktopEnabledData={desktopEnabledQuery.data}
isLoadingDesktopEnabled={desktopEnabledQuery.isLoading}
onSaveDesktopEnabled={saveDesktopEnabledMutation.mutate}
isSavingDesktopEnabled={saveDesktopEnabledMutation.isPending}
isSaveDesktopEnabledError={saveDesktopEnabledMutation.isError}
computerUseProviderData={computerUseProviderQuery.data}
isLoadingComputerUseProvider={computerUseProviderQuery.isLoading}
onSaveComputerUseProvider={saveComputerUseProviderMutation.mutate}
isSavingComputerUseProvider={saveComputerUseProviderMutation.isPending}
computerUseProviderSaveError={saveComputerUseProviderMutation.error}
debugLoggingData={debugLoggingQuery.data}
isLoadingDebugLogging={debugLoggingQuery.isLoading}
onSaveDebugLogging={saveDebugLoggingMutation.mutate}
isSavingDebugLogging={saveDebugLoggingMutation.isPending}
isSaveDebugLoggingError={saveDebugLoggingMutation.isError}
@@ -1,4 +1,5 @@
import type { Meta, StoryObj } from "@storybook/react-vite";
import { useState } from "react";
import { expect, fn, userEvent, waitFor, within } from "storybook/test";
import {
AgentSettingsExperimentsPageView,
@@ -7,13 +8,20 @@ import {
const baseArgs: AgentSettingsExperimentsPageViewProps = {
desktopEnabledData: { enable_desktop: false },
isLoadingDesktopEnabled: false,
onSaveDesktopEnabled: fn(),
isSavingDesktopEnabled: false,
isSaveDesktopEnabledError: false,
computerUseProviderData: { provider: "anthropic" },
isLoadingComputerUseProvider: false,
onSaveComputerUseProvider: fn(),
isSavingComputerUseProvider: false,
computerUseProviderSaveError: null,
debugLoggingData: {
allow_users: false,
forced_by_deployment: false,
},
isLoadingDebugLogging: false,
onSaveDebugLogging: fn(),
isSavingDebugLogging: false,
isSaveDebugLoggingError: false,
@@ -46,6 +54,50 @@ const meta = {
export default meta;
type Story = StoryObj<typeof AgentSettingsExperimentsPageView>;
const getComputerUseProviderSelect = async (canvasElement: HTMLElement) => {
const canvas = within(canvasElement);
return canvas.findByRole("combobox", {
name: "Computer use provider",
});
};
const selectComputerUseProvider = async (
canvasElement: HTMLElement,
currentSelectionName: string,
optionName: string,
) => {
const trigger = await getComputerUseProviderSelect(canvasElement);
expect(trigger).toHaveTextContent(currentSelectionName);
await userEvent.click(trigger);
const body = within(canvasElement.ownerDocument.body);
await userEvent.click(await body.findByRole("option", { name: optionName }));
await waitFor(() => expect(trigger).toHaveTextContent(optionName));
};
function InteractiveComputerUseProviderStory(
args: AgentSettingsExperimentsPageViewProps,
) {
const [computerUseProviderData, setComputerUseProviderData] = useState(
args.computerUseProviderData,
);
return (
<AgentSettingsExperimentsPageView
{...args}
computerUseProviderData={computerUseProviderData}
onSaveComputerUseProvider={(request, options) => {
if (options) {
args.onSaveComputerUseProvider(request, options);
} else {
args.onSaveComputerUseProvider(request);
}
setComputerUseProviderData({ provider: request.provider });
}}
/>
);
}
export const AllowUsersOff: Story = {
play: async ({ canvasElement }) => {
const canvas = within(canvasElement);
@@ -110,6 +162,26 @@ export const DesktopSetting: Story = {
},
};
export const VirtualDesktopLoading: Story = {
args: {
desktopEnabledData: undefined,
isLoadingDesktopEnabled: true,
computerUseProviderData: undefined,
isLoadingComputerUseProvider: true,
},
play: async ({ canvasElement }) => {
const canvas = within(canvasElement);
// While loading, the Switch is replaced by a skeleton placeholder.
expect(
canvas.queryByRole("switch", { name: "Enable" }),
).not.toBeInTheDocument();
const providerSelect = await getComputerUseProviderSelect(canvasElement);
expect(providerSelect).toBeDisabled();
},
};
export const TogglesDesktop: Story = {
play: async ({ canvasElement, args }) => {
const canvas = within(canvasElement);
@@ -123,3 +195,84 @@ export const TogglesDesktop: Story = {
});
},
};
export const ComputerUseProviderAnthropic: Story = {
args: {
desktopEnabledData: { enable_desktop: true },
},
play: async ({ canvasElement }) => {
const canvas = within(canvasElement);
await canvas.findByText("Computer use provider");
const providerSelect = await getComputerUseProviderSelect(canvasElement);
expect(providerSelect).not.toBeDisabled();
expect(providerSelect).toHaveTextContent("Anthropic");
},
};
export const ComputerUseProviderDisabledWhenDesktopDisabled: Story = {
play: async ({ canvasElement }) => {
const providerSelect = await getComputerUseProviderSelect(canvasElement);
expect(providerSelect).toBeDisabled();
},
};
export const SelectsOpenAIProvider: Story = {
args: {
desktopEnabledData: { enable_desktop: true },
onSaveComputerUseProvider: fn(),
},
render: InteractiveComputerUseProviderStory,
play: async ({ canvasElement, args }) => {
await selectComputerUseProvider(canvasElement, "Anthropic", "OpenAI");
await waitFor(() => {
expect(args.onSaveComputerUseProvider).toHaveBeenCalledWith({
provider: "openai",
});
});
},
};
export const SelectsAnthropicProvider: Story = {
args: {
desktopEnabledData: { enable_desktop: true },
computerUseProviderData: { provider: "openai" },
onSaveComputerUseProvider: fn(),
},
render: InteractiveComputerUseProviderStory,
play: async ({ canvasElement, args }) => {
await selectComputerUseProvider(canvasElement, "OpenAI", "Anthropic");
await waitFor(() => {
expect(args.onSaveComputerUseProvider).toHaveBeenCalledWith({
provider: "anthropic",
});
});
},
};
export const ComputerUseProviderSaveError: Story = {
args: {
computerUseProviderSaveError: new Error("Failed to save."),
},
play: async ({ canvasElement }) => {
const canvas = within(canvasElement);
expect(
await canvas.findByText("Failed to save computer use provider."),
).toBeInTheDocument();
},
};
export const ComputerUseProviderSaving: Story = {
args: {
desktopEnabledData: { enable_desktop: true },
isSavingComputerUseProvider: true,
},
play: async ({ canvasElement }) => {
const providerSelect = await getComputerUseProviderSelect(canvasElement);
expect(providerSelect).toBeDisabled();
},
};
@@ -13,6 +13,7 @@ interface MutationCallbacks {
export interface AgentSettingsExperimentsPageViewProps {
desktopEnabledData: TypesGen.ChatDesktopEnabledResponse | undefined;
isLoadingDesktopEnabled: boolean;
onSaveDesktopEnabled: UseMutateFunction<
void,
Error,
@@ -21,7 +22,18 @@ export interface AgentSettingsExperimentsPageViewProps {
>;
isSavingDesktopEnabled: boolean;
isSaveDesktopEnabledError: boolean;
computerUseProviderData: TypesGen.ChatComputerUseProviderResponse | undefined;
isLoadingComputerUseProvider: boolean;
onSaveComputerUseProvider: UseMutateFunction<
void,
Error,
TypesGen.UpdateChatComputerUseProviderRequest,
unknown
>;
isSavingComputerUseProvider: boolean;
computerUseProviderSaveError: Error | null;
debugLoggingData: TypesGen.ChatDebugLoggingAdminSettings | undefined;
isLoadingDebugLogging: boolean;
onSaveDebugLogging: UseMutateFunction<
void,
Error,
@@ -51,10 +63,17 @@ export const AgentSettingsExperimentsPageView: FC<
AgentSettingsExperimentsPageViewProps
> = ({
desktopEnabledData,
isLoadingDesktopEnabled,
onSaveDesktopEnabled,
isSavingDesktopEnabled,
isSaveDesktopEnabledError,
computerUseProviderData,
isLoadingComputerUseProvider,
onSaveComputerUseProvider,
isSavingComputerUseProvider,
computerUseProviderSaveError,
debugLoggingData,
isLoadingDebugLogging,
onSaveDebugLogging,
isSavingDebugLogging,
isSaveDebugLoggingError,
@@ -79,9 +98,15 @@ export const AgentSettingsExperimentsPageView: FC<
/>
<VirtualDesktopSettings
desktopEnabledData={desktopEnabledData}
isLoadingDesktopEnabled={isLoadingDesktopEnabled}
onSaveDesktopEnabled={onSaveDesktopEnabled}
isSavingDesktopEnabled={isSavingDesktopEnabled}
isSaveDesktopEnabledError={isSaveDesktopEnabledError}
computerUseProviderData={computerUseProviderData}
isLoadingComputerUseProvider={isLoadingComputerUseProvider}
onSaveComputerUseProvider={onSaveComputerUseProvider}
isSavingComputerUseProvider={isSavingComputerUseProvider}
computerUseProviderSaveError={computerUseProviderSaveError}
/>
<AdvisorSettings
advisorConfigData={advisorConfigData}
@@ -99,6 +124,7 @@ export const AgentSettingsExperimentsPageView: FC<
/>
<AdminChatDebugLoggingSettings
adminSettings={debugLoggingData}
isLoadingAdminSetting={isLoadingDebugLogging}
onSaveAdminSetting={onSaveDebugLogging}
isSavingAdminSetting={isSavingDebugLogging}
isSaveAdminSettingError={isSaveDebugLoggingError}
@@ -1,10 +1,12 @@
import type { FC } from "react";
import type { UseMutateFunction } from "react-query";
import type * as TypesGen from "#/api/typesGenerated";
import { Skeleton } from "#/components/Skeleton/Skeleton";
import { Switch } from "#/components/Switch/Switch";
interface AdminChatDebugLoggingSettingsProps {
adminSettings: TypesGen.ChatDebugLoggingAdminSettings | undefined;
isLoadingAdminSetting: boolean;
onSaveAdminSetting: UseMutateFunction<
void,
Error,
@@ -19,6 +21,7 @@ export const AdminChatDebugLoggingSettings: FC<
AdminChatDebugLoggingSettingsProps
> = ({
adminSettings,
isLoadingAdminSetting,
onSaveAdminSetting,
isSavingAdminSetting,
isSaveAdminSettingError,
@@ -48,14 +51,24 @@ export const AdminChatDebugLoggingSettings: FC<
</p>
)}
</div>
<Switch
checked={adminAllowsUsers}
onCheckedChange={(checked) =>
onSaveAdminSetting({ allow_users: checked })
}
aria-label="Allow users to enable chat debug logging"
disabled={forcedByDeployment || isSavingAdminSetting}
/>
<div className="flex items-center gap-2">
{isLoadingAdminSetting ? (
<Skeleton className="h-5 w-10 rounded-full" aria-hidden="true" />
) : (
<Switch
checked={adminAllowsUsers}
onCheckedChange={(checked) =>
onSaveAdminSetting({ allow_users: checked })
}
aria-label="Allow users to enable chat debug logging"
disabled={
forcedByDeployment ||
isSavingAdminSetting ||
isLoadingAdminSetting
}
/>
)}
</div>
</div>
{isSaveAdminSettingError && (
<p className="m-0 text-xs text-content-destructive">
@@ -3,6 +3,15 @@ import type { FC } from "react";
import type * as TypesGen from "#/api/typesGenerated";
import { Badge } from "#/components/Badge/Badge";
import { Link } from "#/components/Link/Link";
import {
Select,
SelectContent,
SelectGroup,
SelectItem,
SelectTrigger,
SelectValue,
} from "#/components/Select/Select";
import { Skeleton } from "#/components/Skeleton/Skeleton";
import { Switch } from "#/components/Switch/Switch";
interface MutationCallbacks {
@@ -12,21 +21,57 @@ interface MutationCallbacks {
interface VirtualDesktopSettingsProps {
desktopEnabledData: TypesGen.ChatDesktopEnabledResponse | undefined;
isLoadingDesktopEnabled: boolean;
onSaveDesktopEnabled: (
req: TypesGen.UpdateChatDesktopEnabledRequest,
options?: MutationCallbacks,
) => void;
isSavingDesktopEnabled: boolean;
isSaveDesktopEnabledError: boolean;
computerUseProviderData: TypesGen.ChatComputerUseProviderResponse | undefined;
isLoadingComputerUseProvider: boolean;
onSaveComputerUseProvider: (
req: TypesGen.UpdateChatComputerUseProviderRequest,
options?: MutationCallbacks,
) => void;
isSavingComputerUseProvider: boolean;
computerUseProviderSaveError: Error | null;
}
const computerUseProviderOptions = [
{ label: "Anthropic", value: "anthropic" },
{ label: "OpenAI", value: "openai" },
] as const;
const getComputerUseProviderLabel = (provider: string) => {
return (
computerUseProviderOptions.find((option) => option.value === provider)
?.label ?? provider
);
};
export const VirtualDesktopSettings: FC<VirtualDesktopSettingsProps> = ({
desktopEnabledData,
isLoadingDesktopEnabled,
onSaveDesktopEnabled,
isSavingDesktopEnabled,
isSaveDesktopEnabledError,
computerUseProviderData,
isLoadingComputerUseProvider,
onSaveComputerUseProvider,
isSavingComputerUseProvider,
computerUseProviderSaveError,
}) => {
const desktopEnabled = desktopEnabledData?.enable_desktop ?? false;
const computerUseProvider = computerUseProviderData?.provider ?? "";
const isDesktopSwitchDisabled =
isSavingDesktopEnabled || isLoadingDesktopEnabled;
const isComputerUseProviderSelectDisabled =
!desktopEnabled ||
isSavingDesktopEnabled ||
isLoadingDesktopEnabled ||
isSavingComputerUseProvider ||
isLoadingComputerUseProvider;
return (
<div className="flex flex-col gap-2">
@@ -40,14 +85,20 @@ export const VirtualDesktopSettings: FC<VirtualDesktopSettingsProps> = ({
Experimental feature
</Badge>
</div>
<Switch
checked={desktopEnabled}
onCheckedChange={(checked) =>
onSaveDesktopEnabled({ enable_desktop: checked })
}
aria-label="Enable"
disabled={isSavingDesktopEnabled}
/>
<div className="flex items-center gap-2">
{isLoadingDesktopEnabled ? (
<Skeleton className="h-5 w-10 rounded-full" aria-hidden="true" />
) : (
<Switch
checked={desktopEnabled}
onCheckedChange={(checked) =>
onSaveDesktopEnabled({ enable_desktop: checked })
}
aria-label="Enable"
disabled={isDesktopSwitchDisabled}
/>
)}
</div>
</div>
<div className="m-0 flex-1 text-xs text-content-secondary">
<p className="m-0">
@@ -60,15 +111,65 @@ export const VirtualDesktopSettings: FC<VirtualDesktopSettingsProps> = ({
>
portabledesktop module
</Link>{" "}
to be installed in the workspace and the Anthropic provider to be
configured.
to be installed in the workspace and the selected computer use
provider to be configured.
</p>
</div>
<div className="ml-2 flex flex-col gap-2 border-0 border-l border-solid border-border pl-4 pt-2 sm:ml-4 sm:flex-row sm:items-center sm:justify-between sm:gap-4">
<div className="flex flex-col gap-1">
<h4
id="computer-use-provider-label"
className="m-0 text-sm font-medium text-content-primary"
>
Computer use provider
</h4>
<p
id="computer-use-provider-description"
className="m-0 text-xs text-content-secondary"
>
Select the provider agents use for computer-use actions when virtual
desktop is enabled.
</p>
</div>
<Select
value={computerUseProvider}
onValueChange={(provider) => onSaveComputerUseProvider({ provider })}
disabled={isComputerUseProviderSelectDisabled}
>
<SelectTrigger
aria-labelledby="computer-use-provider-label"
aria-describedby="computer-use-provider-description"
className="w-full sm:w-44"
>
<SelectValue placeholder="Select provider">
{isLoadingComputerUseProvider ? (
<Skeleton className="h-4 w-20" aria-hidden="true" />
) : computerUseProvider ? (
getComputerUseProviderLabel(computerUseProvider)
) : undefined}
</SelectValue>
</SelectTrigger>
<SelectContent align="end" className="min-w-[11rem]">
<SelectGroup>
{computerUseProviderOptions.map((option) => (
<SelectItem key={option.value} value={option.value}>
{option.label}
</SelectItem>
))}
</SelectGroup>
</SelectContent>
</Select>
</div>
{isSaveDesktopEnabledError && (
<p className="m-0 text-xs text-content-destructive">
Failed to save desktop setting.
</p>
)}
{computerUseProviderSaveError && (
<p className="m-0 text-xs text-content-destructive">
Failed to save computer use provider.
</p>
)}
</div>
);
};