diff --git a/site/src/api/api.ts b/site/src/api/api.ts index f8f9027303..06d3437edf 100644 --- a/site/src/api/api.ts +++ b/site/src/api/api.ts @@ -29,9 +29,11 @@ import { } from "../utils/OneWayWebSocket"; import { type FieldError, isApiError } from "./errors"; import type { + AdvisorConfig, DeleteExternalAuthByIDResponse, DynamicParametersRequest, PostWorkspaceUsageRequest, + UpdateAdvisorConfigRequest, UsersRequest, } from "./typesGenerated"; import * as TypesGen from "./typesGenerated"; @@ -3346,6 +3348,19 @@ class ExperimentalApiMethods { await this.axios.put("/api/experimental/chats/config/desktop-enabled", req); }; + getChatAdvisorConfig = async (): Promise => { + const response = await this.axios.get( + "/api/experimental/chats/config/advisor", + ); + return response.data; + }; + + updateChatAdvisorConfig = async ( + req: UpdateAdvisorConfigRequest, + ): Promise => { + await this.axios.put("/api/experimental/chats/config/advisor", req); + }; + getChatWorkspaceTTL = async (): Promise => { const response = await this.axios.get( diff --git a/site/src/api/queries/chats.test.ts b/site/src/api/queries/chats.test.ts index 15bf01bd76..eb0fb9c2bb 100644 --- a/site/src/api/queries/chats.test.ts +++ b/site/src/api/queries/chats.test.ts @@ -11,6 +11,8 @@ import { addChildToParentInCache, archiveChat, cancelChatListRefetches, + chatAdvisorConfig, + chatAdvisorConfigKey, chatCostSummary, chatCostSummaryKey, chatDebugRunsKey, @@ -37,6 +39,7 @@ import { TERMINAL_RUN_STATUSES, unarchiveChat, unpinChat, + updateChatAdvisorConfig, updateChatPlanMode, updateChildInParentCache, updateInfiniteChatsCache, @@ -57,6 +60,8 @@ vi.mock("#/api/api", () => ({ promoteChatQueuedMessage: vi.fn(), proposeChatTitle: vi.fn(), regenerateChatTitle: vi.fn(), + getChatAdvisorConfig: vi.fn(), + updateChatAdvisorConfig: vi.fn(), }, }, })); @@ -124,6 +129,56 @@ const createTestQueryClient = (): QueryClient => }, }); +describe("advisor config query factories", () => { + it("builds the advisor config query and delegates to the API", async () => { + const advisorConfig: TypesGen.AdvisorConfig = { + enabled: true, + max_uses_per_run: 5, + max_output_tokens: 2048, + reasoning_effort: "high", + model_config_id: "00000000-0000-0000-0000-000000000000", + }; + vi.mocked(API.experimental.getChatAdvisorConfig).mockResolvedValue( + advisorConfig, + ); + + const query = chatAdvisorConfig(); + + expect(query.queryKey).toEqual(chatAdvisorConfigKey); + await expect(query.queryFn()).resolves.toEqual(advisorConfig); + expect(API.experimental.getChatAdvisorConfig).toHaveBeenCalled(); + }); + + it("sends the update request and invalidates the advisor config cache", async () => { + const queryClient = createTestQueryClient(); + queryClient.setQueryData(chatAdvisorConfigKey, { + enabled: false, + max_uses_per_run: 0, + max_output_tokens: 0, + reasoning_effort: "", + model_config_id: "", + } as TypesGen.AdvisorConfig); + + const req: TypesGen.UpdateAdvisorConfigRequest = { + enabled: true, + max_uses_per_run: 5, + max_output_tokens: 2048, + reasoning_effort: "high", + model_config_id: "00000000-0000-0000-0000-000000000000", + }; + vi.mocked(API.experimental.updateChatAdvisorConfig).mockResolvedValue(); + + const mutation = updateChatAdvisorConfig(queryClient); + await mutation.mutationFn(req); + expect(API.experimental.updateChatAdvisorConfig).toHaveBeenCalledWith(req); + + await mutation.onSuccess?.(); + expect(queryClient.getQueryState(chatAdvisorConfigKey)?.isInvalidated).toBe( + true, + ); + }); +}); + describe("invalidateChatListQueries", () => { it("invalidates flat and infinite chat list queries", async () => { const queryClient = createTestQueryClient(); diff --git a/site/src/api/queries/chats.ts b/site/src/api/queries/chats.ts index c2124175d3..26b0c1b8f2 100644 --- a/site/src/api/queries/chats.ts +++ b/site/src/api/queries/chats.ts @@ -1330,6 +1330,23 @@ export const updateChatDesktopEnabled = (queryClient: QueryClient) => ({ }); export * from "./chatDebugLogging"; +export const chatAdvisorConfigKey = ["chat-advisor-config"] as const; + +export const chatAdvisorConfig = () => ({ + queryKey: chatAdvisorConfigKey, + queryFn: (): Promise => + API.experimental.getChatAdvisorConfig(), +}); + +export const updateChatAdvisorConfig = (queryClient: QueryClient) => ({ + mutationFn: (req: TypesGen.UpdateAdvisorConfigRequest) => + API.experimental.updateChatAdvisorConfig(req), + onSuccess: async () => { + await queryClient.invalidateQueries({ + queryKey: chatAdvisorConfigKey, + }); + }, +}); const chatWorkspaceTTLKey = ["chat-workspace-ttl"] as const; diff --git a/site/src/pages/AgentsPage/AgentSettingsExperimentsPage.tsx b/site/src/pages/AgentsPage/AgentSettingsExperimentsPage.tsx index 8e5c6294c7..e576783815 100644 --- a/site/src/pages/AgentsPage/AgentSettingsExperimentsPage.tsx +++ b/site/src/pages/AgentsPage/AgentSettingsExperimentsPage.tsx @@ -1,8 +1,11 @@ import type { FC } from "react"; import { useMutation, useQuery, useQueryClient } from "react-query"; import { + chatAdvisorConfig, chatDebugLogging, chatDesktopEnabled, + chatModelConfigs, + updateChatAdvisorConfig, updateChatDebugLogging, updateChatDesktopEnabled, } from "#/api/queries/chats"; @@ -21,12 +24,23 @@ const AgentSettingsExperimentsPage: FC = () => { ...chatDebugLogging(), enabled: permissions.editDeploymentConfig, }); + const advisorConfigQuery = useQuery({ + ...chatAdvisorConfig(), + enabled: permissions.editDeploymentConfig, + }); + const modelConfigsQuery = useQuery({ + ...chatModelConfigs(), + enabled: permissions.editDeploymentConfig, + }); const saveDesktopEnabledMutation = useMutation( updateChatDesktopEnabled(queryClient), ); const saveDebugLoggingMutation = useMutation( updateChatDebugLogging(queryClient), ); + const saveAdvisorConfigMutation = useMutation( + updateChatAdvisorConfig(queryClient), + ); return ( @@ -39,6 +53,18 @@ const AgentSettingsExperimentsPage: FC = () => { onSaveDebugLogging={saveDebugLoggingMutation.mutate} isSavingDebugLogging={saveDebugLoggingMutation.isPending} isSaveDebugLoggingError={saveDebugLoggingMutation.isError} + advisorConfigData={advisorConfigQuery.data} + isAdvisorConfigLoading={advisorConfigQuery.isLoading} + isAdvisorConfigFetching={advisorConfigQuery.isFetching} + isAdvisorConfigLoadError={advisorConfigQuery.isError} + modelConfigsData={modelConfigsQuery.data ?? []} + modelConfigsError={modelConfigsQuery.error} + isLoadingModelConfigs={modelConfigsQuery.isLoading} + isFetchingModelConfigs={modelConfigsQuery.isFetching} + onSaveAdvisorConfig={saveAdvisorConfigMutation.mutate} + isSavingAdvisorConfig={saveAdvisorConfigMutation.isPending} + isSaveAdvisorConfigError={saveAdvisorConfigMutation.isError} + saveAdvisorConfigError={saveAdvisorConfigMutation.error} /> ); diff --git a/site/src/pages/AgentsPage/AgentSettingsExperimentsPageView.stories.tsx b/site/src/pages/AgentsPage/AgentSettingsExperimentsPageView.stories.tsx index 1d9513cde5..444dc6ad2b 100644 --- a/site/src/pages/AgentsPage/AgentSettingsExperimentsPageView.stories.tsx +++ b/site/src/pages/AgentsPage/AgentSettingsExperimentsPageView.stories.tsx @@ -17,6 +17,24 @@ const baseArgs: AgentSettingsExperimentsPageViewProps = { onSaveDebugLogging: fn(), isSavingDebugLogging: false, isSaveDebugLoggingError: false, + advisorConfigData: { + enabled: false, + max_uses_per_run: 0, + max_output_tokens: 0, + reasoning_effort: "", + model_config_id: "", + }, + isAdvisorConfigLoading: false, + isAdvisorConfigFetching: false, + isAdvisorConfigLoadError: false, + modelConfigsData: [], + modelConfigsError: undefined, + isLoadingModelConfigs: false, + isFetchingModelConfigs: false, + onSaveAdvisorConfig: fn(), + isSavingAdvisorConfig: false, + isSaveAdvisorConfigError: false, + saveAdvisorConfigError: undefined, }; const meta = { diff --git a/site/src/pages/AgentsPage/AgentSettingsExperimentsPageView.tsx b/site/src/pages/AgentsPage/AgentSettingsExperimentsPageView.tsx index 2dd2284ebd..5671cbade9 100644 --- a/site/src/pages/AgentsPage/AgentSettingsExperimentsPageView.tsx +++ b/site/src/pages/AgentsPage/AgentSettingsExperimentsPageView.tsx @@ -2,9 +2,15 @@ import type { FC } from "react"; import type { UseMutateFunction } from "react-query"; import type * as TypesGen from "#/api/typesGenerated"; import { AdminChatDebugLoggingSettings } from "./components/AdminChatDebugLoggingSettings"; +import { AdvisorSettings } from "./components/AdvisorSettings"; import { SectionHeader } from "./components/SectionHeader"; import { VirtualDesktopSettings } from "./components/VirtualDesktopSettings"; +interface MutationCallbacks { + onSuccess?: () => void; + onError?: () => void; +} + export interface AgentSettingsExperimentsPageViewProps { desktopEnabledData: TypesGen.ChatDesktopEnabledResponse | undefined; onSaveDesktopEnabled: UseMutateFunction< @@ -24,6 +30,21 @@ export interface AgentSettingsExperimentsPageViewProps { >; isSavingDebugLogging: boolean; isSaveDebugLoggingError: boolean; + advisorConfigData: TypesGen.AdvisorConfig | undefined; + isAdvisorConfigLoading: boolean; + isAdvisorConfigFetching: boolean; + isAdvisorConfigLoadError: boolean; + modelConfigsData: readonly TypesGen.ChatModelConfig[]; + modelConfigsError: unknown; + isLoadingModelConfigs: boolean; + isFetchingModelConfigs: boolean; + onSaveAdvisorConfig: ( + req: TypesGen.UpdateAdvisorConfigRequest, + options?: MutationCallbacks, + ) => void; + isSavingAdvisorConfig: boolean; + isSaveAdvisorConfigError: boolean; + saveAdvisorConfigError: unknown; } export const AgentSettingsExperimentsPageView: FC< @@ -37,6 +58,18 @@ export const AgentSettingsExperimentsPageView: FC< onSaveDebugLogging, isSavingDebugLogging, isSaveDebugLoggingError, + advisorConfigData, + isAdvisorConfigLoading, + isAdvisorConfigFetching, + isAdvisorConfigLoadError, + modelConfigsData, + modelConfigsError, + isLoadingModelConfigs, + isFetchingModelConfigs, + onSaveAdvisorConfig, + isSavingAdvisorConfig, + isSaveAdvisorConfigError, + saveAdvisorConfigError, }) => { return (
@@ -50,6 +83,20 @@ export const AgentSettingsExperimentsPageView: FC< isSavingDesktopEnabled={isSavingDesktopEnabled} isSaveDesktopEnabledError={isSaveDesktopEnabledError} /> + { + options?.onSuccess?.(); + }), + isSavingAdvisorConfig: false, + isSaveAdvisorConfigError: false, + saveAdvisorConfigError: undefined, + }, + decorators: [ + (Story) => ( +
+ +
+ ), + ], +} satisfies Meta; + +export default meta; +type Story = StoryObj; + +export const Default: Story = { + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + const enableAdvisorSwitch = canvas.getByRole("switch", { + name: /Enable advisor/i, + }); + + expect( + canvas.queryByRole("spinbutton", { name: /Max uses per run/i }), + ).not.toBeInTheDocument(); + expect( + canvas.queryByRole("combobox", { name: /Advisor model/i }), + ).not.toBeInTheDocument(); + + await userEvent.click(enableAdvisorSwitch); + + await waitFor(() => { + expect( + canvas.getByRole("spinbutton", { name: /Max uses per run/i }), + ).toBeVisible(); + expect( + canvas.getByRole("combobox", { name: /Advisor model/i }), + ).toBeVisible(); + }); + }, +}; + +export const Enabled: Story = { + args: { + advisorConfigData: { + ...defaultAdvisorConfig, + enabled: true, + }, + }, + play: async ({ canvasElement, args }) => { + const canvas = within(canvasElement); + const body = within(document.body); + const maxUsesInput = await canvas.findByRole("spinbutton", { + name: /Max uses per run/i, + }); + const maxOutputTokensInput = canvas.getByRole("spinbutton", { + name: /Max output tokens/i, + }); + const reasoningEffortSelect = canvas.getByRole("combobox", { + name: /Reasoning effort/i, + }); + const advisorModelSelect = canvas.getByRole("combobox", { + name: /Advisor model/i, + }); + const saveButton = canvas.getByRole("button", { name: /Save/i }); + + expect(saveButton).toBeDisabled(); + + await userEvent.clear(maxUsesInput); + await userEvent.type(maxUsesInput, "5"); + await userEvent.clear(maxOutputTokensInput); + await userEvent.type(maxOutputTokensInput, "2048"); + + await userEvent.click(reasoningEffortSelect); + await userEvent.click(await body.findByRole("option", { name: /^High$/i })); + + await userEvent.click(advisorModelSelect); + expect( + body.queryByRole("option", { name: /GPT-3.5 \(Disabled\)/i }), + ).not.toBeInTheDocument(); + await userEvent.click( + await body.findByRole("option", { name: /Claude Sonnet 4/i }), + ); + + await waitFor(() => { + expect(saveButton).toBeEnabled(); + }); + + await userEvent.click(saveButton); + + await waitFor(() => { + expect(args.onSaveAdvisorConfig).toHaveBeenCalled(); + }); + + const [request, options] = args.onSaveAdvisorConfig.mock.calls[0]; + expect(request).toEqual({ + enabled: true, + max_uses_per_run: 5, + max_output_tokens: 2048, + reasoning_effort: "high", + model_config_id: "model-2", + }); + expect(typeof options?.onSuccess).toBe("function"); + + await waitFor(() => { + expect(saveButton).toBeDisabled(); + }); + }, +}; + +export const SaveWithUseChatModel: Story = { + args: { + advisorConfigData: { + ...defaultAdvisorConfig, + enabled: true, + }, + }, + play: async ({ canvasElement, args }) => { + const canvas = within(canvasElement); + const maxUsesInput = await canvas.findByRole("spinbutton", { + name: /Max uses per run/i, + }); + + expect( + canvas.getByRole("combobox", { name: /Advisor model/i }), + ).toHaveTextContent(/Use chat model/i); + + await userEvent.clear(maxUsesInput); + await userEvent.type(maxUsesInput, "3"); + + const saveButton = canvas.getByRole("button", { name: /Save/i }); + await waitFor(() => { + expect(saveButton).toBeEnabled(); + }); + await userEvent.click(saveButton); + + await waitFor(() => { + expect(args.onSaveAdvisorConfig).toHaveBeenCalled(); + }); + const [request] = args.onSaveAdvisorConfig.mock.calls[0]; + expect(request.model_config_id).toBe(nilUUID); + }, +}; + +export const NilUUIDInitialRoundTrip: Story = { + args: { + advisorConfigData: { + ...defaultAdvisorConfig, + enabled: true, + model_config_id: nilUUID, + }, + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + const advisorModelSelect = await canvas.findByRole("combobox", { + name: /Advisor model/i, + }); + + expect(advisorModelSelect).toHaveTextContent(/Use chat model/i); + }, +}; + +export const CustomConfig: Story = { + args: { + advisorConfigData: { + enabled: true, + max_uses_per_run: 7, + max_output_tokens: 8192, + reasoning_effort: "medium", + model_config_id: "model-2", + }, + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + const maxUsesInput = await canvas.findByRole("spinbutton", { + name: /Max uses per run/i, + }); + const maxOutputTokensInput = canvas.getByRole("spinbutton", { + name: /Max output tokens/i, + }); + + expect(maxUsesInput).toHaveValue(7); + expect(maxOutputTokensInput).toHaveValue(8192); + expect( + canvas.getByRole("combobox", { name: /Reasoning effort/i }), + ).toHaveTextContent(/Medium/i); + expect( + canvas.getByRole("combobox", { name: /Advisor model/i }), + ).toHaveTextContent(/Claude Sonnet 4/i); + }, +}; + +export const UnavailableSelectedModel: Story = { + args: { + advisorConfigData: { + ...defaultAdvisorConfig, + enabled: true, + model_config_id: "22222222-2222-2222-2222-222222222222", + }, + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + const advisorModelSelect = await canvas.findByRole("combobox", { + name: /Advisor model/i, + }); + + expect(advisorModelSelect).toHaveTextContent( + /Unavailable model \(22222222-2222-2222-2222-222222222222\)/i, + ); + }, +}; + +export const Loading: Story = { + args: { + advisorConfigData: undefined, + isAdvisorConfigLoading: true, + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + + expect( + canvas.getByRole("switch", { name: /Enable advisor/i }), + ).toBeDisabled(); + expect(canvas.getByRole("button", { name: /Save/i })).toBeDisabled(); + }, +}; + +export const Refetching: Story = { + args: { + advisorConfigData: { + ...defaultAdvisorConfig, + enabled: true, + }, + isAdvisorConfigFetching: true, + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + + expect( + canvas.getByRole("switch", { name: /Enable advisor/i }), + ).toBeDisabled(); + expect( + canvas.getByRole("spinbutton", { name: /Max uses per run/i }), + ).toBeDisabled(); + expect( + canvas.getByRole("combobox", { name: /Reasoning effort/i }), + ).toBeDisabled(); + expect(canvas.getByRole("button", { name: /Save/i })).toBeDisabled(); + }, +}; + +export const LoadingModelConfigs: Story = { + args: { + advisorConfigData: { + ...defaultAdvisorConfig, + enabled: true, + model_config_id: "model-2", + }, + modelConfigs: [], + isLoadingModelConfigs: true, + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + const advisorModelSelect = await canvas.findByRole("combobox", { + name: /Advisor model/i, + }); + + expect(advisorModelSelect).toBeDisabled(); + expect(advisorModelSelect).toHaveTextContent(/Loading/i); + expect( + canvas.getByText(/Loading chat model overrides\./i), + ).toBeInTheDocument(); + }, +}; + +export const ModelConfigsError: Story = { + args: { + advisorConfigData: { + ...defaultAdvisorConfig, + enabled: true, + model_config_id: "model-2", + }, + modelConfigsError: new Error("fail"), + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + const advisorModelSelect = await canvas.findByRole("combobox", { + name: /Advisor model/i, + }); + + expect(advisorModelSelect).toBeDisabled(); + expect( + canvas.getByText( + /Model overrides are unavailable\. The current selection will be sent unchanged\./i, + ), + ).toBeInTheDocument(); + }, +}; + +export const ModelConfigsErrorWithUnsetSelection: Story = { + args: { + advisorConfigData: { + ...defaultAdvisorConfig, + enabled: true, + }, + modelConfigsError: new Error("fail"), + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + + expect( + canvas.getByText( + /Model overrides are unavailable\. Saving will keep using the chat model\./i, + ), + ).toBeInTheDocument(); + }, +}; + +export const LoadError: Story = { + args: { + advisorConfigData: undefined, + isAdvisorConfigLoadError: true, + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + + expect( + canvas.getByText(/Failed to load advisor settings\./i), + ).toBeInTheDocument(); + }, +}; + +export const Saving: Story = { + args: { + advisorConfigData: { + ...defaultAdvisorConfig, + enabled: true, + }, + isSavingAdvisorConfig: true, + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + + expect( + canvas.getByRole("switch", { name: /Enable advisor/i }), + ).toBeDisabled(); + expect( + canvas.getByRole("spinbutton", { name: /Max uses per run/i }), + ).toBeDisabled(); + expect( + canvas.getByRole("combobox", { name: /Reasoning effort/i }), + ).toBeDisabled(); + expect(canvas.getByRole("button", { name: /Save/i })).toBeDisabled(); + }, +}; + +export const SaveError: Story = { + args: { + advisorConfigData: { + ...defaultAdvisorConfig, + enabled: true, + }, + isSaveAdvisorConfigError: true, + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + + expect( + canvas.getByText(/Failed to save advisor settings\./i), + ).toBeInTheDocument(); + }, +}; + +export const SaveErrorWithDetail: Story = { + args: { + advisorConfigData: { + ...defaultAdvisorConfig, + enabled: true, + }, + isSaveAdvisorConfigError: true, + saveAdvisorConfigError: new Error( + "reasoning_effort must be one of: low, medium, high.", + ), + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + + expect( + canvas.getByText(/reasoning_effort must be one of: low, medium, high\./i), + ).toBeInTheDocument(); + }, +}; + +export const DeselectModelBackToUseChatModel: Story = { + args: { + advisorConfigData: { + ...defaultAdvisorConfig, + enabled: true, + model_config_id: "model-2", + }, + }, + play: async ({ canvasElement, args }) => { + const canvas = within(canvasElement); + const body = within(document.body); + const advisorModelSelect = await canvas.findByRole("combobox", { + name: /Advisor model/i, + }); + + expect(advisorModelSelect).toHaveTextContent(/Claude Sonnet 4/i); + + await userEvent.click(advisorModelSelect); + await userEvent.click( + await body.findByRole("option", { name: /^Use chat model$/i }), + ); + + await waitFor(() => { + expect(advisorModelSelect).toHaveTextContent(/Use chat model/i); + }); + + const saveButton = canvas.getByRole("button", { name: /Save/i }); + await waitFor(() => { + expect(saveButton).toBeEnabled(); + }); + await userEvent.click(saveButton); + + await waitFor(() => { + expect(args.onSaveAdvisorConfig).toHaveBeenCalled(); + }); + const [request] = args.onSaveAdvisorConfig.mock.calls[0]; + expect(request.model_config_id).toBe(nilUUID); + }, +}; + +export const DisableAdvisorWithDeletedModel: Story = { + args: { + advisorConfigData: { + enabled: true, + max_uses_per_run: 5, + max_output_tokens: 2048, + reasoning_effort: "high", + model_config_id: "22222222-2222-2222-2222-222222222222", + }, + }, + play: async ({ canvasElement, args }) => { + const canvas = within(canvasElement); + const enableAdvisorSwitch = await canvas.findByRole("switch", { + name: /Enable advisor/i, + }); + + expect( + canvas.getByRole("combobox", { name: /Advisor model/i }), + ).toHaveTextContent( + /Unavailable model \(22222222-2222-2222-2222-222222222222\)/i, + ); + + await userEvent.click(enableAdvisorSwitch); + + const saveButton = canvas.getByRole("button", { name: /Save/i }); + await waitFor(() => { + expect(saveButton).toBeEnabled(); + }); + await userEvent.click(saveButton); + + await waitFor(() => { + expect(args.onSaveAdvisorConfig).toHaveBeenCalled(); + }); + + // The backend rejects unknown non-nil model IDs, so disabling must + // scrub the stale override rather than forwarding it and failing + // the save with a 400. + const [request] = args.onSaveAdvisorConfig.mock.calls[0]; + expect(request).toEqual({ + enabled: false, + max_uses_per_run: 5, + max_output_tokens: 2048, + reasoning_effort: "high", + model_config_id: nilUUID, + }); + }, +}; + +export const DisableAdvisorWhileModelConfigsLoadingPreservesOverride: Story = { + args: { + advisorConfigData: { + enabled: true, + max_uses_per_run: 5, + max_output_tokens: 2048, + reasoning_effort: "high", + model_config_id: "22222222-2222-2222-2222-222222222222", + }, + modelConfigs: [], + isLoadingModelConfigs: true, + }, + play: async ({ canvasElement, args }) => { + const canvas = within(canvasElement); + const enableAdvisorSwitch = await canvas.findByRole("switch", { + name: /Enable advisor/i, + }); + + await userEvent.click(enableAdvisorSwitch); + + const saveButton = canvas.getByRole("button", { name: /Save/i }); + await waitFor(() => { + expect(saveButton).toBeEnabled(); + }); + await userEvent.click(saveButton); + + await waitFor(() => { + expect(args.onSaveAdvisorConfig).toHaveBeenCalled(); + }); + + // While the model-configs query is in flight we cannot distinguish a + // genuinely deleted override from one we simply have not fetched yet, + // so the override must be forwarded unchanged. The backend will + // surface a specific 400 if the ID really has been deleted. + const [request] = args.onSaveAdvisorConfig.mock.calls[0]; + expect(request).toEqual({ + enabled: false, + max_uses_per_run: 5, + max_output_tokens: 2048, + reasoning_effort: "high", + model_config_id: "22222222-2222-2222-2222-222222222222", + }); + }, +}; + +export const DisableAdvisorWithModelConfigsErrorPreservesOverride: Story = { + args: { + advisorConfigData: { + enabled: true, + max_uses_per_run: 5, + max_output_tokens: 2048, + reasoning_effort: "high", + model_config_id: "22222222-2222-2222-2222-222222222222", + }, + modelConfigs: [], + modelConfigsError: new Error("fail"), + }, + play: async ({ canvasElement, args }) => { + const canvas = within(canvasElement); + const enableAdvisorSwitch = await canvas.findByRole("switch", { + name: /Enable advisor/i, + }); + + await userEvent.click(enableAdvisorSwitch); + + const saveButton = canvas.getByRole("button", { name: /Save/i }); + await waitFor(() => { + expect(saveButton).toBeEnabled(); + }); + await userEvent.click(saveButton); + + await waitFor(() => { + expect(args.onSaveAdvisorConfig).toHaveBeenCalled(); + }); + + // When the model-configs query has failed we cannot verify whether + // the override still exists, so the override must be forwarded + // unchanged rather than silently dropped. The backend surfaces a + // specific 400 if the ID really has been deleted. + const [request] = args.onSaveAdvisorConfig.mock.calls[0]; + expect(request).toEqual({ + enabled: false, + max_uses_per_run: 5, + max_output_tokens: 2048, + reasoning_effort: "high", + model_config_id: "22222222-2222-2222-2222-222222222222", + }); + }, +}; + +export const DisableAdvisorWhileModelConfigsRefetchingPreservesOverride: Story = + { + args: { + advisorConfigData: { + enabled: true, + max_uses_per_run: 5, + max_output_tokens: 2048, + reasoning_effort: "high", + model_config_id: "22222222-2222-2222-2222-222222222222", + }, + // Simulate a background refetch with stale cached data: react-query + // keeps `isLoading` false once cached data exists, so only + // `isFetching` flags the in-flight refetch. The cached list does not + // contain the override ID. + modelConfigs: mockModelConfigs, + isLoadingModelConfigs: false, + isFetchingModelConfigs: true, + }, + play: async ({ canvasElement, args }) => { + const canvas = within(canvasElement); + const enableAdvisorSwitch = await canvas.findByRole("switch", { + name: /Enable advisor/i, + }); + + await userEvent.click(enableAdvisorSwitch); + + const saveButton = canvas.getByRole("button", { name: /Save/i }); + await waitFor(() => { + expect(saveButton).toBeEnabled(); + }); + await userEvent.click(saveButton); + + await waitFor(() => { + expect(args.onSaveAdvisorConfig).toHaveBeenCalled(); + }); + + // While a background refetch is in flight the cached model list may + // be stale, so the scrub guard must treat the absence of the + // override as indeterminate and forward the ID unchanged. Otherwise + // a still-valid override could be silently dropped just because the + // cache lags behind the server. + const [request] = args.onSaveAdvisorConfig.mock.calls[0]; + expect(request).toEqual({ + enabled: false, + max_uses_per_run: 5, + max_output_tokens: 2048, + reasoning_effort: "high", + model_config_id: "22222222-2222-2222-2222-222222222222", + }); + }, + }; + +export const DisableAdvisorWithDeletedModelAndEmptyModelConfigs: Story = { + args: { + advisorConfigData: { + enabled: true, + max_uses_per_run: 5, + max_output_tokens: 2048, + reasoning_effort: "high", + model_config_id: "22222222-2222-2222-2222-222222222222", + }, + modelConfigs: [], + modelConfigsError: undefined, + isLoadingModelConfigs: false, + }, + play: async ({ canvasElement, args }) => { + const canvas = within(canvasElement); + const enableAdvisorSwitch = await canvas.findByRole("switch", { + name: /Enable advisor/i, + }); + + await userEvent.click(enableAdvisorSwitch); + + const saveButton = canvas.getByRole("button", { name: /Save/i }); + await waitFor(() => { + expect(saveButton).toBeEnabled(); + }); + await userEvent.click(saveButton); + + await waitFor(() => { + expect(args.onSaveAdvisorConfig).toHaveBeenCalled(); + }); + + // An empty model-configs list after a successful load is a definitive + // answer that the override no longer exists, so disabling must scrub + // the stale ID rather than forwarding it and failing the save with a + // 400. This covers the recovery case where every model config has + // been deleted. + const [request] = args.onSaveAdvisorConfig.mock.calls[0]; + expect(request).toEqual({ + enabled: false, + max_uses_per_run: 5, + max_output_tokens: 2048, + reasoning_effort: "high", + model_config_id: nilUUID, + }); + }, +}; + +export const ValidationBlocksSave: Story = { + args: { + advisorConfigData: { + ...defaultAdvisorConfig, + enabled: true, + max_uses_per_run: 5, + }, + }, + play: async ({ canvasElement }) => { + const canvas = within(canvasElement); + const maxUsesInput = await canvas.findByRole("spinbutton", { + name: /Max uses per run/i, + }); + const saveButton = canvas.getByRole("button", { name: /Save/i }); + + expect(saveButton).toBeDisabled(); + + // Dirty-but-invalid state: the field is blank, so client validation + // must keep Save disabled even though the form is now dirty. + await userEvent.clear(maxUsesInput); + await waitFor(() => { + expect(saveButton).toBeDisabled(); + }); + + await userEvent.type(maxUsesInput, "3"); + await waitFor(() => { + expect(saveButton).toBeEnabled(); + }); + }, +}; + +export const DisableThenSave: Story = { + args: { + advisorConfigData: { + enabled: true, + max_uses_per_run: 5, + max_output_tokens: 2048, + reasoning_effort: "high", + model_config_id: "model-2", + }, + }, + play: async ({ canvasElement, args }) => { + const canvas = within(canvasElement); + const enableAdvisorSwitch = await canvas.findByRole("switch", { + name: /Enable advisor/i, + }); + + expect( + canvas.getByRole("spinbutton", { name: /Max uses per run/i }), + ).toBeVisible(); + + // Clear a numeric field to an invalid value before disabling. After + // disabling the field is hidden, and saving must not silently overwrite + // the stored limit with a coerced value. + const maxUsesInput = canvas.getByRole("spinbutton", { + name: /Max uses per run/i, + }); + await userEvent.clear(maxUsesInput); + + await userEvent.click(enableAdvisorSwitch); + + await waitFor(() => { + expect( + canvas.queryByRole("spinbutton", { name: /Max uses per run/i }), + ).not.toBeInTheDocument(); + }); + + const saveButton = canvas.getByRole("button", { name: /Save/i }); + await waitFor(() => { + expect(saveButton).toBeEnabled(); + }); + + await userEvent.click(saveButton); + + await waitFor(() => { + expect(args.onSaveAdvisorConfig).toHaveBeenCalled(); + }); + + const [request] = args.onSaveAdvisorConfig.mock.calls[0]; + expect(request).toEqual({ + enabled: false, + max_uses_per_run: 5, + max_output_tokens: 2048, + reasoning_effort: "high", + model_config_id: "model-2", + }); + }, +}; diff --git a/site/src/pages/AgentsPage/components/AdvisorSettings.tsx b/site/src/pages/AgentsPage/components/AdvisorSettings.tsx new file mode 100644 index 0000000000..2ea74aa5b1 --- /dev/null +++ b/site/src/pages/AgentsPage/components/AdvisorSettings.tsx @@ -0,0 +1,498 @@ +import { useFormik } from "formik"; +import { TriangleAlertIcon } from "lucide-react"; +import { type FC, useEffect, useId, useRef } from "react"; +import { getErrorMessage } from "#/api/errors"; +import type { + AdvisorConfig, + ChatModelConfig, + UpdateAdvisorConfigRequest, +} from "#/api/typesGenerated"; +import { Badge } from "#/components/Badge/Badge"; +import { Button } from "#/components/Button/Button"; +import { Input } from "#/components/Input/Input"; +import { Label } from "#/components/Label/Label"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "#/components/Select/Select"; +import { Switch } from "#/components/Switch/Switch"; + +const nilUUID = "00000000-0000-0000-0000-000000000000"; +const advisorReasoningEfforts = ["", "low", "medium", "high"] as const; +type AdvisorReasoningEffort = (typeof advisorReasoningEfforts)[number]; +const chatModelFallbackValue = "__use-chat-model__"; +const unavailableModelValue = "__unavailable-model__"; +const chatReasoningFallbackValue = "__use-chat-reasoning__"; + +interface MutationCallbacks { + onSuccess?: () => void; + onError?: () => void; +} + +interface AdvisorSettingsProps { + advisorConfigData: AdvisorConfig | undefined; + isAdvisorConfigLoading: boolean; + isAdvisorConfigFetching: boolean; + isAdvisorConfigLoadError: boolean; + modelConfigs: readonly ChatModelConfig[]; + modelConfigsError: unknown; + isLoadingModelConfigs: boolean; + isFetchingModelConfigs: boolean; + onSaveAdvisorConfig: ( + req: UpdateAdvisorConfigRequest, + options?: MutationCallbacks, + ) => void; + isSavingAdvisorConfig: boolean; + isSaveAdvisorConfigError: boolean; + saveAdvisorConfigError: unknown; +} + +type AdvisorSettingsFormValues = { + enabled: boolean; + max_uses_per_run: string; + max_output_tokens: string; + reasoning_effort: AdvisorReasoningEffort; + model_config_id: string; +}; + +const isUnsetModelConfigId = (id: string): boolean => + id === "" || id === nilUUID; + +const isAdvisorReasoningEffort = ( + value: string, +): value is AdvisorReasoningEffort => { + return advisorReasoningEfforts.includes(value as AdvisorReasoningEffort); +}; + +const normalizeNonNegativeInteger = ( + value: number | string | undefined, +): number => { + const parsed = typeof value === "number" ? value : Number(value); + if (!Number.isFinite(parsed) || parsed < 0) { + return 0; + } + return Math.trunc(parsed); +}; + +const normalizeAdvisorConfig = ( + config: AdvisorConfig | undefined, +): AdvisorSettingsFormValues => { + const reasoningEffort = config?.reasoning_effort ?? ""; + return { + enabled: config?.enabled ?? false, + max_uses_per_run: String( + normalizeNonNegativeInteger(config?.max_uses_per_run), + ), + max_output_tokens: String( + normalizeNonNegativeInteger(config?.max_output_tokens), + ), + reasoning_effort: isAdvisorReasoningEffort(reasoningEffort) + ? reasoningEffort + : "", + model_config_id: + typeof config?.model_config_id === "string" && + !isUnsetModelConfigId(config.model_config_id) + ? config.model_config_id + : "", + }; +}; + +const toAdvisorConfigRequest = ( + values: AdvisorSettingsFormValues, +): UpdateAdvisorConfigRequest => ({ + enabled: values.enabled, + max_uses_per_run: normalizeNonNegativeInteger(values.max_uses_per_run), + max_output_tokens: normalizeNonNegativeInteger(values.max_output_tokens), + reasoning_effort: values.reasoning_effort, + model_config_id: isUnsetModelConfigId(values.model_config_id) + ? nilUUID + : values.model_config_id, +}); + +const isNonNegativeIntegerString = (value: string): boolean => { + if (value.trim() === "") { + return false; + } + const parsed = Number(value); + return Number.isFinite(parsed) && parsed >= 0 && Number.isInteger(parsed); +}; + +const validateAdvisorConfig = (values: AdvisorSettingsFormValues) => { + const errors: Partial> = {}; + + // Skip validation of the advisor-only fields when the feature is disabled. + // Those inputs are hidden, so an admin disabling the advisor should not be + // blocked by stale invalid values left in hidden fields. + if (!values.enabled) { + return errors; + } + + if (!isNonNegativeIntegerString(values.max_uses_per_run)) { + errors.max_uses_per_run = + "Max uses per run must be a non-negative integer."; + } + + if (!isNonNegativeIntegerString(values.max_output_tokens)) { + errors.max_output_tokens = + "Max output tokens must be a non-negative integer."; + } + + if (!isAdvisorReasoningEffort(values.reasoning_effort)) { + errors.reasoning_effort = "Select a valid reasoning effort."; + } + + return errors; +}; + +const getModelDisplayName = (config: ChatModelConfig): string => + config.display_name.trim() || config.model; + +const getReasoningEffortLabel = (value: AdvisorReasoningEffort): string => { + switch (value) { + case "low": + return "Low"; + case "medium": + return "Medium"; + case "high": + return "High"; + default: + return "Use chat model default"; + } +}; + +export const AdvisorSettings: FC = ({ + advisorConfigData, + isAdvisorConfigLoading, + isAdvisorConfigFetching, + isAdvisorConfigLoadError, + modelConfigs, + modelConfigsError, + isLoadingModelConfigs, + isFetchingModelConfigs, + onSaveAdvisorConfig, + isSavingAdvisorConfig, + isSaveAdvisorConfigError, + saveAdvisorConfigError, +}) => { + const maxUsesId = useId(); + const maxOutputTokensId = useId(); + const hasLoadedAdvisorConfig = advisorConfigData !== undefined; + const enabledModelConfigs = modelConfigs.filter((config) => config.enabled); + + // Track the most recent committed advisor values (the server's view or the + // last successful save). Reading `advisorConfigData` directly in `onSubmit` + // can yield a stale snapshot when a refetch is in flight or has failed, + // which would silently roll back recently saved limits if the user then + // disables the advisor before the query settles. + const committedValuesRef = useRef( + normalizeAdvisorConfig(advisorConfigData), + ); + useEffect(() => { + committedValuesRef.current = normalizeAdvisorConfig(advisorConfigData); + }, [advisorConfigData]); + + const form = useFormik({ + enableReinitialize: true, + validateOnMount: true, + initialValues: normalizeAdvisorConfig(advisorConfigData), + validate: validateAdvisorConfig, + onSubmit: (values, { resetForm }) => { + // When disabling, preserve the last committed values for the hidden + // fields so potentially invalid in-flight edits (empty strings, + // fractional numbers) cannot silently overwrite previously + // configured limits, and so a pending or failed refetch of the + // advisor config cannot revert recently saved values. + let source: AdvisorSettingsFormValues = values.enabled + ? values + : { ...committedValuesRef.current, enabled: false }; + // If the last committed model override references a model config + // that no longer exists, the backend rejects the stale ID with a + // 400. When disabling, clear the override so a simple disable + // stays reliable in that edge case; the override is unusable + // anyway and the admin will reselect one on re-enable. Only scrub + // when model configs have loaded successfully and no refetch is in + // flight: during an initial load, a background refetch, or on + // error we cannot distinguish "truly missing" from "not loaded + // yet", and deciding from stale cache could either preserve a + // now-deleted ID (causing a 400 on disable/save) or silently drop + // an override that is actually still valid but missing from a + // stale cache. `isLoading` alone is insufficient because + // react-query keeps it false during background refetches when + // cached data already exists, so `isFetching` covers that gap. An + // empty list after a successful load is a definitive answer, so + // the scrub still fires (covers the recovery case where every + // model config has been deleted). + if ( + !source.enabled && + !isUnsetModelConfigId(source.model_config_id) && + !isLoadingModelConfigs && + !isFetchingModelConfigs && + !modelConfigsError && + !modelConfigs.some((config) => config.id === source.model_config_id) + ) { + source = { ...source, model_config_id: "" }; + } + const request = toAdvisorConfigRequest(source); + onSaveAdvisorConfig(request, { + onSuccess: () => { + const nextValues = normalizeAdvisorConfig(request); + committedValuesRef.current = nextValues; + resetForm({ values: nextValues }); + }, + }); + }, + }); + + const isFormDisabled = + isSavingAdvisorConfig || + isAdvisorConfigLoading || + isAdvisorConfigFetching || + !hasLoadedAdvisorConfig; + const isModelSelectDisabled = + isFormDisabled || isLoadingModelConfigs || Boolean(modelConfigsError); + const hasUnavailableSelectedModel = + !isLoadingModelConfigs && + !isUnsetModelConfigId(form.values.model_config_id) && + !enabledModelConfigs.some( + (config) => config.id === form.values.model_config_id, + ); + const selectedModelConfig = modelConfigs.find( + (config) => config.id === form.values.model_config_id, + ); + const selectedModelLabel = isUnsetModelConfigId(form.values.model_config_id) + ? "Use chat model" + : isLoadingModelConfigs + ? "Loading..." + : selectedModelConfig + ? getModelDisplayName(selectedModelConfig) + : `Unavailable model (${form.values.model_config_id})`; + const selectedModelValue = isUnsetModelConfigId(form.values.model_config_id) + ? chatModelFallbackValue + : hasUnavailableSelectedModel + ? unavailableModelValue + : form.values.model_config_id; + const modelHelperText = isLoadingModelConfigs + ? "Loading chat model overrides." + : modelConfigsError + ? isUnsetModelConfigId(form.values.model_config_id) + ? "Model overrides are unavailable. Saving will keep using the chat model." + : "Model overrides are unavailable. The current selection will be sent unchanged." + : "Choose a dedicated advisor model, or leave this unset to reuse the chat model."; + + return ( +
+
+

+ Advisor +

+ + + Experimental feature + +
+
+
+

+ Allow root agent chats to call the advisor tool for strategic + guidance. +

+

+ When enabled, you can cap advisor usage per run and optionally use + an override model. +

+
+ + void form.setFieldValue("enabled", checked) + } + aria-label="Enable advisor" + disabled={isFormDisabled} + /> +
+ + {form.values.enabled && ( +
+
+ + + void form.setFieldValue( + "max_uses_per_run", + event.currentTarget.value, + ) + } + onBlur={form.handleBlur} + aria-invalid={Boolean(form.errors.max_uses_per_run)} + disabled={isFormDisabled} + className="h-9 bg-surface-primary text-[13px]" + /> +

+ Set to 0 to leave the per-run call count unlimited. +

+
+ +
+ + + void form.setFieldValue( + "max_output_tokens", + event.currentTarget.value, + ) + } + onBlur={form.handleBlur} + aria-invalid={Boolean(form.errors.max_output_tokens)} + disabled={isFormDisabled} + className="h-9 bg-surface-primary text-[13px]" + /> +

+ Set to 0 to use the server default output limit. +

+
+ +
+ + +

+ Controls how hard the advisor model reasons before responding. + Leave unset to use the model's default. +

+
+ +
+ + +

+ {modelHelperText} +

+
+
+ )} + +
+ +
+ + {isSaveAdvisorConfigError && ( +

+ {getErrorMessage( + saveAdvisorConfigError, + "Failed to save advisor settings.", + )} +

+ )} + {isAdvisorConfigLoadError && ( +

+ Failed to load advisor settings. +

+ )} +
+ ); +};