feat(site): show best-effort model pricing on the model form (#28290)

This commit is contained in:
Danielle Maywood
2026-08-19 11:45:30 +01:00
committed by GitHub
parent 34e95c46bf
commit 8405bbb26c
8 changed files with 458 additions and 10 deletions
+11
View File
@@ -3477,6 +3477,17 @@ class ExperimentalApiMethods {
return response.data;
};
getAIModelPrices = async (filter: {
provider?: string;
model?: string;
}): Promise<TypesGen.AIModelPrice[]> => {
const response = await this.axios.get<TypesGen.AIModelPrice[]>(
"/api/experimental/ai/model-prices",
{ params: filter },
);
return response.data;
};
listAIProviders = async (): Promise<TypesGen.AIProvider[]> => {
const response = await this.axios.get<TypesGen.AIProvider[]>(
aiProviderConfigsPath,
+8
View File
@@ -10,6 +10,14 @@ import type {
const aiProvidersListKey = ["ai", "providers"] as const;
const aiModelPricesKey = ["ai", "model-prices"] as const;
export const aiModelPrices = (provider: string, model: string) =>
queryOptions({
queryKey: [...aiModelPricesKey, provider, model] as const,
queryFn: () => API.experimental.getAIModelPrices({ provider, model }),
});
export const aiProviderKeyFor = (idOrName: string) =>
[...aiProvidersListKey, idOrName] as const;
@@ -1,21 +1,32 @@
import type { Meta, StoryObj } from "@storybook/react-vite";
import { expect, fn, screen, userEvent, within } from "storybook/test";
import { expect, fn, screen, spyOn, userEvent, within } from "storybook/test";
import { reactRouterParameters } from "storybook-addon-remix-react-router";
import { withToaster } from "#/testHelpers/storybook";
import { API } from "#/api/api";
import { aiModelPrices } from "#/api/queries/aiProviders";
import type * as TypesGen from "#/api/typesGenerated";
import { withDashboardProvider, withToaster } from "#/testHelpers/storybook";
import {
MockAnthropicProviderState,
MockAzureProviderState,
MockDisabledProviderState,
MockOpenAIProviderState,
mockClaude,
mockGPT5,
mockProviderDisabledModel,
} from "../testFixtures";
import { ModelForm } from "./ModelForm";
const onUpdateModel = fn(
async (
_modelConfigId: string,
_req: TypesGen.UpdateChatModelConfigRequest,
): Promise<unknown> => undefined,
);
const meta: Meta<typeof ModelForm> = {
title: "pages/AISettingsPage/ModelsPage/ModelForm",
component: ModelForm,
decorators: [withToaster],
decorators: [withToaster, withDashboardProvider],
args: {
providerStates: [MockOpenAIProviderState, MockAnthropicProviderState],
selectedProviderState: MockOpenAIProviderState,
@@ -23,9 +34,10 @@ const meta: Meta<typeof ModelForm> = {
isSaving: false,
isDeleting: false,
onCreateModel: fn(async () => undefined),
onUpdateModel: fn(async () => undefined),
onUpdateModel,
},
parameters: {
features: ["aibridge"],
reactRouter: reactRouterParameters({
location: { path: "/ai/settings/models/add" },
routing: [
@@ -411,19 +423,239 @@ export const ReasoningEffortValidationError: Story = {
},
};
export const NativeCostTrackingIsUnavailable: Story = {
// The catalog fallback covers an entitled deployment with no matching price
// book row. Values come from the baked-in catalog and must not be submitted.
export const CostEstimateFieldsAreImmutable: Story = {
args: {
editingModel: mockClaude,
selectedProviderState: MockAnthropicProviderState,
onDeleteModel: fn(async () => undefined),
},
parameters: {
queries: [
{
key: aiModelPrices("anthropic", "claude-sonnet-4-5").queryKey,
data: [],
},
],
},
play: async ({ canvasElement }) => {
const canvas = within(canvasElement);
await userEvent.click(
canvas.getByRole("button", { name: /cost estimate/i }),
);
const expectedValues: [RegExp, string][] = [
[/^input$/i, "3"],
[/^output$/i, "15"],
[/cache read/i, "0.30"],
[/cache write/i, "3.75"],
];
for (const [name, value] of expectedValues) {
const field = canvas.getByLabelText(name);
await expect(field).toHaveValue(value);
await expect(field).toHaveAttribute("readonly");
}
// Update is disabled until the form is dirty. A display name edit
// unlocks it so the payload can be checked for pricing keys.
await userEvent.type(canvas.getByLabelText(/display name/i), " (updated)");
await userEvent.click(
canvas.getByRole("button", { name: /^update model$/i }),
);
await expect(onUpdateModel).toHaveBeenCalledTimes(1);
expect(onUpdateModel.mock.calls[0]?.[1]).toStrictEqual({
display_name: "Claude Sonnet 4.5 (updated)",
model_config: {},
});
},
};
// With the AI Gateway feature entitled, prices come from the live price
// book, so admin overrides and models missing from the catalog are
// reflected. mockGPT5 is openai/gpt-5, absent from the catalog but present
// in the price book. Prices are micro-units per million tokens.
export const CostEstimateFromLivePriceBook: Story = {
args: {
editingModel: mockGPT5,
onDeleteModel: fn(async () => undefined),
},
parameters: {
queries: [
{
key: aiModelPrices("openai", "gpt-5").queryKey,
data: [
{
provider: "openai",
model: "gpt-5",
input_price: 1250000,
output_price: 10000000,
cache_read_price: 125000,
cache_write_price: null,
created_at: "2026-02-18T12:00:00.000Z",
updated_at: "2026-02-18T12:00:00.000Z",
},
],
},
],
},
play: async ({ canvasElement }) => {
const canvas = within(canvasElement);
await userEvent.click(
canvas.getByRole("button", { name: /cost estimate/i }),
);
await expect(canvas.getByLabelText(/^input$/i)).toHaveValue("1.25");
await expect(canvas.getByLabelText(/^output$/i)).toHaveValue("10");
await expect(canvas.getByLabelText(/cache read/i)).toHaveValue("0.125");
await expect(canvas.getByLabelText(/cache write/i)).toHaveValue("");
},
};
// A price book row is the deployment's own pricing, so it wins outright. A
// null category on the row means the model is unpriced there and bills as
// zero, so the field stays blank instead of falling back to the catalog.
// The catalog only fills in when the model has no row at all.
export const CostEstimateRowWinsOverCatalog: Story = {
args: {
editingModel: mockClaude,
selectedProviderState: MockAnthropicProviderState,
onDeleteModel: fn(async () => undefined),
},
parameters: {
queries: [
{
key: aiModelPrices("anthropic", "claude-sonnet-4-5").queryKey,
data: [
{
provider: "anthropic",
model: "claude-sonnet-4-5",
input_price: 4000000,
output_price: null,
cache_read_price: null,
cache_write_price: null,
created_at: "2026-02-18T12:00:00.000Z",
updated_at: "2026-02-18T12:00:00.000Z",
},
],
},
],
},
play: async ({ canvasElement }) => {
const canvas = within(canvasElement);
await userEvent.click(
canvas.getByRole("button", { name: /cost estimate/i }),
);
// The override sets input to $4. Output and both cache categories are
// null on the row, so they render blank even though the catalog has
// values for them.
await expect(canvas.getByLabelText(/^input$/i)).toHaveValue("4");
await expect(canvas.getByLabelText(/^output$/i)).toHaveValue("");
await expect(canvas.getByLabelText(/cache read/i)).toHaveValue("");
await expect(canvas.getByLabelText(/cache write/i)).toHaveValue("");
},
};
// Without the AI Gateway entitlement the price endpoint is not queried, so
// a model with no catalog entry gets the empty state.
export const CostEstimateUnavailableForUnknownModel: Story = {
args: {
editingModel: mockGPT5,
onDeleteModel: fn(async () => undefined),
},
parameters: {
features: [],
},
play: async ({ canvasElement }) => {
const canvas = within(canvasElement);
await userEvent.click(
canvas.getByRole("button", { name: /cost estimate/i }),
);
await expect(
canvas.getByRole("button", { name: /provider configuration/i }),
).toBeVisible();
canvas.getByText("No pricing data for this model."),
).toBeInTheDocument();
await expect(canvas.queryByLabelText(/^input$/i)).not.toBeInTheDocument();
},
};
// The entitlement gate means the endpoint is not called at all without the
// AI Gateway feature. A catalog model still shows catalog prices.
export const CostEstimateCatalogOnlyWhenNotEntitled: Story = {
args: {
editingModel: mockClaude,
selectedProviderState: MockAnthropicProviderState,
onDeleteModel: fn(async () => undefined),
},
parameters: {
features: [],
},
beforeEach: () => {
spyOn(API.experimental, "getAIModelPrices").mockResolvedValue([]);
},
play: async ({ canvasElement }) => {
const canvas = within(canvasElement);
await userEvent.click(
canvas.getByRole("button", { name: /cost estimate/i }),
);
await expect(canvas.getByLabelText(/^input$/i)).toHaveValue("3");
await expect(canvas.getByLabelText(/^output$/i)).toHaveValue("15");
expect(API.experimental.getAIModelPrices).not.toHaveBeenCalled();
},
};
// While the price book lookup is in flight, the four boxes stay rendered
// with a loading placeholder in each instead of a catalog price. The
// catalog must not appear because the model may have a deployment override.
export const CostEstimateLoading: Story = {
args: {
editingModel: mockClaude,
selectedProviderState: MockAnthropicProviderState,
onDeleteModel: fn(async () => undefined),
},
beforeEach: () => {
spyOn(API.experimental, "getAIModelPrices").mockImplementation(
() => new Promise(() => {}),
);
},
play: async ({ canvasElement }) => {
const canvas = within(canvasElement);
await userEvent.click(
canvas.getByRole("button", { name: /cost estimate/i }),
);
// Each box shows a loading placeholder while the lookup is in flight.
for (const label of ["Input", "Output", "Cache read", "Cache write"]) {
await expect(
canvas.getByLabelText(`${label} price loading`),
).toBeInTheDocument();
}
// The catalog numbers must not render while the lookup is pending.
expect(canvas.queryByDisplayValue("3")).not.toBeInTheDocument();
},
};
// When the price book lookup fails, the section says so instead of falling
// back to the catalog, because the model may have a deployment override.
export const CostEstimateError: Story = {
args: {
editingModel: mockClaude,
selectedProviderState: MockAnthropicProviderState,
onDeleteModel: fn(async () => undefined),
},
beforeEach: () => {
spyOn(API.experimental, "getAIModelPrices").mockRejectedValue(
new Error("request failed"),
);
},
play: async ({ canvasElement }) => {
const canvas = within(canvasElement);
await userEvent.click(
canvas.getByRole("button", { name: /cost estimate/i }),
);
await expect(
canvas.queryByRole("button", { name: /cost tracking/i }),
).not.toBeInTheDocument();
canvas.getByText("Couldn't load pricing."),
).toBeInTheDocument();
// The catalog numbers must not render on error.
expect(canvas.queryByDisplayValue("3")).not.toBeInTheDocument();
expect(canvas.queryByDisplayValue("15")).not.toBeInTheDocument();
},
};
@@ -88,6 +88,7 @@ export const ModelForm: FC<ModelFormProps> = ({
...(isDuplicating && { isDefault: false }),
};
const [showAdvanced, setShowAdvanced] = useState(false);
const [showCostEstimate, setShowCostEstimate] = useState(false);
const [showProviderConfig, setShowProviderConfig] = useState(false);
const [confirmingDelete, setConfirmingDelete] = useState(false);
const [confirmingReplaceDefault, setConfirmingReplaceDefault] =
@@ -330,6 +331,8 @@ export const ModelForm: FC<ModelFormProps> = ({
displayNameField={displayNameField}
setDefaultDisabled={setDefaultDisabled}
modelConfigFormBuildResult={modelConfigFormBuildResult}
showCostEstimate={showCostEstimate}
setShowCostEstimate={setShowCostEstimate}
showProviderConfig={showProviderConfig}
setShowProviderConfig={setShowProviderConfig}
showAdvanced={showAdvanced}
@@ -28,6 +28,7 @@ import type { ProviderState } from "#/modules/aiModels/providerStates";
import {
GeneralModelConfigFields,
ModelConfigFields,
PricingEstimateFields,
ReasoningEffortConfigFields,
} from "#/pages/AgentsPage/components/ChatModelAdminPanel/ModelConfigFields";
import { ModelIdentifierField } from "#/pages/AgentsPage/components/ChatModelAdminPanel/ModelIdentifierField";
@@ -101,6 +102,8 @@ export const ModelFormFields: FC<{
displayNameField: FormHelpers;
setDefaultDisabled: boolean;
modelConfigFormBuildResult: ModelConfigFormBuildResult;
showCostEstimate: boolean;
setShowCostEstimate: (open: boolean) => void;
showProviderConfig: boolean;
setShowProviderConfig: (open: boolean) => void;
showAdvanced: boolean;
@@ -124,6 +127,8 @@ export const ModelFormFields: FC<{
displayNameField,
setDefaultDisabled,
modelConfigFormBuildResult,
showCostEstimate,
setShowCostEstimate,
showProviderConfig,
setShowProviderConfig,
showAdvanced,
@@ -239,6 +244,19 @@ export const ModelFormFields: FC<{
</div>
<div className="overflow-hidden rounded-lg border border-solid border-border">
<CollapsibleSection
title="Cost estimate"
description="Estimated price per million tokens in USD. Prices are read-only."
open={showCostEstimate}
onOpenChange={setShowCostEstimate}
contentClassName="grid grid-cols-2 gap-3 pt-3 pl-6 sm:grid-cols-4"
>
<PricingEstimateFields
provider={selectedProviderType}
model={form.values.model}
/>
</CollapsibleSection>
{hasProviderConfigFields && (
<CollapsibleSection
title="Provider configuration"
@@ -1,6 +1,7 @@
import { type FormikContextType, getIn } from "formik";
import { InfoIcon } from "lucide-react";
import { type FC, Fragment, type ReactNode } from "react";
import { type FC, Fragment, type ReactNode, useId } from "react";
import { useQuery } from "react-query";
import {
type FieldSchema,
getVisibleGeneralFields,
@@ -9,6 +10,7 @@ import {
snakeToCamel,
toFormFieldKey,
} from "#/api/chatModelOptions";
import { aiModelPrices } from "#/api/queries/aiProviders";
import { Input } from "#/components/Input/Input";
import {
InputGroup,
@@ -23,6 +25,7 @@ import {
SelectTrigger,
SelectValue,
} from "#/components/Select/Select";
import { Skeleton } from "#/components/Skeleton/Skeleton";
import { Textarea } from "#/components/Textarea/Textarea";
import {
Tooltip,
@@ -30,7 +33,14 @@ import {
TooltipTrigger,
} from "#/components/Tooltip/Tooltip";
import { normalizeProvider } from "#/modules/aiModels/helpers";
import { useFeatureVisibility } from "#/modules/dashboard/useFeatureVisibility";
import { cn } from "#/utils/cn";
import { microsToDollars } from "#/utils/currency";
import {
findKnownModelByCanonicalId,
findKnownModelByExactAlias,
formatPricePerMillionTokens,
} from "./knownModels";
import {
isFieldConflictDisabled,
isVisibleWhenSatisfied,
@@ -647,3 +657,113 @@ export const GeneralModelConfigFields: FC<ModelConfigFieldsProps> = ({
</>
);
};
type ModelCosts = {
inputCost?: number;
outputCost?: number;
cacheReadCost?: number;
cacheWriteCost?: number;
};
const priceEstimateFields: ReadonlyArray<[string, keyof ModelCosts]> = [
["Input", "inputCost"],
["Output", "outputCost"],
["Cache read", "cacheReadCost"],
["Cache write", "cacheWriteCost"],
];
const priceOrUndefined = (micros: number | null): number | undefined =>
micros === null ? undefined : microsToDollars(micros);
export const PricingEstimateFields: FC<{
provider: string;
model: string;
}> = ({ provider, model }) => {
const fieldIdPrefix = useId();
const aibridgeEntitled = Boolean(useFeatureVisibility().aibridge);
const normalizedProvider = normalizeProvider(provider);
const trimmedModel = model.trim();
const livePricesQuery = useQuery({
...aiModelPrices(normalizedProvider, trimmedModel),
enabled:
aibridgeEntitled && normalizedProvider !== "" && trimmedModel !== "",
});
const livePriceLoading =
livePricesQuery.fetchStatus !== "idle" && !livePricesQuery.isSuccess;
const livePrice = livePricesQuery.data?.[0];
const knownModel =
findKnownModelByCanonicalId(normalizedProvider, trimmedModel) ??
findKnownModelByExactAlias(normalizedProvider, trimmedModel);
// A price book row is the deployment's own pricing for the model, so it
// wins outright. A null field on that row means the category is unpriced
// (the cost engine bills it as zero), not that the catalog should fill it
// in. The catalog is only the fallback when the model has no row at all.
const costs: ModelCosts | undefined = livePrice
? {
inputCost: priceOrUndefined(livePrice.input_price),
outputCost: priceOrUndefined(livePrice.output_price),
cacheReadCost: priceOrUndefined(livePrice.cache_read_price),
cacheWriteCost: priceOrUndefined(livePrice.cache_write_price),
}
: knownModel;
if (livePricesQuery.isError) {
return (
<p className="m-0 flex items-center gap-1.5 text-xs text-content-secondary sm:col-span-full">
<InfoIcon className="size-3.5 shrink-0" />
Couldn't load pricing.
</p>
);
}
if (
!livePriceLoading &&
(costs === undefined ||
priceEstimateFields.every(([, key]) => costs[key] === undefined))
) {
return (
<p className="m-0 text-xs text-content-secondary sm:col-span-full">
No pricing data for this model.
</p>
);
}
return (
<>
{priceEstimateFields.map(([label, key]) => {
const cost = costs?.[key];
const fieldId = `${fieldIdPrefix}-${label.toLowerCase().replace(/\s+/g, "-")}`;
const displayValue =
cost === undefined ? "" : formatPricePerMillionTokens(cost).slice(1);
return (
<div key={label} className="flex min-w-0 flex-col gap-1.5">
<FieldLabel htmlFor={fieldId} label={label} />
<InputGroup className="cursor-not-allowed bg-surface-secondary">
<InputGroupAddon align="inline-start">$</InputGroupAddon>
{livePriceLoading ? (
<Skeleton
aria-label={`${label} price loading`}
className="mx-3 h-2 w-2/5 flex-1 rounded-full"
/>
) : (
<InputGroupInput
id={fieldId}
className="min-w-0 cursor-not-allowed text-content-secondary"
value={displayValue}
readOnly
/>
)}
<InputGroupAddon align="inline-end">
<span className="text-xs text-content-disabled">
USD/1M tokens
</span>
</InputGroupAddon>
</InputGroup>
</div>
);
})}
</>
);
};
@@ -3,6 +3,7 @@ import {
findKnownModelByCanonicalId,
findKnownModelByExactAlias,
formatContextBadge,
formatPricePerMillionTokens,
getKnownModelsForProvider,
searchKnownModels,
} from "./index";
@@ -123,6 +124,42 @@ describe("findKnownModelByExactAlias", () => {
});
});
describe("formatPricePerMillionTokens", () => {
it("formats whole-dollar prices", () => {
expect(formatPricePerMillionTokens(10)).toBe("$10");
});
it("formats fractional prices without dropping precision", () => {
expect(formatPricePerMillionTokens(1.25)).toBe("$1.25");
expect(formatPricePerMillionTokens(0.1)).toBe("$0.10");
expect(formatPricePerMillionTokens(0.3)).toBe("$0.30");
});
it("keeps sub-cent prices visible", () => {
expect(formatPricePerMillionTokens(0.075)).toBe("$0.075");
expect(formatPricePerMillionTokens(0.003625)).toBe("$0.0036");
expect(formatPricePerMillionTokens(0.125)).toBe("$0.125");
});
it("shows a threshold for positive prices below four decimals", () => {
expect(formatPricePerMillionTokens(0.000001)).toBe("<$0.0001");
expect(formatPricePerMillionTokens(0.000049)).toBe("<$0.0001");
expect(formatPricePerMillionTokens(0.0001)).toBe("$0.0001");
});
it("formats zero", () => {
expect(formatPricePerMillionTokens(0)).toBe("$0");
});
it("rejects non-finite values", () => {
for (const invalidValue of [Number.NaN, Number.POSITIVE_INFINITY]) {
expect(() => formatPricePerMillionTokens(invalidValue)).toThrow(
"price must be a finite number",
);
}
});
});
describe("findKnownModelByCanonicalId", () => {
it("returns exact canonical lookup", () => {
expect(findKnownModelByCanonicalId("openai", "gpt-5.5")?.displayName).toBe(
@@ -100,3 +100,22 @@ export const formatContextBadge = (contextLimit: number): string => {
}
return `${formatCompactNumber(contextLimit / 1_000_000)}M context`;
};
export const formatPricePerMillionTokens = (value: number): string => {
if (!Number.isFinite(value)) {
throw new Error("price must be a finite number");
}
if (Number.isInteger(value)) {
return `$${value}`;
}
// A positive price too small to show at four decimals would otherwise
// render as $0.00 and read as free, so show it as a threshold instead.
if (value > 0 && value < 0.0001) {
return "<$0.0001";
}
// Keep two decimals so cents read as cents ($0.10, not $0.1). Keep up to
// four so sub-cent prices stay visible ($0.075, $0.0036).
const [whole, decimals = ""] = value.toFixed(4).split(".");
const trimmed = decimals.slice(0, 2) + decimals.slice(2).replace(/0+$/, "");
return `$${whole}.${trimmed}`;
};