mirror of
https://github.com/coder/coder.git
synced 2026-09-24 15:04:27 +08:00
## Stack Context
This stack removes native chat cost tracking and native chat usage
limits, making the AI Gateway the single source of AI spend data and
budget enforcement.
1. **This PR:** re-back the per-chat cost endpoint with AI Gateway data.
2. Remove native chat usage limits end to end, rewiring the sidebar
indicator to gateway spend.
3. Remove native chat cost tracking end to end, deleting the
Analytics/Spend cost UI.
## What?
`GET /api/experimental/chats/{chat}/cost` summed
`chat_messages.total_cost_micros`, which native chat cost tracking
maintained. It now aggregates AI Gateway interception data instead, and
has no native fallback.
- New `GetAIBridgeChatCost` query, authorized through the root chat so
members can read their own chat's cost without gaining access to raw
interception rows.
- Response fields renamed: `priced_message_count` -> `request_count`,
`unpriced_messages_having_usage_count` -> `unpriced_request_count`.
- The chat summary sidebar keys its cost cache by root chat, and hides
the cost row where the AI Gateway is off or unlicensed. The root cost is
invalidated when a chat leaves an active status and when a generated
title lands, since title generation bills its own gateway request.
`GetChatModelUsageCostByChatID` and the rest of native cost tracking are
untouched here; PR 3 removes them.
## Why?
Native cost tracking duplicates what the AI Gateway already records, and
the two disagree. Repointing the endpoint first means the cost UI keeps
working while the native implementation is deleted later in the stack.
Two behaviour changes follow from gateway semantics and are intentional:
- **Requests, not messages.** The gateway records interceptions, so
counts are requests. Title-generation traffic now counts.
- **Whole-tree totals.** The gateway records the *spawning* chat's ID as
the interception session ID, so a subagent's requests are attributed to
its immediate parent, not always the root. Only a whole chat tree can be
summed, so the query resolves the root and aggregates the tree, and
every chat in a tree reports the same total. Native returned per-subtree
totals.
## Attribution and counting semantics
The aggregate groups token usage per interception before counting, so
the reported numbers are per request even though a request records one
usage row per provider response:
- `RequestCount` counts finished `Coder Agents` interceptions in the
tree, including unpriced ones.
- `UnpricedRequestCount` counts requests with at least one usage row the
gateway could not price. It is a subset of `RequestCount`.
- `TotalCostMicros` omits only unpriced usage, so a partially priced
request still contributes its priced portion. The sidebar therefore says
`Excludes unpriced usage from N request(s)` rather than claiming whole
requests were dropped.
A recorded cost of zero is a free request, not an unpriced one. Usage
without an effective group is excluded, matching what never reached
`ai_user_daily_spend`.
## Authorization
Reads go through `ExtractChatParam` plus `ResourceChat`, with no
cost-specific RBAC widening. `TestGetChatCost/MemberCanReadOwnChat`
covers a scoped `agents-access` member reading their own chat's cost,
and `MemberCannotReadOtherUsersChat` still asserts 404 for a non-owner.
Plain members without `agents-access` cannot create or read chats at
all, so they never reach this endpoint.
## Known limitation
AI Gateway data has its own retention period, 60 days by default and
configured independently of chat retention, so spend for requests older
than that is no longer reported. A chat whose gateway records have all
been purged reports zero cost, which is indistinguishable from genuinely
free usage under this contract. The endpoint documents the caveat;
#27330 documents it on the Spend Management page.
In-flight interceptions are excluded, since cost is only known once the
response is recorded. A chat's cost therefore lags the active turn by
one request.
## Rebase note
Rebased onto `main` after #27579 removed the `ai-gateway-cost-control`
experiment. The per-chat cost row is now gated on the `aibridge` feature
alone, matching how #27579 degated the other cost-control surfaces.
> Mux prepared this PR on Mike's behalf.
410 lines
13 KiB
Go
410 lines
13 KiB
Go
package coderd
|
|
|
|
import (
|
|
"database/sql"
|
|
"encoding/json"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"testing"
|
|
|
|
"github.com/go-chi/chi/v5"
|
|
"github.com/google/uuid"
|
|
"github.com/stretchr/testify/require"
|
|
"go.uber.org/mock/gomock"
|
|
"golang.org/x/xerrors"
|
|
|
|
"cdr.dev/slog/v3"
|
|
"cdr.dev/slog/v3/sloggers/slogtest"
|
|
"github.com/coder/coder/v2/coderd/database"
|
|
"github.com/coder/coder/v2/coderd/database/dbauthz"
|
|
"github.com/coder/coder/v2/coderd/database/dbmock"
|
|
"github.com/coder/coder/v2/coderd/httpmw"
|
|
"github.com/coder/coder/v2/codersdk"
|
|
"github.com/coder/coder/v2/testutil"
|
|
)
|
|
|
|
// ExtractChatParam authorizes the read, then GetAIBridgeChatCost authorizes it
|
|
// again. A denial on the second check means the ACL changed in between (a
|
|
// read-authz race). Assert it surfaces as 404, not 500.
|
|
func TestGetChatCostSurfacesReadAuthzRace(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
ctrl := gomock.NewController(t)
|
|
dbm := dbmock.NewMockStore(ctrl)
|
|
chat := database.Chat{
|
|
ID: uuid.New(),
|
|
OrganizationID: uuid.New(),
|
|
OwnerID: uuid.New(),
|
|
}
|
|
|
|
dbm.EXPECT().GetChatByID(gomock.Any(), chat.ID).Return(chat, nil)
|
|
dbm.EXPECT().GetAIBridgeChatCost(gomock.Any(), chat.ID).Return(
|
|
database.GetAIBridgeChatCostRow{},
|
|
dbauthz.NotAuthorizedError{Err: sql.ErrNoRows},
|
|
)
|
|
|
|
api := &API{Options: &Options{Database: dbm}}
|
|
rtr := chi.NewRouter()
|
|
rtr.With(httpmw.ExtractChatParam(dbm)).Get("/chats/{chat}/cost", api.getChatCost)
|
|
|
|
req := httptest.NewRequest(http.MethodGet, "/chats/"+chat.ID.String()+"/cost", nil)
|
|
rec := httptest.NewRecorder()
|
|
rtr.ServeHTTP(rec, req)
|
|
resp := rec.Result()
|
|
defer resp.Body.Close()
|
|
|
|
require.Equal(t, http.StatusNotFound, resp.StatusCode)
|
|
}
|
|
|
|
// AI Gateway attributes a subagent's requests to the chat that spawned it, so
|
|
// a subagent request must be answered with its root chat's tree cost.
|
|
func TestGetChatCostQueriesRootChat(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
ctrl := gomock.NewController(t)
|
|
dbm := dbmock.NewMockStore(ctrl)
|
|
rootID := uuid.New()
|
|
child := database.Chat{
|
|
ID: uuid.New(),
|
|
OrganizationID: uuid.New(),
|
|
OwnerID: uuid.New(),
|
|
ParentChatID: uuid.NullUUID{UUID: rootID, Valid: true},
|
|
RootChatID: uuid.NullUUID{UUID: rootID, Valid: true},
|
|
}
|
|
|
|
dbm.EXPECT().GetChatByID(gomock.Any(), child.ID).Return(child, nil)
|
|
dbm.EXPECT().GetAIBridgeChatCost(gomock.Any(), rootID).Return(
|
|
database.GetAIBridgeChatCostRow{
|
|
TotalCostMicros: 250,
|
|
RequestCount: 2,
|
|
UnpricedRequestCount: 1,
|
|
},
|
|
nil,
|
|
)
|
|
|
|
api := &API{Options: &Options{Database: dbm}}
|
|
rtr := chi.NewRouter()
|
|
rtr.With(httpmw.ExtractChatParam(dbm)).Get("/chats/{chat}/cost", api.getChatCost)
|
|
|
|
req := httptest.NewRequest(http.MethodGet, "/chats/"+child.ID.String()+"/cost", nil)
|
|
rec := httptest.NewRecorder()
|
|
rtr.ServeHTTP(rec, req)
|
|
resp := rec.Result()
|
|
defer resp.Body.Close()
|
|
|
|
require.Equal(t, http.StatusOK, resp.StatusCode)
|
|
var cost codersdk.ChatCost
|
|
require.NoError(t, json.NewDecoder(resp.Body).Decode(&cost))
|
|
require.Equal(t, child.ID, cost.ChatID)
|
|
require.Equal(t, int64(250), cost.TotalCostMicros)
|
|
require.Equal(t, int64(2), cost.RequestCount)
|
|
require.Equal(t, int64(1), cost.UnpricedRequestCount)
|
|
}
|
|
|
|
func TestGetChatCostFallsBackToParentChat(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
dbm := dbmock.NewMockStore(gomock.NewController(t))
|
|
parentID := uuid.New()
|
|
// chats.parent_chat_id and chats.root_chat_id are both ON DELETE SET NULL,
|
|
// so deleting a root leaves descendants with only a parent.
|
|
child := database.Chat{
|
|
ID: uuid.New(),
|
|
OwnerID: uuid.New(),
|
|
ParentChatID: uuid.NullUUID{UUID: parentID, Valid: true},
|
|
}
|
|
|
|
dbm.EXPECT().GetChatByID(gomock.Any(), child.ID).Return(child, nil)
|
|
dbm.EXPECT().GetAIBridgeChatCost(gomock.Any(), parentID).Return(
|
|
database.GetAIBridgeChatCostRow{TotalCostMicros: 125, RequestCount: 1},
|
|
nil,
|
|
)
|
|
|
|
api := &API{Options: &Options{Database: dbm}}
|
|
rtr := chi.NewRouter()
|
|
rtr.With(httpmw.ExtractChatParam(dbm)).Get("/chats/{chat}/cost", api.getChatCost)
|
|
|
|
req := httptest.NewRequest(http.MethodGet, "/chats/"+child.ID.String()+"/cost", nil)
|
|
rec := httptest.NewRecorder()
|
|
rtr.ServeHTTP(rec, req)
|
|
resp := rec.Result()
|
|
defer resp.Body.Close()
|
|
|
|
require.Equal(t, http.StatusOK, resp.StatusCode)
|
|
var cost codersdk.ChatCost
|
|
require.NoError(t, json.NewDecoder(resp.Body).Decode(&cost))
|
|
require.Equal(t, int64(125), cost.TotalCostMicros)
|
|
}
|
|
|
|
func TestEnrichMissingChatAgentIDs(t *testing.T) {
|
|
t.Parallel()
|
|
newAPI := func(t *testing.T) (*API, *dbmock.MockStore) {
|
|
t.Helper()
|
|
mDB := dbmock.NewMockStore(gomock.NewController(t))
|
|
logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}).Leveled(slog.LevelDebug)
|
|
return &API{Options: &Options{Database: mDB, Logger: logger}}, mDB
|
|
}
|
|
workspaceID, otherWorkspaceID := uuid.New(), uuid.New()
|
|
rootAgentID, otherAgentID := uuid.New(), uuid.New()
|
|
row := func(workspaceID, id uuid.UUID, parentID uuid.NullUUID, name string) database.GetWorkspaceAgentsInLatestBuildByWorkspaceIDsRow {
|
|
return database.GetWorkspaceAgentsInLatestBuildByWorkspaceIDsRow{
|
|
WorkspaceID: workspaceID,
|
|
WorkspaceAgent: database.WorkspaceAgent{
|
|
ID: id,
|
|
ParentID: parentID,
|
|
Name: name,
|
|
},
|
|
}
|
|
}
|
|
t.Run("batch selection and shared workspace", func(t *testing.T) {
|
|
t.Parallel()
|
|
api, mDB := newAPI(t)
|
|
mDB.EXPECT().GetWorkspaceAgentsInLatestBuildByWorkspaceIDs(gomock.Any(), gomock.Any()).DoAndReturn(func(_ any, ids []uuid.UUID) ([]database.GetWorkspaceAgentsInLatestBuildByWorkspaceIDsRow, error) {
|
|
require.ElementsMatch(t, []uuid.UUID{workspaceID, otherWorkspaceID}, ids)
|
|
return []database.GetWorkspaceAgentsInLatestBuildByWorkspaceIDsRow{
|
|
row(workspaceID, uuid.New(), uuid.NullUUID{UUID: rootAgentID, Valid: true}, "sub"), row(workspaceID, rootAgentID, uuid.NullUUID{}, "root"), row(otherWorkspaceID, otherAgentID, uuid.NullUUID{}, "root"),
|
|
}, nil
|
|
}).Times(1)
|
|
chats := []codersdk.Chat{{WorkspaceID: &workspaceID, Children: []codersdk.Chat{{WorkspaceID: &workspaceID}}}, {WorkspaceID: &otherWorkspaceID}}
|
|
api.enrichChatWithWorkspaceAgentIDs(testutil.Context(t, testutil.WaitShort), chats)
|
|
require.Equal(t, rootAgentID, *chats[0].AgentID)
|
|
require.Equal(t, rootAgentID, *chats[0].Children[0].AgentID)
|
|
require.Equal(t, otherAgentID, *chats[1].AgentID)
|
|
})
|
|
t.Run("query error", func(t *testing.T) {
|
|
t.Parallel()
|
|
api, mDB := newAPI(t)
|
|
mDB.EXPECT().GetWorkspaceAgentsInLatestBuildByWorkspaceIDs(gomock.Any(), gomock.Any()).Return(nil, xerrors.New("boom"))
|
|
chats := []codersdk.Chat{{WorkspaceID: &workspaceID}, {WorkspaceID: &otherWorkspaceID}}
|
|
api.enrichChatWithWorkspaceAgentIDs(testutil.Context(t, testutil.WaitShort), chats)
|
|
require.Nil(t, chats[0].AgentID)
|
|
require.Nil(t, chats[1].AgentID)
|
|
})
|
|
t.Run("selection error and skips bound or unbound", func(t *testing.T) {
|
|
t.Parallel()
|
|
api, mDB := newAPI(t)
|
|
mDB.EXPECT().GetWorkspaceAgentsInLatestBuildByWorkspaceIDs(gomock.Any(), []uuid.UUID{workspaceID}).Return([]database.GetWorkspaceAgentsInLatestBuildByWorkspaceIDsRow{row(workspaceID, uuid.New(), uuid.NullUUID{UUID: rootAgentID, Valid: true}, "sub")}, nil)
|
|
bound := otherAgentID
|
|
chats := []codersdk.Chat{{}, {WorkspaceID: &workspaceID}, {WorkspaceID: &workspaceID, AgentID: &bound}}
|
|
api.enrichChatWithWorkspaceAgentIDs(testutil.Context(t, testutil.WaitShort), chats)
|
|
require.Nil(t, chats[1].AgentID)
|
|
require.Equal(t, bound, *chats[2].AgentID)
|
|
})
|
|
}
|
|
|
|
func TestValidateChatModelProviderOptions_AnthropicThinkingDisplay(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tests := []struct {
|
|
name string
|
|
display string
|
|
wantErr string
|
|
}{
|
|
{name: "Summarized", display: "summarized"},
|
|
{name: "Omitted", display: " omitted "},
|
|
{name: "Empty", display: " "},
|
|
{
|
|
name: "Invalid",
|
|
display: "summrized",
|
|
wantErr: "provider_options.anthropic.thinking_display must be one of summarized, omitted",
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
display := tt.display
|
|
err := validateChatModelProviderOptions(&codersdk.ChatModelProviderOptions{
|
|
Anthropic: &codersdk.ChatModelAnthropicProviderOptions{
|
|
ThinkingDisplay: &display,
|
|
},
|
|
})
|
|
if tt.wantErr != "" {
|
|
require.EqualError(t, err, tt.wantErr)
|
|
return
|
|
}
|
|
require.NoError(t, err)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestValidateChatModelConfigProviderModel(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tests := []struct {
|
|
name string
|
|
model string
|
|
provider database.AIProvider
|
|
wantErr bool
|
|
wantDetail string
|
|
}{
|
|
{
|
|
name: "OpenRouterNameWithOpenAITypeAndSlashModel",
|
|
model: "anthropic/claude-opus-4.6",
|
|
provider: database.AIProvider{
|
|
Name: "openrouter",
|
|
Type: database.AIProviderTypeOpenai,
|
|
},
|
|
wantErr: true,
|
|
wantDetail: "Change the AI provider type to openrouter or openai-compat.",
|
|
},
|
|
{
|
|
name: "OpenRouterNameWithWhitespaceAndCase",
|
|
model: "anthropic/claude-opus-4.6",
|
|
provider: database.AIProvider{
|
|
Name: " OpenRouter ",
|
|
Type: database.AIProviderTypeOpenai,
|
|
},
|
|
wantErr: true,
|
|
wantDetail: "Change the AI provider type to openrouter or openai-compat.",
|
|
},
|
|
{
|
|
name: "OpenRouterHostWithOpenAITypeAndSlashModel",
|
|
model: "anthropic/claude-opus-4.6",
|
|
provider: database.AIProvider{
|
|
Name: "private-relay",
|
|
Type: database.AIProviderTypeOpenai,
|
|
BaseUrl: "https://openrouter.ai/api/v1",
|
|
},
|
|
wantErr: true,
|
|
wantDetail: "Change the AI provider type to openrouter or openai-compat.",
|
|
},
|
|
{
|
|
name: "OpenRouterHostWithPort",
|
|
model: "anthropic/claude-opus-4.6",
|
|
provider: database.AIProvider{
|
|
Name: "private-relay",
|
|
Type: database.AIProviderTypeOpenai,
|
|
BaseUrl: "https://openrouter.ai:443/api/v1",
|
|
},
|
|
wantErr: true,
|
|
wantDetail: "Change the AI provider type to openrouter or openai-compat.",
|
|
},
|
|
{
|
|
name: "OpenRouterSubdomainWithOpenAIType",
|
|
model: "anthropic/claude-opus-4.6",
|
|
provider: database.AIProvider{
|
|
Name: "private-relay",
|
|
Type: database.AIProviderTypeOpenai,
|
|
BaseUrl: "https://api.openrouter.ai/v1",
|
|
},
|
|
wantErr: true,
|
|
wantDetail: "Change the AI provider type to openrouter or openai-compat.",
|
|
},
|
|
{
|
|
name: "OpenRouterTypeAllowsSlashModel",
|
|
model: "anthropic/claude-opus-4.6",
|
|
provider: database.AIProvider{
|
|
Name: "openrouter",
|
|
Type: database.AIProviderTypeOpenrouter,
|
|
},
|
|
},
|
|
{
|
|
name: "OpenAICompatTypeAllowsSlashModel",
|
|
model: "anthropic/claude-opus-4.6",
|
|
provider: database.AIProvider{
|
|
Name: "openrouter",
|
|
Type: database.AIProviderTypeOpenaiCompat,
|
|
},
|
|
},
|
|
{
|
|
name: "PrivateOpenAIProxyAllowsSlashModel",
|
|
model: "anthropic/claude-opus-4.6",
|
|
provider: database.AIProvider{
|
|
Name: "private-relay",
|
|
Type: database.AIProviderTypeOpenai,
|
|
BaseUrl: "https://llm-relay.internal/v1",
|
|
},
|
|
},
|
|
{
|
|
name: "OpenRouterNameWithPlainModelAllowed",
|
|
model: "gpt-4.1",
|
|
provider: database.AIProvider{
|
|
Name: "openrouter",
|
|
Type: database.AIProviderTypeOpenai,
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
got := validateChatModelConfigProviderModel(tt.provider, tt.model)
|
|
if tt.wantErr {
|
|
require.NotNil(t, got)
|
|
require.Contains(t, got.Response.Detail, tt.wantDetail)
|
|
return
|
|
}
|
|
require.Nil(t, got)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestRewriteChatStartWorkspaceManualUpdateResponse(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tests := []struct {
|
|
name string
|
|
resp codersdk.Response
|
|
fallbackDetail string
|
|
wantDetail string
|
|
}{
|
|
{
|
|
name: "NoValidationsAndEmptyDetail",
|
|
resp: codersdk.Response{
|
|
Message: "missing required parameter",
|
|
},
|
|
fallbackDetail: "wrapped missing required parameter",
|
|
wantDetail: "missing required parameter",
|
|
},
|
|
{
|
|
name: "NoValidationsAndExistingDetail",
|
|
resp: codersdk.Response{
|
|
Message: "missing required parameter",
|
|
Detail: "region must be set before the workspace can start",
|
|
},
|
|
fallbackDetail: "wrapped missing required parameter",
|
|
wantDetail: "missing required parameter: region must be set before the workspace can start",
|
|
},
|
|
{
|
|
name: "ValidationsAndEmptyDetail",
|
|
resp: codersdk.Response{
|
|
Message: "missing required parameter",
|
|
Validations: []codersdk.ValidationError{{
|
|
Field: "region",
|
|
Detail: "region must be set before the workspace can start",
|
|
}},
|
|
},
|
|
fallbackDetail: "wrapped missing required parameter",
|
|
wantDetail: "wrapped missing required parameter",
|
|
},
|
|
{
|
|
name: "ValidationsAndExistingDetail",
|
|
resp: codersdk.Response{
|
|
Message: "missing required parameter",
|
|
Detail: "region must be set before the workspace can start",
|
|
Validations: []codersdk.ValidationError{{
|
|
Field: "region",
|
|
Detail: "region must be set before the workspace can start",
|
|
}},
|
|
},
|
|
fallbackDetail: "wrapped missing required parameter",
|
|
wantDetail: "region must be set before the workspace can start",
|
|
},
|
|
}
|
|
|
|
const retryInstructions = "Use read_template before retrying start_workspace."
|
|
for _, tt := range tests {
|
|
tt := tt
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
got := rewriteChatStartWorkspaceManualUpdateResponse(tt.resp, tt.fallbackDetail, retryInstructions)
|
|
require.Equal(t, retryInstructions, got.Message)
|
|
require.Equal(t, tt.wantDetail, got.Detail)
|
|
require.Equal(t, tt.resp.Validations, got.Validations)
|
|
})
|
|
}
|
|
}
|