mirror of
https://github.com/coder/coder.git
synced 2026-09-24 15:04:27 +08:00
fix: price AI usage by configured provider type (#27836)
## Problem AI Gateway records the aibridge provider on each interception, which is the upstream wire format and only ever `anthropic`, `openai`, or `copilot`. Prices are matched on exact provider and model equality, so a provider configured as Azure, Bedrock, Google, OpenRouter, or Vercel is priced as if it were native OpenAI or Anthropic, matching either the wrong price or no price at all. ## Changes - Resolve the configured provider type from `ai_providers` by provider name, which is unique among live providers, and key the price lookup on it instead of the aibridge provider. No schema change is needed. - Label `unpriced_token_usage_records_total` with the same provider value used for the lookup, so it names a provider an operator actually configured. - Treat a provider that cannot be resolved as unpriced, consistent with how a missing price is handled today. Closes https://linear.app/codercom/issue/AIGOV-570/resolve-ai-model-prices-using-the-configured-provider-type Depends on the follow-up that extends the shipped price book to the remaining provider types: https://linear.app/codercom/issue/AIGOV-571/ship-prices-for-all-ai-governance-provider-types > [!NOTE] > Initially generated by Claude Opus 5, modified and reviewed by @ssncferreira
This commit is contained in:
@@ -78,6 +78,7 @@ type store interface {
|
||||
// Cost-attribution queries, used to snapshot price and effective group on
|
||||
// each token usage record.
|
||||
GetAIBridgeInterceptionByID(ctx context.Context, id uuid.UUID) (database.AIBridgeInterception, error)
|
||||
GetAIProviderByName(ctx context.Context, name string) (database.AIProvider, error)
|
||||
GetAIModelPriceByProviderModel(ctx context.Context, arg database.GetAIModelPriceByProviderModelParams) (database.AIModelPrice, error)
|
||||
GetUserAIBudgetOverride(ctx context.Context, userID uuid.UUID) (database.UserAIBudgetOverride, error)
|
||||
GetHighestGroupAIBudgetByUser(ctx context.Context, userID uuid.UUID) (database.GetHighestGroupAIBudgetByUserRow, error)
|
||||
|
||||
@@ -1684,6 +1684,17 @@ func TestRecordTokenUsage(t *testing.T) {
|
||||
now = time.Date(2026, 6, 25, 14, 30, 0, 0, time.UTC)
|
||||
)
|
||||
|
||||
// Budget resolution falls through to the Everyone group, for cases that vary
|
||||
// only provider resolution.
|
||||
expectBudgetLookups := func(db *dbmock.MockStore, intc database.AIBridgeInterception) {
|
||||
db.EXPECT().GetAIBridgeInterceptionByID(gomock.Any(), intc.ID).Return(intc, nil)
|
||||
db.EXPECT().GetUserAIBudgetOverride(gomock.Any(), intc.InitiatorID).
|
||||
Return(database.UserAIBudgetOverride{}, sql.ErrNoRows)
|
||||
db.EXPECT().GetHighestGroupAIBudgetByUser(gomock.Any(), intc.InitiatorID).
|
||||
Return(database.GetHighestGroupAIBudgetByUserRow{}, sql.ErrNoRows)
|
||||
db.EXPECT().GetUserEveryoneFallbackGroup(gomock.Any(), intc.InitiatorID).Return(uuid.New(), nil)
|
||||
}
|
||||
|
||||
testRecordMethod(t,
|
||||
func(srv *aibridgedserver.Server, ctx context.Context, req *proto.RecordTokenUsageRequest) (*proto.RecordTokenUsageResponse, error) {
|
||||
return srv.RecordTokenUsage(ctx, req)
|
||||
@@ -2131,6 +2142,123 @@ func TestRecordTokenUsage(t *testing.T) {
|
||||
db.EXPECT().IncrementUserAIDailySpend(gomock.Any(), gomock.Any()).Times(0)
|
||||
},
|
||||
},
|
||||
{
|
||||
// An azure provider has the openai upstream wire format but bills at
|
||||
// its own rates.
|
||||
name: "openai wire format priced as azure",
|
||||
request: &proto.RecordTokenUsageRequest{
|
||||
InterceptionId: uuid.NewString(),
|
||||
MsgId: "msg_123",
|
||||
InputTokens: 100,
|
||||
CreatedAt: timestamppb.Now(),
|
||||
},
|
||||
setupMocks: func(t *testing.T, db *dbmock.MockStore, req *proto.RecordTokenUsageRequest) {
|
||||
interceptionID, err := uuid.Parse(req.GetInterceptionId())
|
||||
assert.NoError(t, err, "parse interception UUID")
|
||||
|
||||
intc := newTestInterception(interceptionID)
|
||||
intc.Provider = "openai"
|
||||
intc.ProviderName = "azure-prod"
|
||||
intc.Model = "gpt-5-mini"
|
||||
expectBudgetLookups(db, intc)
|
||||
|
||||
db.EXPECT().GetAIProviderByName(gomock.Any(), intc.ProviderName).
|
||||
Return(database.AIProvider{Name: intc.ProviderName, Type: database.AIProviderTypeAzure}, nil)
|
||||
db.EXPECT().GetAIModelPriceByProviderModel(gomock.Any(), database.GetAIModelPriceByProviderModelParams{
|
||||
Provider: string(database.AIProviderTypeAzure),
|
||||
Model: intc.Model,
|
||||
}).Return(database.AIModelPrice{
|
||||
Provider: string(database.AIProviderTypeAzure),
|
||||
Model: intc.Model,
|
||||
InputPrice: sql.NullInt64{Int64: 3_000_000, Valid: true},
|
||||
}, nil)
|
||||
|
||||
db.EXPECT().InTx(gomock.Any(), nil).DoAndReturn(
|
||||
func(fn func(database.Store) error, _ *database.TxOptions) error { return fn(db) },
|
||||
)
|
||||
db.EXPECT().InsertAIBridgeTokenUsage(gomock.Any(), gomock.Cond(func(p database.InsertAIBridgeTokenUsageParams) bool {
|
||||
return assert.Equal(t, sql.NullInt64{Int64: 3_000_000, Valid: true}, p.InputPriceMicros, "input price") &&
|
||||
assert.Equal(t, sql.NullInt64{Int64: 300, Valid: true}, p.CostMicros, "cost")
|
||||
})).Return(database.AIBridgeTokenUsage{ID: uuid.New(), InterceptionID: interceptionID}, nil)
|
||||
db.EXPECT().IncrementUserAIDailySpend(gomock.Any(), gomock.Any()).
|
||||
Return(database.AIUserDailySpend{}, nil)
|
||||
},
|
||||
},
|
||||
{
|
||||
// A bedrock provider has the anthropic upstream wire format but bills
|
||||
// at its own rates.
|
||||
name: "anthropic wire format priced as bedrock",
|
||||
request: &proto.RecordTokenUsageRequest{
|
||||
InterceptionId: uuid.NewString(),
|
||||
MsgId: "msg_123",
|
||||
InputTokens: 100,
|
||||
CreatedAt: timestamppb.Now(),
|
||||
},
|
||||
setupMocks: func(t *testing.T, db *dbmock.MockStore, req *proto.RecordTokenUsageRequest) {
|
||||
interceptionID, err := uuid.Parse(req.GetInterceptionId())
|
||||
assert.NoError(t, err, "parse interception UUID")
|
||||
|
||||
intc := newTestInterception(interceptionID)
|
||||
intc.ProviderName = "bedrock-eu"
|
||||
expectBudgetLookups(db, intc)
|
||||
|
||||
db.EXPECT().GetAIProviderByName(gomock.Any(), intc.ProviderName).
|
||||
Return(database.AIProvider{Name: intc.ProviderName, Type: database.AIProviderTypeBedrock}, nil)
|
||||
db.EXPECT().GetAIModelPriceByProviderModel(gomock.Any(), database.GetAIModelPriceByProviderModelParams{
|
||||
Provider: string(database.AIProviderTypeBedrock),
|
||||
Model: intc.Model,
|
||||
}).Return(database.AIModelPrice{
|
||||
Provider: string(database.AIProviderTypeBedrock),
|
||||
Model: intc.Model,
|
||||
InputPrice: sql.NullInt64{Int64: 3_000_000, Valid: true},
|
||||
}, nil)
|
||||
|
||||
db.EXPECT().InTx(gomock.Any(), nil).DoAndReturn(
|
||||
func(fn func(database.Store) error, _ *database.TxOptions) error { return fn(db) },
|
||||
)
|
||||
db.EXPECT().InsertAIBridgeTokenUsage(gomock.Any(), gomock.Cond(func(p database.InsertAIBridgeTokenUsageParams) bool {
|
||||
return assert.Equal(t, sql.NullInt64{Int64: 3_000_000, Valid: true}, p.InputPriceMicros, "input price") &&
|
||||
assert.Equal(t, sql.NullInt64{Int64: 300, Valid: true}, p.CostMicros, "cost")
|
||||
})).Return(database.AIBridgeTokenUsage{ID: uuid.New(), InterceptionID: interceptionID}, nil)
|
||||
db.EXPECT().IncrementUserAIDailySpend(gomock.Any(), gomock.Any()).
|
||||
Return(database.AIUserDailySpend{}, nil)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "unresolved provider is unpriced",
|
||||
request: &proto.RecordTokenUsageRequest{
|
||||
InterceptionId: uuid.NewString(),
|
||||
MsgId: "msg_123",
|
||||
InputTokens: 100,
|
||||
CreatedAt: timestamppb.Now(),
|
||||
},
|
||||
setupMocks: func(t *testing.T, db *dbmock.MockStore, req *proto.RecordTokenUsageRequest) {
|
||||
interceptionID, err := uuid.Parse(req.GetInterceptionId())
|
||||
assert.NoError(t, err, "parse interception UUID")
|
||||
|
||||
intc := newTestInterception(interceptionID)
|
||||
expectBudgetLookups(db, intc)
|
||||
|
||||
db.EXPECT().GetAIProviderByName(gomock.Any(), intc.ProviderName).
|
||||
Return(database.AIProvider{}, sql.ErrNoRows)
|
||||
// Without a provider there is nothing to key the price on.
|
||||
db.EXPECT().GetAIModelPriceByProviderModel(gomock.Any(), gomock.Any()).Times(0)
|
||||
|
||||
db.EXPECT().InTx(gomock.Any(), nil).DoAndReturn(
|
||||
func(fn func(database.Store) error, _ *database.TxOptions) error { return fn(db) },
|
||||
)
|
||||
db.EXPECT().InsertAIBridgeTokenUsage(gomock.Any(), gomock.Cond(func(p database.InsertAIBridgeTokenUsageParams) bool {
|
||||
return assert.False(t, p.InputPriceMicros.Valid, "input price null") &&
|
||||
assert.False(t, p.CostMicros.Valid, "cost null")
|
||||
})).Return(database.AIBridgeTokenUsage{ID: uuid.New(), InterceptionID: interceptionID}, nil)
|
||||
db.EXPECT().IncrementUserAIDailySpend(gomock.Any(), gomock.Any()).Times(0)
|
||||
},
|
||||
// The metric names the provider that failed to resolve.
|
||||
assertMetrics: func(t *testing.T, reg *prometheus.Registry) {
|
||||
require.Equal(t, 1, promhelp.CounterValue(t, reg, "cost_control_unpriced_token_usage_records_total",
|
||||
prometheus.Labels{"provider": "anthropic-eu", "model": "claude-sonnet-4-6"}))
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "invalid interception ID",
|
||||
request: &proto.RecordTokenUsageRequest{
|
||||
@@ -2162,6 +2290,27 @@ func TestRecordTokenUsage(t *testing.T) {
|
||||
},
|
||||
expectedErr: "get interception",
|
||||
},
|
||||
{
|
||||
name: "provider lookup error",
|
||||
request: &proto.RecordTokenUsageRequest{
|
||||
InterceptionId: uuid.NewString(),
|
||||
MsgId: "msg_123",
|
||||
InputTokens: 100,
|
||||
CreatedAt: timestamppb.Now(),
|
||||
},
|
||||
setupMocks: func(t *testing.T, db *dbmock.MockStore, req *proto.RecordTokenUsageRequest) {
|
||||
interceptionID, err := uuid.Parse(req.GetInterceptionId())
|
||||
assert.NoError(t, err, "parse interception UUID")
|
||||
|
||||
// An unexpected provider lookup error (not sql.ErrNoRows) fails
|
||||
// the record.
|
||||
intc := newTestInterception(interceptionID)
|
||||
expectBudgetLookups(db, intc)
|
||||
db.EXPECT().GetAIProviderByName(gomock.Any(), intc.ProviderName).
|
||||
Return(database.AIProvider{}, sql.ErrConnDone)
|
||||
},
|
||||
expectedErr: "get configured provider",
|
||||
},
|
||||
{
|
||||
name: "price lookup error",
|
||||
request: &proto.RecordTokenUsageRequest{
|
||||
@@ -2185,6 +2334,8 @@ func TestRecordTokenUsage(t *testing.T) {
|
||||
Return(database.GetHighestGroupAIBudgetByUserRow{}, sql.ErrNoRows)
|
||||
db.EXPECT().GetUserEveryoneFallbackGroup(gomock.Any(), intc.InitiatorID).
|
||||
Return(uuid.New(), nil)
|
||||
db.EXPECT().GetAIProviderByName(gomock.Any(), intc.ProviderName).
|
||||
Return(database.AIProvider{Name: intc.ProviderName, Type: database.AIProviderTypeAnthropic}, nil)
|
||||
db.EXPECT().GetAIModelPriceByProviderModel(gomock.Any(), gomock.Any()).
|
||||
Return(database.AIModelPrice{}, sql.ErrConnDone)
|
||||
},
|
||||
@@ -2287,10 +2438,18 @@ func TestRecordTokenUsageAuthorized(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, rawDB.UpsertAIModelPrices(ctx, priceSeed), "seed model prices")
|
||||
|
||||
// The interception's provider name resolves to this provider, whose type keys
|
||||
// the price lookup.
|
||||
aiProvider := dbgen.AIProvider(t, rawDB, database.AIProvider{
|
||||
Name: "anthropic-eu",
|
||||
Type: database.AIProviderTypeAnthropic,
|
||||
})
|
||||
|
||||
intc := dbgen.AIBridgeInterception(t, rawDB, database.InsertAIBridgeInterceptionParams{
|
||||
InitiatorID: user.ID,
|
||||
Provider: provider,
|
||||
Model: model,
|
||||
InitiatorID: user.ID,
|
||||
Provider: provider,
|
||||
ProviderName: aiProvider.Name,
|
||||
Model: model,
|
||||
}, nil)
|
||||
|
||||
// Use fixed dates to keep the test deterministic.
|
||||
@@ -2348,6 +2507,152 @@ func TestRecordTokenUsageAuthorized(t *testing.T) {
|
||||
require.Equal(t, wantCost, spend.SpendMicros, "spend micros")
|
||||
}
|
||||
|
||||
// TestRecordTokenUsageProviderResolution covers provider resolution against a real
|
||||
// database through dbauthz, where the live-row filter and name reuse apply.
|
||||
func TestRecordTokenUsageProviderResolution(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
const claudeModel, gptModel = "claude-sonnet-4-6", "gpt-5-mini"
|
||||
const anthropicInputPrice, bedrockInputPrice, openaiInputPrice, azureInputPrice int64 = 2_000_000, 3_000_000, 4_000_000, 5_000_000
|
||||
|
||||
setupCtx := testutil.Context(t, testutil.WaitLong)
|
||||
logger := testutil.Logger(t)
|
||||
|
||||
rawDB, _ := dbtestutil.NewDB(t)
|
||||
authzDB := dbauthz.New(rawDB, rbac.NewStrictAuthorizer(prometheus.NewRegistry()), logger, coderdtest.AccessControlStorePointer())
|
||||
|
||||
user := dbgen.User(t, rawDB, database.User{})
|
||||
|
||||
// Prices differ per provider type so the asserted cost identifies which type resolved.
|
||||
priceSeed, err := json.Marshal([]map[string]any{
|
||||
{"provider": string(database.AIProviderTypeAnthropic), "model": claudeModel, "input_price": anthropicInputPrice},
|
||||
{"provider": string(database.AIProviderTypeBedrock), "model": claudeModel, "input_price": bedrockInputPrice},
|
||||
{"provider": string(database.AIProviderTypeOpenai), "model": gptModel, "input_price": openaiInputPrice},
|
||||
{"provider": string(database.AIProviderTypeAzure), "model": gptModel, "input_price": azureInputPrice},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, rawDB.UpsertAIModelPrices(setupCtx, priceSeed), "seed model prices")
|
||||
|
||||
srv, err := aibridgedserver.NewServer(setupCtx, aibridgedserver.Options{
|
||||
Store: authzDB,
|
||||
AISeatTracker: agplaiseats.Noop{},
|
||||
AccessURL: "/",
|
||||
GatewayCfg: codersdk.AIBridgeConfig{},
|
||||
Experiments: requiredExperiments,
|
||||
Logger: logger,
|
||||
Clock: quartz.NewReal(),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
// wireProvider is the upstream wire format recorded on the interception.
|
||||
wireProvider string
|
||||
// providerName is the provider instance name recorded on the interception.
|
||||
providerName string
|
||||
// providerType is the configured provider type of the live provider.
|
||||
providerType database.AIProviderType
|
||||
model string
|
||||
// setupProvider creates and deletes the case's providers.
|
||||
setupProvider func(t *testing.T, ctx context.Context, providerName string, providerType database.AIProviderType)
|
||||
wantInputPrice sql.NullInt64
|
||||
wantCost sql.NullInt64
|
||||
}{
|
||||
{
|
||||
// The common configuration, where the configured provider type matches
|
||||
// the upstream wire format.
|
||||
name: "provider named after its type",
|
||||
wireProvider: "anthropic",
|
||||
providerName: "anthropic",
|
||||
providerType: database.AIProviderTypeAnthropic,
|
||||
model: claudeModel,
|
||||
// One live anthropic provider.
|
||||
setupProvider: func(t *testing.T, _ context.Context, providerName string, providerType database.AIProviderType) {
|
||||
dbgen.AIProvider(t, rawDB, database.AIProvider{Name: providerName, Type: providerType})
|
||||
},
|
||||
wantInputPrice: sql.NullInt64{Int64: anthropicInputPrice, Valid: true},
|
||||
// 100 input tokens at the anthropic input price: $0.0002.
|
||||
wantCost: sql.NullInt64{Int64: 200, Valid: true},
|
||||
},
|
||||
{
|
||||
name: "priced by configured provider type",
|
||||
wireProvider: "anthropic",
|
||||
providerName: "bedrock-eu",
|
||||
providerType: database.AIProviderTypeBedrock,
|
||||
model: claudeModel,
|
||||
// One live bedrock provider.
|
||||
setupProvider: func(t *testing.T, _ context.Context, providerName string, providerType database.AIProviderType) {
|
||||
dbgen.AIProvider(t, rawDB, database.AIProvider{Name: providerName, Type: providerType})
|
||||
},
|
||||
wantInputPrice: sql.NullInt64{Int64: bedrockInputPrice, Valid: true},
|
||||
// 100 input tokens at the bedrock input price: $0.0003.
|
||||
wantCost: sql.NullInt64{Int64: 300, Valid: true},
|
||||
},
|
||||
{
|
||||
name: "deleted provider is unpriced",
|
||||
wireProvider: "anthropic",
|
||||
providerName: "bedrock-deleted",
|
||||
providerType: database.AIProviderTypeBedrock,
|
||||
model: claudeModel,
|
||||
// One bedrock provider, deleted before the usage is recorded.
|
||||
setupProvider: func(t *testing.T, ctx context.Context, providerName string, providerType database.AIProviderType) {
|
||||
provider := dbgen.AIProvider(t, rawDB, database.AIProvider{Name: providerName, Type: providerType})
|
||||
require.NoError(t, rawDB.DeleteAIProviderByID(ctx, provider.ID), "delete provider")
|
||||
},
|
||||
wantInputPrice: sql.NullInt64{Valid: false},
|
||||
wantCost: sql.NullInt64{Valid: false},
|
||||
},
|
||||
{
|
||||
// Names are unique only among live providers, so a deleted name can be
|
||||
// reused by a provider of a different configured provider type.
|
||||
name: "reused name resolves to the live provider",
|
||||
wireProvider: "openai",
|
||||
providerName: "reused-name",
|
||||
providerType: database.AIProviderTypeAzure,
|
||||
model: gptModel,
|
||||
// A deleted openai provider and a live azure provider sharing the name.
|
||||
setupProvider: func(t *testing.T, ctx context.Context, providerName string, providerType database.AIProviderType) {
|
||||
deleted := dbgen.AIProvider(t, rawDB, database.AIProvider{Name: providerName, Type: database.AIProviderTypeOpenai})
|
||||
require.NoError(t, rawDB.DeleteAIProviderByID(ctx, deleted.ID), "delete provider")
|
||||
dbgen.AIProvider(t, rawDB, database.AIProvider{Name: providerName, Type: providerType})
|
||||
},
|
||||
wantInputPrice: sql.NullInt64{Int64: azureInputPrice, Valid: true},
|
||||
// 100 input tokens at the azure input price: $0.0005.
|
||||
wantCost: sql.NullInt64{Int64: 500, Valid: true},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx := testutil.Context(t, testutil.WaitLong)
|
||||
tc.setupProvider(t, ctx, tc.providerName, tc.providerType)
|
||||
|
||||
intc := dbgen.AIBridgeInterception(t, rawDB, database.InsertAIBridgeInterceptionParams{
|
||||
InitiatorID: user.ID,
|
||||
Provider: tc.wireProvider,
|
||||
ProviderName: tc.providerName,
|
||||
Model: tc.model,
|
||||
}, nil)
|
||||
|
||||
_, err := srv.RecordTokenUsage(ctx, &proto.RecordTokenUsageRequest{
|
||||
InterceptionId: intc.ID.String(),
|
||||
MsgId: "msg_e2e",
|
||||
InputTokens: 100,
|
||||
CreatedAt: timestamppb.Now(),
|
||||
})
|
||||
require.NoError(t, err, "record token usage")
|
||||
|
||||
tokenUsages, err := rawDB.GetAIBridgeTokenUsagesByInterceptionID(ctx, intc.ID)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, tokenUsages, 1)
|
||||
require.Equal(t, tc.wantInputPrice, tokenUsages[0].InputPriceMicros, "input price")
|
||||
require.Equal(t, tc.wantCost, tokenUsages[0].CostMicros, "cost")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestRecordTokenUsageBudgetNotifications verifies that recording token usage
|
||||
// enqueues the right budget notifications: the warning template when spend
|
||||
// crosses the warning threshold, the limit-reached template at 100%, both when
|
||||
@@ -2900,21 +3205,25 @@ func TestRecordTokenUsageBudgetAdminNotification(t *testing.T) {
|
||||
}
|
||||
|
||||
// newTestInterception returns an interception with a fixed initiator, provider,
|
||||
// and model for cost-attribution test setup.
|
||||
// and model for cost-attribution test setup. The provider name intentionally
|
||||
// differs from the upstream wire format.
|
||||
func newTestInterception(id uuid.UUID) database.AIBridgeInterception {
|
||||
return database.AIBridgeInterception{
|
||||
ID: id,
|
||||
InitiatorID: uuid.New(),
|
||||
Provider: "anthropic",
|
||||
Model: "claude-sonnet-4-6",
|
||||
ID: id,
|
||||
InitiatorID: uuid.New(),
|
||||
Provider: "anthropic",
|
||||
ProviderName: "anthropic-eu",
|
||||
Model: "claude-sonnet-4-6",
|
||||
}
|
||||
}
|
||||
|
||||
// expectTokenUsageCostLookups mocks the store lookups made by resolveTokenUsageCost
|
||||
// (budget resolution and the price lookup). A nil override, group, everyoneGroupID, or
|
||||
// price makes that lookup return sql.ErrNoRows. Budget resolution mirrors production code:
|
||||
// a non-nil override wins and skips the group lookup, and the Everyone fallback is consulted
|
||||
// only when both override and group are nil.
|
||||
// (budget resolution, provider resolution, and the price lookup). A nil override, group,
|
||||
// everyoneGroupID, or price makes that lookup return sql.ErrNoRows. Budget resolution
|
||||
// mirrors production code: a non-nil override wins and skips the group lookup, and the
|
||||
// Everyone fallback is consulted only when both override and group are nil. The provider
|
||||
// name resolves to a provider whose configured provider type equals the interception's
|
||||
// upstream wire format.
|
||||
func expectTokenUsageCostLookups(
|
||||
db *dbmock.MockStore,
|
||||
intc database.AIBridgeInterception,
|
||||
@@ -2945,6 +3254,11 @@ func expectTokenUsageCostLookups(
|
||||
}
|
||||
}
|
||||
|
||||
db.EXPECT().GetAIProviderByName(gomock.Any(), intc.ProviderName).Return(database.AIProvider{
|
||||
Name: intc.ProviderName,
|
||||
Type: database.AIProviderType(intc.Provider),
|
||||
}, nil)
|
||||
|
||||
if price != nil {
|
||||
db.EXPECT().GetAIModelPriceByProviderModel(gomock.Any(), database.GetAIModelPriceByProviderModelParams{
|
||||
Provider: intc.Provider,
|
||||
|
||||
@@ -34,10 +34,11 @@ type tokenUsageCost struct {
|
||||
}
|
||||
|
||||
// resolveTokenUsageCost resolves the effective group and per-token prices for an
|
||||
// interception and computes its cost. Two independent conditions yield a NULL
|
||||
// interception and computes its cost. Three independent conditions yield a NULL
|
||||
// column rather than an error: an unresolved effective group (the user has no
|
||||
// org membership), and a model absent from the price table leaves prices and
|
||||
// cost NULL (a NULL cost unambiguously means "model not priced").
|
||||
// org membership), an interception whose provider name matches no configured
|
||||
// provider, and a model absent from the price table. The latter two leave prices
|
||||
// and cost NULL (a NULL cost unambiguously means "model not priced").
|
||||
// Any other error is returned.
|
||||
func (s *Server) resolveTokenUsageCost(ctx context.Context, intc database.AIBridgeInterception, in *proto.RecordTokenUsageRequest) (tokenUsageCost, error) {
|
||||
var result tokenUsageCost
|
||||
@@ -63,22 +64,40 @@ func (s *Server) resolveTokenUsageCost(ctx context.Context, intc database.AIBrid
|
||||
}
|
||||
}
|
||||
|
||||
// The interception records one of three upstream wire formats. Prices are
|
||||
// keyed on the configured provider type, the provider actually serving the
|
||||
// request, resolved by provider name. Names are unique among live providers.
|
||||
provider, err := s.store.GetAIProviderByName(ctx, intc.ProviderName)
|
||||
switch {
|
||||
case errors.Is(err, sql.ErrNoRows):
|
||||
// Only reachable if the provider was deleted mid-request.
|
||||
s.logger.Info(ctx, "no configured provider found for interception, recording token usage with NULL cost",
|
||||
slog.F("provider_name", intc.ProviderName), slog.F("model", intc.Model))
|
||||
if s.metrics != nil {
|
||||
s.metrics.UnpricedTokenUsageRecords.WithLabelValues(intc.ProviderName, intc.Model).Inc()
|
||||
}
|
||||
return result, nil
|
||||
case err != nil:
|
||||
return tokenUsageCost{}, xerrors.Errorf("get configured provider %q: %w", intc.ProviderName, err)
|
||||
}
|
||||
configuredType := string(provider.Type)
|
||||
|
||||
// Snapshot the price for this (provider, model) and compute cost.
|
||||
price, err := s.store.GetAIModelPriceByProviderModel(ctx, database.GetAIModelPriceByProviderModelParams{
|
||||
Provider: intc.Provider,
|
||||
Provider: configuredType,
|
||||
Model: intc.Model,
|
||||
})
|
||||
switch {
|
||||
case errors.Is(err, sql.ErrNoRows):
|
||||
// Model not in the price table: record tokens but leave cost NULL.
|
||||
s.logger.Info(ctx, "no price found for model, recording token usage with NULL cost",
|
||||
slog.F("provider", intc.Provider), slog.F("model", intc.Model))
|
||||
slog.F("provider", configuredType), slog.F("model", intc.Model))
|
||||
if s.metrics != nil {
|
||||
s.metrics.UnpricedTokenUsageRecords.WithLabelValues(intc.Provider, intc.Model).Inc()
|
||||
s.metrics.UnpricedTokenUsageRecords.WithLabelValues(configuredType, intc.Model).Inc()
|
||||
}
|
||||
return result, nil
|
||||
case err != nil:
|
||||
return tokenUsageCost{}, xerrors.Errorf("look up model price for %s/%s: %w", intc.Provider, intc.Model, err)
|
||||
return tokenUsageCost{}, xerrors.Errorf("look up model price for %s/%s: %w", configuredType, intc.Model, err)
|
||||
}
|
||||
|
||||
result.inputPriceMicros = price.InputPrice
|
||||
|
||||
@@ -50,12 +50,11 @@ func NewMetrics(reg prometheus.Registerer) *Metrics {
|
||||
Name: "blocked_users",
|
||||
Help: "The number of users currently over their AI budget.",
|
||||
}, []string{"group_id"}),
|
||||
// Pessimistic cardinality: 3 providers, 5 models = up to 15.
|
||||
// Pessimistic cardinality: one series per configured provider and model.
|
||||
UnpricedTokenUsageRecords: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{
|
||||
Subsystem: "cost_control",
|
||||
Name: "unpriced_token_usage_records_total",
|
||||
Help: "The number of recorded AI token-usage records for which no model price was found " +
|
||||
"(provider: anthropic, openai, copilot).",
|
||||
Help: "The number of recorded AI token-usage records for which no (provider, model) price was found.",
|
||||
}, []string{"provider", "model"}),
|
||||
// Pessimistic cardinality: 3 outcomes, 8 buckets + 3 extra series
|
||||
// (count, sum, +Inf) = up to 33.
|
||||
|
||||
Reference in New Issue
Block a user