mirror of
https://github.com/coder/coder.git
synced 2026-08-31 01:03:45 +08:00
feat: report unpriced AI models to owners (#28419)
Closes [AIGOV-568](https://linear.app/codercom/issue/AIGOV-568/notify-admins-about-unpriced-ai-models). AI Bridge records token usage for a model with no price at a NULL cost, so its spend is neither reported nor enforced against any budget. Until now that only reached admins through an info log and a Prometheus metric. Owners now receive a weekly report listing models used without a price in the past week. The report links to documentation explaining how Coder calculates spend from the default price book and how owners can configure missing prices with `coder exp ai-model-prices`. ## Screenshots <img width="726" height="531" alt="image" src="https://github.com/user-attachments/assets/168ae378-60cb-4c96-8f60-36c6edc0b268" /> <img width="470" height="417" alt="image" src="https://github.com/user-attachments/assets/7dc71a1e-0c57-44f1-86f7-df800603c865" /> ## Design **Derived, not tracked.** The unpriced set is computed at report time from interceptions that recorded token usage, joined against the current `ai_model_prices` table. Requests that produced no token usage are excluded. **No new loop or table.** `reportUnpricedAIModels` is a sibling of `reportFailedWorkspaceBuilds` in the existing report generator, reusing its ticker and `notification_report_generator_logs`. Each report runs in its own transaction under a separate advisory lock, so failures and lock contention do not couple otherwise independent reports. The weekly frequency is enforced by the persisted timestamp rather than by the ticker, which restarts with the process and runs on a different phase in each replica. ## Behaviour | Situation | Outcome | |---|---| | Model used without a price | Listed in the next weekly report | | Price set | Disappears from the next report | | Still unpriced and still in use | Reported again each week | | Model stops being used | Drops out of the report | | Nothing unpriced | No notification; window still advances | | More than 100 unpriced models | Top 100 by usage, with the total reported alongside | | openai-compat models | Excluded, they cannot be priced | | Many replicas | Exactly one report per week | | First ever run | Reports models used in the preceding week immediately | ## Design choices 1. **openai-compat** is excluded because it cannot be priced and would produce permanent, unactionable notifications. 2. **Token volume** orders the list but is not shown. The 100-model cap therefore prioritizes models responsible for the most unpriced usage. 3. **Placement** remains in the generic report generator rather than `aibridgedserver`. This keeps one periodic-report lifecycle at the cost of coupling the notifications package to an AI cost-control query. 4. **No license gate.** The reporter runs even when AI Gateway is disabled. With no recorded token usage the query is empty and no notification is sent. --- Created by Coder Agents on behalf of @evgeniy-scherbina.
This commit is contained in:
Yevhenii Shcherbina
committed by
GitHub
GitHub
parent
ac95e7b743
commit
0bc2858b6e
+5
-4
@@ -1141,10 +1141,6 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd.
|
||||
// nolint:gocritic // We need to run the manager in a notifier context.
|
||||
notificationsManager.Run(dbauthz.AsNotifier(ctx))
|
||||
|
||||
// Run report generator to distribute periodic reports.
|
||||
notificationReportGenerator := reports.NewReportGenerator(ctx, logger.Named("notifications.report_generator"), options.Database, options.NotificationsEnqueuer, quartz.NewReal())
|
||||
defer notificationReportGenerator.Close()
|
||||
|
||||
// We use a separate coderAPICloser so the Enterprise API
|
||||
// can have its own close functions. This is cleaner
|
||||
// than abstracting the Coder API itself.
|
||||
@@ -1173,6 +1169,11 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd.
|
||||
// Must run after newAPI so options.Database is dbcrypt-wrapped.
|
||||
coderd.BackfillBedrockProviderType(aibridgeInitCtx, options.Database, logger.Named("aibridge.backfill"))
|
||||
|
||||
// Run report generator to distribute periodic reports.
|
||||
// Must run after newAPI so prices and providers are initialized.
|
||||
notificationReportGenerator := reports.NewReportGenerator(ctx, logger.Named("notifications.report_generator"), options.Database, options.NotificationsEnqueuer, quartz.NewReal())
|
||||
defer notificationReportGenerator.Close()
|
||||
|
||||
// In-memory aibridge daemon. Registered on coderd so chatd can
|
||||
// dispatch LLM requests via the in-process transport without
|
||||
// crossing the gated /api/v2/ai-gateway HTTP route. The HTTP route
|
||||
|
||||
@@ -5146,6 +5146,13 @@ func (q *querier) GetUnexpiredLicenses(ctx context.Context) ([]database.License,
|
||||
return q.db.GetUnexpiredLicenses(ctx)
|
||||
}
|
||||
|
||||
func (q *querier) GetUnpricedAIModelsSince(ctx context.Context, arg database.GetUnpricedAIModelsSinceParams) ([]database.GetUnpricedAIModelsSinceRow, error) {
|
||||
if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceAiModelPrice); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return q.db.GetUnpricedAIModelsSince(ctx, arg)
|
||||
}
|
||||
|
||||
func (q *querier) GetUserAIBudgetOverride(ctx context.Context, userID uuid.UUID) (database.UserAIBudgetOverride, error) {
|
||||
if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceUserObject(userID)); err != nil {
|
||||
return database.UserAIBudgetOverride{}, err
|
||||
|
||||
@@ -7133,6 +7133,11 @@ func (s *MethodTestSuite) TestAIBridge() {
|
||||
check.Args(database.GetAIModelPricesParams{}).Asserts(rbac.ResourceAiModelPrice, policy.ActionRead)
|
||||
}))
|
||||
|
||||
s.Run("GetUnpricedAIModelsSince", s.Mocked(func(db *dbmock.MockStore, _ *gofakeit.Faker, check *expects) {
|
||||
db.EXPECT().GetUnpricedAIModelsSince(gomock.Any(), gomock.Any()).Return([]database.GetUnpricedAIModelsSinceRow{}, nil).AnyTimes()
|
||||
check.Args(database.GetUnpricedAIModelsSinceParams{}).Asserts(rbac.ResourceAiModelPrice, policy.ActionRead)
|
||||
}))
|
||||
|
||||
s.Run("GetOrganizationGroupsAISpend", s.Mocked(func(dbm *dbmock.MockStore, faker *gofakeit.Faker, check *expects) {
|
||||
org := testutil.Fake(s.T(), faker, database.Organization{})
|
||||
row1 := testutil.Fake(s.T(), faker, database.GetOrganizationGroupsAISpendRow{OrganizationID: org.ID})
|
||||
|
||||
+8
@@ -3232,6 +3232,14 @@ func (m queryMetricsStore) GetUnexpiredLicenses(ctx context.Context) ([]database
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
func (m queryMetricsStore) GetUnpricedAIModelsSince(ctx context.Context, since database.GetUnpricedAIModelsSinceParams) ([]database.GetUnpricedAIModelsSinceRow, error) {
|
||||
start := time.Now()
|
||||
r0, r1 := m.s.GetUnpricedAIModelsSince(ctx, since)
|
||||
m.queryLatencies.WithLabelValues("GetUnpricedAIModelsSince").Observe(time.Since(start).Seconds())
|
||||
m.queryCounts.WithLabelValues(httpmw.ExtractHTTPRoute(ctx), httpmw.ExtractHTTPMethod(ctx), "GetUnpricedAIModelsSince").Inc()
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
func (m queryMetricsStore) GetUserAIBudgetOverride(ctx context.Context, userID uuid.UUID) (database.UserAIBudgetOverride, error) {
|
||||
start := time.Now()
|
||||
r0, r1 := m.s.GetUserAIBudgetOverride(ctx, userID)
|
||||
|
||||
Generated
+15
@@ -6089,6 +6089,21 @@ func (mr *MockStoreMockRecorder) GetUnexpiredLicenses(ctx any) *gomock.Call {
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetUnexpiredLicenses", reflect.TypeOf((*MockStore)(nil).GetUnexpiredLicenses), ctx)
|
||||
}
|
||||
|
||||
// GetUnpricedAIModelsSince mocks base method.
|
||||
func (m *MockStore) GetUnpricedAIModelsSince(ctx context.Context, arg database.GetUnpricedAIModelsSinceParams) ([]database.GetUnpricedAIModelsSinceRow, error) {
|
||||
m.ctrl.T.Helper()
|
||||
ret := m.ctrl.Call(m, "GetUnpricedAIModelsSince", ctx, arg)
|
||||
ret0, _ := ret[0].([]database.GetUnpricedAIModelsSinceRow)
|
||||
ret1, _ := ret[1].(error)
|
||||
return ret0, ret1
|
||||
}
|
||||
|
||||
// GetUnpricedAIModelsSince indicates an expected call of GetUnpricedAIModelsSince.
|
||||
func (mr *MockStoreMockRecorder) GetUnpricedAIModelsSince(ctx, arg any) *gomock.Call {
|
||||
mr.mock.ctrl.T.Helper()
|
||||
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetUnpricedAIModelsSince", reflect.TypeOf((*MockStore)(nil).GetUnpricedAIModelsSince), ctx, arg)
|
||||
}
|
||||
|
||||
// GetUserAIBudgetOverride mocks base method.
|
||||
func (m *MockStore) GetUserAIBudgetOverride(ctx context.Context, userID uuid.UUID) (database.UserAIBudgetOverride, error) {
|
||||
m.ctrl.T.Helper()
|
||||
|
||||
@@ -19,6 +19,7 @@ const (
|
||||
// Deprecated: Reserved to prevent reuse. Do not use at runtime.
|
||||
LockIDChatModelConfigWrites
|
||||
LockIDChatCapacityAdmission
|
||||
LockIDNotifyUnpricedAIModels
|
||||
)
|
||||
|
||||
// Per-setting advisory lock IDs for the chat instruction settings. These
|
||||
|
||||
@@ -32,6 +32,8 @@ func TestChatInstructionLockIDsDistinct(t *testing.T) {
|
||||
"LockIDBoundaryUsageStats": LockIDBoundaryUsageStats,
|
||||
"LockIDAIProvidersEnvSeed": LockIDAIProvidersEnvSeed,
|
||||
"LockIDChatModelConfigWrites": LockIDChatModelConfigWrites,
|
||||
"LockIDChatCapacityAdmission": LockIDChatCapacityAdmission,
|
||||
"LockIDNotifyUnpricedAIModels": LockIDNotifyUnpricedAIModels,
|
||||
}
|
||||
|
||||
// The two generated IDs are pairwise distinct.
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
DELETE FROM notification_templates
|
||||
WHERE id = '1b7d9fa7-f5a8-4e46-8078-5cf53abfed94';
|
||||
@@ -0,0 +1,30 @@
|
||||
INSERT INTO notification_templates (
|
||||
id,
|
||||
name,
|
||||
title_template,
|
||||
body_template,
|
||||
actions,
|
||||
"group",
|
||||
method,
|
||||
kind,
|
||||
enabled_by_default
|
||||
)
|
||||
VALUES (
|
||||
'1b7d9fa7-f5a8-4e46-8078-5cf53abfed94',
|
||||
'Missing AI Model Prices',
|
||||
E'Missing AI Model Prices',
|
||||
$$These models were used in the last {{.Data.report_frequency}}, but they have no price, so their usage is missing from AI spend and does not count toward any AI budget. Reported spend is lower than actual, and a user who calls only these models has no effective limit.
|
||||
|
||||
{{range $model := .Data.models}}
|
||||
* {{$model.provider}}/{{$model.model}}
|
||||
{{- end}}
|
||||
{{if .Data.truncated}}
|
||||
{{len .Data.models}} of {{.Data.total_count}} models with no price are shown, ordered by usage.
|
||||
{{end}}
|
||||
Every Coder release ships with prices for most models, so only the models above need one: see [how spend is calculated](https://coder.com/docs/ai-coder/ai-gateway/cost-controls#how-spend-is-calculated) and [how to configure prices](https://coder.com/docs/ai-coder/ai-gateway/cost-controls#configure-model-prices). Prices are not retroactive, so usage recorded before you set a price stays unpriced.$$,
|
||||
'[]'::jsonb,
|
||||
'AI Cost Control Admin Events',
|
||||
NULL,
|
||||
'system'::notification_template_kind,
|
||||
true
|
||||
);
|
||||
Generated
+3
@@ -910,6 +910,9 @@ type sqlcQuerier interface {
|
||||
// rollup and accept day-granularity bounds.
|
||||
GetTotalUsageHBAgentRuntimeV1(ctx context.Context, arg GetTotalUsageHBAgentRuntimeV1Params) (int64, error)
|
||||
GetUnexpiredLicenses(ctx context.Context) ([]License, error)
|
||||
// Returns the models used since the given time that hold no price, most used
|
||||
// first. openai-compat providers cannot be priced, so their models are excluded.
|
||||
GetUnpricedAIModelsSince(ctx context.Context, arg GetUnpricedAIModelsSinceParams) ([]GetUnpricedAIModelsSinceRow, error)
|
||||
GetUserAIBudgetOverride(ctx context.Context, userID uuid.UUID) (UserAIBudgetOverride, error)
|
||||
GetUserAIProviderKeyByProviderID(ctx context.Context, arg GetUserAIProviderKeyByProviderIDParams) (UserAIProviderKey, error)
|
||||
// GetUserAIProviderKeys is used by dbcrypt key rotation. Request paths should use
|
||||
|
||||
@@ -19315,6 +19315,80 @@ func TestOAuth2ProviderScopeNotEmpty(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestGetUnpricedAIModelsSince(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx := testutil.Context(t, testutil.WaitShort)
|
||||
db, _ := dbtestutil.NewDB(t)
|
||||
now := dbtime.Now()
|
||||
initiator := dbgen.User(t, db, database.User{})
|
||||
anthropic := dbgen.AIProvider(t, db, database.AIProvider{
|
||||
Name: "anthropic",
|
||||
Type: database.AIProviderTypeAnthropic,
|
||||
})
|
||||
openai := dbgen.AIProvider(t, db, database.AIProvider{
|
||||
Name: "openai",
|
||||
Type: database.AIProviderTypeOpenai,
|
||||
})
|
||||
|
||||
seedUsage := func(provider database.AIProvider, model string, startedAt time.Time, inputTokens, outputTokens int64, cost sql.NullInt64) {
|
||||
interception := dbgen.AIBridgeInterception(t, db, database.InsertAIBridgeInterceptionParams{
|
||||
InitiatorID: initiator.ID,
|
||||
Provider: string(provider.Type),
|
||||
ProviderName: provider.Name,
|
||||
Model: model,
|
||||
StartedAt: startedAt,
|
||||
}, nil)
|
||||
dbgen.AIBridgeTokenUsage(t, db, database.InsertAIBridgeTokenUsageParams{
|
||||
InterceptionID: interception.ID,
|
||||
InputTokens: inputTokens,
|
||||
OutputTokens: outputTokens,
|
||||
CostMicros: cost,
|
||||
CreatedAt: startedAt,
|
||||
})
|
||||
}
|
||||
|
||||
// Included and aggregated into one result with 500 total tokens.
|
||||
seedUsage(anthropic, "model-a", now, 200, 100, sql.NullInt64{})
|
||||
seedUsage(anthropic, "model-a", now, 100, 100, sql.NullInt64{})
|
||||
// Included after model-a because it has only 200 total tokens.
|
||||
seedUsage(anthropic, "model-b", now, 100, 100, sql.NullInt64{})
|
||||
// Excluded because its recorded cost is not NULL.
|
||||
seedUsage(anthropic, "costed-model", now, 100, 100, sql.NullInt64{Int64: 1, Valid: true})
|
||||
// Excluded because its interception started before the requested window.
|
||||
seedUsage(anthropic, "old-model", now.Add(-2*time.Hour), 100, 100, sql.NullInt64{})
|
||||
// Excluded because OpenAI is not in the supplied priceable provider list.
|
||||
seedUsage(openai, "excluded-provider-model", now, 100, 100, sql.NullInt64{})
|
||||
|
||||
// Excluded because an interception without token usage does not represent
|
||||
// model usage.
|
||||
dbgen.AIBridgeInterception(t, db, database.InsertAIBridgeInterceptionParams{
|
||||
InitiatorID: initiator.ID,
|
||||
Provider: string(anthropic.Type),
|
||||
ProviderName: anthropic.Name,
|
||||
Model: "model-without-usage",
|
||||
StartedAt: now,
|
||||
}, nil)
|
||||
|
||||
// Excluded because a current price exists for this provider and model.
|
||||
const pricedSeed = `[{"provider":"anthropic","model":"priced-model","input_price":1,"output_price":2,"cache_read_price":null,"cache_write_price":null}]`
|
||||
require.NoError(t, db.UpsertAIModelPrices(ctx, database.UpsertAIModelPricesParams{
|
||||
Seed: []byte(pricedSeed),
|
||||
Source: database.AIModelPriceSourceCustom,
|
||||
}))
|
||||
seedUsage(anthropic, "priced-model", now, 100, 100, sql.NullInt64{})
|
||||
|
||||
got, err := db.GetUnpricedAIModelsSince(ctx, database.GetUnpricedAIModelsSinceParams{
|
||||
Since: now.Add(-time.Hour),
|
||||
PriceableProviders: []string{string(database.AIProviderTypeAnthropic)},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, []database.GetUnpricedAIModelsSinceRow{
|
||||
{ProviderType: "anthropic", Model: "model-a", TokenCount: 500},
|
||||
{ProviderType: "anthropic", Model: "model-b", TokenCount: 200},
|
||||
}, got)
|
||||
}
|
||||
|
||||
func TestGetAIModelPriceByProviderModel(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
Generated
+65
@@ -3410,6 +3410,71 @@ func (q *sqlQuerier) GetOverBudgetUsersPerGroup(ctx context.Context, periodStart
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const getUnpricedAIModelsSince = `-- name: GetUnpricedAIModelsSince :many
|
||||
SELECT
|
||||
providers.type::text AS provider_type,
|
||||
interceptions.model AS model,
|
||||
SUM(
|
||||
token_usages.input_tokens
|
||||
+ token_usages.output_tokens
|
||||
+ token_usages.cache_read_input_tokens
|
||||
+ token_usages.cache_write_input_tokens
|
||||
)::bigint AS token_count
|
||||
FROM aibridge_interceptions AS interceptions
|
||||
JOIN aibridge_token_usages AS token_usages
|
||||
ON token_usages.interception_id = interceptions.id
|
||||
JOIN ai_providers AS providers
|
||||
ON providers.name = interceptions.provider_name
|
||||
AND providers.deleted = false
|
||||
WHERE interceptions.started_at >= $1::timestamptz
|
||||
AND token_usages.cost_micros IS NULL
|
||||
AND providers.type::text = ANY($2::text[])
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM ai_model_prices AS prices
|
||||
WHERE prices.provider = providers.type::text
|
||||
AND prices.model = interceptions.model
|
||||
)
|
||||
GROUP BY providers.type, interceptions.model
|
||||
ORDER BY token_count DESC, provider_type ASC, model ASC
|
||||
`
|
||||
|
||||
type GetUnpricedAIModelsSinceParams struct {
|
||||
Since time.Time `db:"since" json:"since"`
|
||||
PriceableProviders []string `db:"priceable_providers" json:"priceable_providers"`
|
||||
}
|
||||
|
||||
type GetUnpricedAIModelsSinceRow struct {
|
||||
ProviderType string `db:"provider_type" json:"provider_type"`
|
||||
Model string `db:"model" json:"model"`
|
||||
TokenCount int64 `db:"token_count" json:"token_count"`
|
||||
}
|
||||
|
||||
// Returns the models used since the given time that hold no price, most used
|
||||
// first. openai-compat providers cannot be priced, so their models are excluded.
|
||||
func (q *sqlQuerier) GetUnpricedAIModelsSince(ctx context.Context, arg GetUnpricedAIModelsSinceParams) ([]GetUnpricedAIModelsSinceRow, error) {
|
||||
rows, err := q.db.QueryContext(ctx, getUnpricedAIModelsSince, arg.Since, pq.Array(arg.PriceableProviders))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []GetUnpricedAIModelsSinceRow
|
||||
for rows.Next() {
|
||||
var i GetUnpricedAIModelsSinceRow
|
||||
if err := rows.Scan(&i.ProviderType, &i.Model, &i.TokenCount); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, i)
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const getUserAIBudgetOverride = `-- name: GetUserAIBudgetOverride :one
|
||||
SELECT user_id, group_id, spend_limit_micros, created_at, updated_at
|
||||
FROM user_ai_budget_overrides
|
||||
|
||||
@@ -482,3 +482,33 @@ GROUP BY
|
||||
ai.provider,
|
||||
ai.provider_name
|
||||
ORDER BY ai.initiator_id, tu.effective_group_id, ai.provider, ai.provider_name, ai.model;
|
||||
|
||||
-- name: GetUnpricedAIModelsSince :many
|
||||
-- Returns the models used since the given time that hold no price, most used
|
||||
-- first. openai-compat providers cannot be priced, so their models are excluded.
|
||||
SELECT
|
||||
providers.type::text AS provider_type,
|
||||
interceptions.model AS model,
|
||||
SUM(
|
||||
token_usages.input_tokens
|
||||
+ token_usages.output_tokens
|
||||
+ token_usages.cache_read_input_tokens
|
||||
+ token_usages.cache_write_input_tokens
|
||||
)::bigint AS token_count
|
||||
FROM aibridge_interceptions AS interceptions
|
||||
JOIN aibridge_token_usages AS token_usages
|
||||
ON token_usages.interception_id = interceptions.id
|
||||
JOIN ai_providers AS providers
|
||||
ON providers.name = interceptions.provider_name
|
||||
AND providers.deleted = false
|
||||
WHERE interceptions.started_at >= @since::timestamptz
|
||||
AND token_usages.cost_micros IS NULL
|
||||
AND providers.type::text = ANY(@priceable_providers::text[])
|
||||
AND NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM ai_model_prices AS prices
|
||||
WHERE prices.provider = providers.type::text
|
||||
AND prices.model = interceptions.model
|
||||
)
|
||||
GROUP BY providers.type, interceptions.model
|
||||
ORDER BY token_count DESC, provider_type ASC, model ASC;
|
||||
|
||||
@@ -76,4 +76,5 @@ var (
|
||||
TemplateAIBudgetLimitReachedUser = uuid.MustParse("cdcf2ecd-f003-4169-9800-abb2661ea522")
|
||||
TemplateAIBudgetWarningAdmin = uuid.MustParse("2a7b0ac1-00e1-4625-9cd5-1e5933972c77")
|
||||
TemplateAIBudgetLimitReachedAdmin = uuid.MustParse("0bafe0ea-a78b-4217-ad05-1ef12e92e025")
|
||||
TemplateAIModelsUnpricedReport = uuid.MustParse("1b7d9fa7-f5a8-4e46-8078-5cf53abfed94")
|
||||
)
|
||||
|
||||
@@ -1619,6 +1619,27 @@ func TestNotificationTemplates_Golden(t *testing.T) {
|
||||
Data: map[string]any{},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "TemplateAIModelsUnpricedReport",
|
||||
id: notifications.TemplateAIModelsUnpricedReport,
|
||||
payload: types.MessagePayload{
|
||||
UserName: "Bobby",
|
||||
UserEmail: "bobby@coder.com",
|
||||
UserUsername: "bobby",
|
||||
Labels: map[string]string{},
|
||||
// We need to use floats as `json.Unmarshal` unmarshal numbers in `map[string]any` to floats.
|
||||
Data: map[string]any{
|
||||
"report_frequency": "week",
|
||||
"models": []map[string]any{
|
||||
{"provider": "anthropic", "model": "claude-opus-4-8"},
|
||||
{"provider": "openai", "model": "gpt-5.7"},
|
||||
{"provider": "openrouter", "model": "z-ai/glm-5.4"},
|
||||
},
|
||||
"total_count": 15.0,
|
||||
"truncated": true,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// We must have a test case for every notification_template. This is enforced below:
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
"cdr.dev/slog/v3"
|
||||
"github.com/coder/coder/v2/coderd/aibridge/prices/providers"
|
||||
"github.com/coder/coder/v2/coderd/database"
|
||||
"github.com/coder/coder/v2/coderd/database/dbauthz"
|
||||
"github.com/coder/coder/v2/coderd/database/dbtime"
|
||||
@@ -25,6 +26,30 @@ const (
|
||||
delay = 15 * time.Minute
|
||||
)
|
||||
|
||||
// runReport executes one report in its own transaction, guarded by its own
|
||||
// advisory lock so that only one replica generates it. A replica that cannot
|
||||
// take the lock skips this tick and tries again on the next one; how often a
|
||||
// report is actually sent is enforced by the report itself, against the
|
||||
// timestamp it persists.
|
||||
func runReport(ctx context.Context, logger slog.Logger, db database.Store, lockID int64, name string, report func(tx database.Store) error) {
|
||||
err := db.InTx(func(tx database.Store) error {
|
||||
ok, err := tx.TryAcquireLock(ctx, lockID)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("failed to acquire report lock: %w", err)
|
||||
}
|
||||
if !ok {
|
||||
logger.Debug(ctx, "unable to acquire lock for generating periodic report, skipping", slog.F("report", name))
|
||||
return nil
|
||||
}
|
||||
return report(tx)
|
||||
}, nil)
|
||||
if err != nil {
|
||||
logger.Error(ctx, "failed to generate report", slog.F("report", name), slog.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
// NewReportGenerator periodically generates failed workspace build and
|
||||
// unpriced AI model reports.
|
||||
func NewReportGenerator(ctx context.Context, logger slog.Logger, db database.Store, enqueuer notifications.Enqueuer, clk quartz.Clock) io.Closer {
|
||||
closed := make(chan struct{})
|
||||
|
||||
@@ -35,32 +60,19 @@ func NewReportGenerator(ctx context.Context, logger slog.Logger, db database.Sto
|
||||
// Start the ticker with the initial delay.
|
||||
ticker := clk.NewTicker(delay)
|
||||
ticker.Stop()
|
||||
doTick := func(start time.Time) {
|
||||
doTick := func(_ time.Time) {
|
||||
defer ticker.Reset(delay)
|
||||
// Start a transaction to grab advisory lock, we don't want to run generator jobs at the same time (multiple replicas).
|
||||
if err := db.InTx(func(tx database.Store) error {
|
||||
// Acquire a lock to ensure that only one instance of the generator is running at a time.
|
||||
ok, err := tx.TryAcquireLock(ctx, database.LockIDNotificationsReportGenerator)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("failed to acquire report generator lock: %w", err)
|
||||
}
|
||||
if !ok {
|
||||
logger.Debug(ctx, "unable to acquire lock for generating periodic reports, skipping")
|
||||
return nil
|
||||
}
|
||||
|
||||
err = reportFailedWorkspaceBuilds(ctx, logger, tx, enqueuer, clk)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("unable to generate reports with failed workspace builds: %w", err)
|
||||
}
|
||||
|
||||
logger.Info(ctx, "report generator finished", slog.F("duration", clk.Since(start)))
|
||||
|
||||
return nil
|
||||
}, nil); err != nil {
|
||||
logger.Error(ctx, "failed to generate reports", slog.Error(err))
|
||||
return
|
||||
}
|
||||
// Reports are independent, so each runs in its own transaction under
|
||||
// its own advisory lock. Sharing either would couple them.
|
||||
runReport(ctx, logger, db, database.LockIDNotificationsReportGenerator, "failed workspace builds",
|
||||
func(tx database.Store) error {
|
||||
return reportFailedWorkspaceBuilds(ctx, logger, tx, enqueuer, clk)
|
||||
})
|
||||
runReport(ctx, logger, db, database.LockIDNotifyUnpricedAIModels, "unpriced AI models",
|
||||
func(tx database.Store) error {
|
||||
return reportUnpricedAIModels(ctx, logger, tx, enqueuer, clk)
|
||||
})
|
||||
}
|
||||
|
||||
go func() {
|
||||
@@ -330,3 +342,97 @@ func findTemplateAdmins(ctx context.Context, db database.Store, stats database.G
|
||||
})
|
||||
return templateAdmins, nil
|
||||
}
|
||||
|
||||
const (
|
||||
unpricedAIModelsReportFrequency = 7 * 24 * time.Hour
|
||||
unpricedAIModelsReportFrequencyLabel = "week"
|
||||
// unpricedAIModelsLimit caps how many models a single report lists.
|
||||
// A deployment can accumulate more unpriced models than we want to display
|
||||
// in a single notification, so the remaining models are reported as a count.
|
||||
unpricedAIModelsLimit = 100
|
||||
)
|
||||
|
||||
// reportUnpricedAIModels notifies owners about models used without a price
|
||||
// in the preceding week. Unpriced usage is recorded but contributes nothing to
|
||||
// spend, so it is neither reported nor enforced against a budget.
|
||||
//
|
||||
// The set of unpriced models is derived at report time from interceptions and
|
||||
// the price table, so setting a price removes the model from the next report.
|
||||
func reportUnpricedAIModels(ctx context.Context, logger slog.Logger, db database.Store, enqueuer notifications.Enqueuer, clk quartz.Clock) error {
|
||||
now := clk.Now()
|
||||
since := now.Add(-unpricedAIModelsReportFrequency)
|
||||
|
||||
reportLog, err := db.GetNotificationReportGeneratorLogByTemplate(ctx, notifications.TemplateAIModelsUnpricedReport)
|
||||
if err != nil && !xerrors.Is(err, sql.ErrNoRows) {
|
||||
return xerrors.Errorf("unable to read report generator log: %w", err)
|
||||
}
|
||||
|
||||
// Check if the job has not been running recently. The ticker alone cannot
|
||||
// enforce the frequency: it restarts with the process and each replica
|
||||
// runs on its own phase.
|
||||
if !reportLog.LastGeneratedAt.IsZero() && reportLog.LastGeneratedAt.Add(unpricedAIModelsReportFrequency).After(now) {
|
||||
return nil // reports sent recently, no need to send them now
|
||||
}
|
||||
|
||||
// Fetch the models used without a price.
|
||||
unpricedModels, err := db.GetUnpricedAIModelsSince(ctx, database.GetUnpricedAIModelsSinceParams{
|
||||
Since: dbtime.Time(since).UTC(),
|
||||
PriceableProviders: providers.SupportedStrings(),
|
||||
})
|
||||
if err != nil {
|
||||
return xerrors.Errorf("unable to fetch unpriced AI models: %w", err)
|
||||
}
|
||||
|
||||
if len(unpricedModels) > 0 {
|
||||
owners, err := db.GetUsers(ctx, database.GetUsersParams{
|
||||
RbacRole: []string{codersdk.RoleOwner},
|
||||
})
|
||||
if err != nil {
|
||||
return xerrors.Errorf("unable to fetch owners: %w", err)
|
||||
}
|
||||
|
||||
reportData := buildDataForReportUnpricedAIModels(unpricedModels)
|
||||
for _, owner := range owners {
|
||||
if _, err := enqueuer.EnqueueWithData(ctx, owner.ID, notifications.TemplateAIModelsUnpricedReport,
|
||||
map[string]string{},
|
||||
reportData,
|
||||
"report_generator",
|
||||
); err != nil {
|
||||
logger.Warn(ctx, "failed to send a report with unpriced AI models", slog.F("user_id", owner.ID), slog.Error(err))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Update the timestamp in the generator log. This happens even
|
||||
// when nothing was reported, so the next report covers one week rather
|
||||
// than every week since usage was last seen.
|
||||
err = db.UpsertNotificationReportGeneratorLog(ctx, database.UpsertNotificationReportGeneratorLogParams{
|
||||
NotificationTemplateID: notifications.TemplateAIModelsUnpricedReport,
|
||||
LastGeneratedAt: dbtime.Time(now).UTC(),
|
||||
})
|
||||
if err != nil {
|
||||
return xerrors.Errorf("unable to update report generator logs: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// buildDataForReportUnpricedAIModels renders the models most used first, so
|
||||
// the models dropped by the limit are the ones with the least unreported
|
||||
// usage.
|
||||
func buildDataForReportUnpricedAIModels(unpricedModels []database.GetUnpricedAIModelsSinceRow) map[string]any {
|
||||
reportedCount := min(len(unpricedModels), unpricedAIModelsLimit)
|
||||
reportedModels := make([]map[string]any, 0, reportedCount)
|
||||
for _, row := range unpricedModels[:reportedCount] {
|
||||
reportedModels = append(reportedModels, map[string]any{
|
||||
"provider": row.ProviderType,
|
||||
"model": row.Model,
|
||||
})
|
||||
}
|
||||
|
||||
return map[string]any{
|
||||
"report_frequency": unpricedAIModelsReportFrequencyLabel,
|
||||
"models": reportedModels,
|
||||
"total_count": len(unpricedModels),
|
||||
"truncated": len(unpricedModels) > reportedCount,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,374 @@
|
||||
package reports
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/coder/coder/v2/coderd/database"
|
||||
"github.com/coder/coder/v2/coderd/database/dbgen"
|
||||
"github.com/coder/coder/v2/coderd/notifications"
|
||||
"github.com/coder/coder/v2/coderd/notifications/notificationstest"
|
||||
"github.com/coder/coder/v2/codersdk"
|
||||
"github.com/coder/coder/v2/testutil"
|
||||
)
|
||||
|
||||
func TestReportGenerator_TicksUnpricedAIModels(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx := testutil.Context(t, testutil.WaitShort)
|
||||
_, logger, db, _, notifEnq, clk := setup(t)
|
||||
seedOwner(t, db)
|
||||
|
||||
// The ticker is reset after each generator run, so this trap signals when
|
||||
// the immediate or ticker-driven run has finished.
|
||||
resetTrap := clk.Trap().TickerReset()
|
||||
defer resetTrap.Close()
|
||||
|
||||
generator := NewReportGenerator(ctx, logger, db, notifEnq, clk)
|
||||
t.Cleanup(func() {
|
||||
require.NoError(t, generator.Close())
|
||||
})
|
||||
|
||||
// The generator runs once immediately without waiting for the ticker delay.
|
||||
resetTrap.MustWait(ctx).MustRelease(ctx)
|
||||
require.Empty(t, notifEnq.Sent())
|
||||
|
||||
// The initial run records the current time. Backdate it beyond the weekly
|
||||
// frequency so the report is eligible on the next 15-minute ticker run.
|
||||
require.NoError(t, db.UpsertNotificationReportGeneratorLog(ctx, database.UpsertNotificationReportGeneratorLogParams{
|
||||
NotificationTemplateID: notifications.TemplateAIModelsUnpricedReport,
|
||||
LastGeneratedAt: clk.Now().Add(-unpricedAIModelsReportFrequency - time.Minute),
|
||||
}))
|
||||
seedUnpricedUsage(t, db, "anthropic", database.AIProviderTypeAnthropic, "claude-opus-4-8", clk.Now())
|
||||
|
||||
// Advance to the first ticker-driven run and wait for it to finish.
|
||||
advance := clk.Advance(delay)
|
||||
resetTrap.MustWait(ctx).MustRelease(ctx)
|
||||
advance.MustWait(ctx)
|
||||
|
||||
sent := notifEnq.Sent()
|
||||
require.Len(t, sent, 1)
|
||||
require.Equal(t, notifications.TemplateAIModelsUnpricedReport, sent[0].TemplateID)
|
||||
}
|
||||
|
||||
func TestReportUnpricedAIModels(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
t.Run("FirstRun_ReportsPrecedingWeek", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx, logger, db, _, notifEnq, clk := setup(t)
|
||||
owner := seedOwner(t, db)
|
||||
seedUnpricedUsage(t, db, "anthropic", database.AIProviderTypeAnthropic, "claude-opus-4-8", clk.Now())
|
||||
require.NoError(t, reportUnpricedAIModels(ctx, logger, db, notifEnq, clk))
|
||||
|
||||
sent := notifEnq.Sent(notificationstest.WithTemplateID(notifications.TemplateAIModelsUnpricedReport))
|
||||
require.Len(t, sent, 1)
|
||||
require.Equal(t, owner.ID, sent[0].UserID)
|
||||
require.Equal(t, "week", sent[0].Data["report_frequency"])
|
||||
require.Equal(t, []map[string]any{
|
||||
{"provider": "anthropic", "model": "claude-opus-4-8"},
|
||||
}, modelsFromPayload(t, sent[0].Data))
|
||||
require.EqualValues(t, 1, sent[0].Data["total_count"])
|
||||
require.Equal(t, false, sent[0].Data["truncated"])
|
||||
})
|
||||
|
||||
t.Run("ReportsOnlyOncePerFrequency", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx, logger, db, _, notifEnq, clk := setup(t)
|
||||
seedOwner(t, db)
|
||||
seedUnpricedUsage(t, db, "anthropic", database.AIProviderTypeAnthropic, "claude-opus-4-8", clk.Now())
|
||||
|
||||
require.NoError(t, reportUnpricedAIModels(ctx, logger, db, notifEnq, clk))
|
||||
require.Len(t, notifEnq.Sent(notificationstest.WithTemplateID(notifications.TemplateAIModelsUnpricedReport)), 1)
|
||||
|
||||
notifEnq.Clear()
|
||||
clk.Advance(time.Hour)
|
||||
require.NoError(t, reportUnpricedAIModels(ctx, logger, db, notifEnq, clk))
|
||||
|
||||
require.Empty(t, notifEnq.Sent())
|
||||
})
|
||||
|
||||
t.Run("StillUnpricedAndInUse_IsReportedAgain", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx, logger, db, _, notifEnq, clk := setup(t)
|
||||
seedOwner(t, db)
|
||||
initiator := dbgen.User(t, db, database.User{})
|
||||
provider := seedProvider(t, db, "anthropic", database.AIProviderTypeAnthropic)
|
||||
|
||||
seedUnpricedModelUsage(t, db, initiator, provider, "claude-opus-4-8", clk.Now(), 100)
|
||||
require.NoError(t, reportUnpricedAIModels(ctx, logger, db, notifEnq, clk))
|
||||
require.Len(t, notifEnq.Sent(notificationstest.WithTemplateID(notifications.TemplateAIModelsUnpricedReport)), 1)
|
||||
|
||||
notifEnq.Clear()
|
||||
clk.Advance(unpricedAIModelsReportFrequency + time.Minute)
|
||||
seedUnpricedModelUsage(t, db, initiator, provider, "claude-opus-4-8", clk.Now(), 100)
|
||||
require.NoError(t, reportUnpricedAIModels(ctx, logger, db, notifEnq, clk))
|
||||
|
||||
require.Len(t, notifEnq.Sent(notificationstest.WithTemplateID(notifications.TemplateAIModelsUnpricedReport)), 1)
|
||||
})
|
||||
|
||||
t.Run("NoLongerUsed_IsNotReported", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx, logger, db, _, notifEnq, clk := setup(t)
|
||||
seedOwner(t, db)
|
||||
initiator := dbgen.User(t, db, database.User{})
|
||||
provider := seedProvider(t, db, "anthropic", database.AIProviderTypeAnthropic)
|
||||
seedUnpricedModelUsage(t, db, initiator, provider, "claude-opus-4-8", clk.Now(), 100)
|
||||
|
||||
require.NoError(t, reportUnpricedAIModels(ctx, logger, db, notifEnq, clk))
|
||||
require.Len(t, notifEnq.Sent(notificationstest.WithTemplateID(notifications.TemplateAIModelsUnpricedReport)), 1)
|
||||
notifEnq.Clear()
|
||||
|
||||
clk.Advance(unpricedAIModelsReportFrequency + time.Minute)
|
||||
require.NoError(t, reportUnpricedAIModels(ctx, logger, db, notifEnq, clk))
|
||||
|
||||
require.Empty(t, notifEnq.Sent())
|
||||
})
|
||||
|
||||
t.Run("InterceptionWithoutTokenUsage_IsNotReported", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx, logger, db, _, notifEnq, clk := setup(t)
|
||||
seedOwner(t, db)
|
||||
initiator := dbgen.User(t, db, database.User{})
|
||||
provider := seedProvider(t, db, "anthropic", database.AIProviderTypeAnthropic)
|
||||
seedInterception(t, db, initiator, provider, "mistyped-model", clk.Now())
|
||||
|
||||
require.NoError(t, reportUnpricedAIModels(ctx, logger, db, notifEnq, clk))
|
||||
require.Empty(t, notifEnq.Sent())
|
||||
})
|
||||
|
||||
t.Run("HistoricallyCostedUsage_IsNotReported", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx, logger, db, _, notifEnq, clk := setup(t)
|
||||
seedOwner(t, db)
|
||||
initiator := dbgen.User(t, db, database.User{})
|
||||
provider := seedProvider(t, db, "anthropic", database.AIProviderTypeAnthropic)
|
||||
interception := seedInterception(t, db, initiator, provider, "previously-priced-model", clk.Now())
|
||||
dbgen.AIBridgeTokenUsage(t, db, database.InsertAIBridgeTokenUsageParams{
|
||||
InterceptionID: interception.ID,
|
||||
InputTokens: 100,
|
||||
CostMicros: sql.NullInt64{Int64: 10, Valid: true},
|
||||
CreatedAt: clk.Now(),
|
||||
})
|
||||
|
||||
require.NoError(t, reportUnpricedAIModels(ctx, logger, db, notifEnq, clk))
|
||||
require.Empty(t, notifEnq.Sent())
|
||||
})
|
||||
|
||||
t.Run("PricedModel_IsNotReported", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx, logger, db, _, notifEnq, clk := setup(t)
|
||||
seedOwner(t, db)
|
||||
initiator := dbgen.User(t, db, database.User{})
|
||||
provider := seedProvider(t, db, "anthropic", database.AIProviderTypeAnthropic)
|
||||
seedUnpricedModelUsage(t, db, initiator, provider, "claude-opus-4-8", clk.Now(), 100)
|
||||
|
||||
seedPrice(ctx, t, db, "anthropic", "claude-opus-4-8")
|
||||
require.NoError(t, reportUnpricedAIModels(ctx, logger, db, notifEnq, clk))
|
||||
|
||||
require.Empty(t, notifEnq.Sent())
|
||||
})
|
||||
|
||||
t.Run("PriceMustMatchProviderAndModel", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx, logger, db, _, notifEnq, clk := setup(t)
|
||||
seedOwner(t, db)
|
||||
initiator := dbgen.User(t, db, database.User{})
|
||||
provider := seedProvider(t, db, "anthropic", database.AIProviderTypeAnthropic)
|
||||
seedUnpricedModelUsage(t, db, initiator, provider, "target-model", clk.Now(), 100)
|
||||
|
||||
// The model matches the OpenAI price, and the provider matches the other
|
||||
// Anthropic price. Neither price matches anthropic/target-model exactly.
|
||||
seedPrice(ctx, t, db, "openai", "target-model")
|
||||
seedPrice(ctx, t, db, "anthropic", "different-model")
|
||||
require.NoError(t, reportUnpricedAIModels(ctx, logger, db, notifEnq, clk))
|
||||
|
||||
sent := notifEnq.Sent(notificationstest.WithTemplateID(notifications.TemplateAIModelsUnpricedReport))
|
||||
require.Len(t, sent, 1)
|
||||
require.Equal(t, []map[string]any{
|
||||
{"provider": "anthropic", "model": "target-model"},
|
||||
}, modelsFromPayload(t, sent[0].Data))
|
||||
})
|
||||
|
||||
t.Run("OpenAICompat_IsNotReported", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx, logger, db, _, notifEnq, clk := setup(t)
|
||||
seedOwner(t, db)
|
||||
initiator := dbgen.User(t, db, database.User{})
|
||||
provider := seedProvider(t, db, "self-hosted", database.AIProviderTypeOpenaiCompat)
|
||||
seedUnpricedModelUsage(t, db, initiator, provider, "llama-4", clk.Now(), 100)
|
||||
|
||||
require.NoError(t, reportUnpricedAIModels(ctx, logger, db, notifEnq, clk))
|
||||
|
||||
require.Empty(t, notifEnq.Sent())
|
||||
})
|
||||
|
||||
t.Run("ReportsEveryOwner", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx, logger, db, _, notifEnq, clk := setup(t)
|
||||
firstOwner := seedOwner(t, db)
|
||||
secondOwner := seedOwner(t, db)
|
||||
member := dbgen.User(t, db, database.User{})
|
||||
initiator := dbgen.User(t, db, database.User{})
|
||||
provider := seedProvider(t, db, "anthropic", database.AIProviderTypeAnthropic)
|
||||
seedUnpricedModelUsage(t, db, initiator, provider, "claude-opus-4-8", clk.Now(), 100)
|
||||
|
||||
require.NoError(t, reportUnpricedAIModels(ctx, logger, db, notifEnq, clk))
|
||||
|
||||
sent := notifEnq.Sent(notificationstest.WithTemplateID(notifications.TemplateAIModelsUnpricedReport))
|
||||
require.Len(t, sent, 2)
|
||||
recipients := []uuid.UUID{sent[0].UserID, sent[1].UserID}
|
||||
require.Contains(t, recipients, firstOwner.ID)
|
||||
require.Contains(t, recipients, secondOwner.ID)
|
||||
require.NotContains(t, recipients, member.ID)
|
||||
})
|
||||
|
||||
t.Run("TruncatesToLimit", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx, logger, db, _, notifEnq, clk := setup(t)
|
||||
seedOwner(t, db)
|
||||
initiator := dbgen.User(t, db, database.User{})
|
||||
provider := seedProvider(t, db, "anthropic", database.AIProviderTypeAnthropic)
|
||||
|
||||
// Give the first model more token usage so it sorts ahead of the others.
|
||||
const overflow = 5
|
||||
seedUnpricedModelUsage(t, db, initiator, provider, "most-used-model", clk.Now(), 1_000)
|
||||
for i := range unpricedAIModelsLimit + overflow - 1 {
|
||||
seedUnpricedModelUsage(t, db, initiator, provider, fmt.Sprintf("model-%03d", i), clk.Now(), 100)
|
||||
}
|
||||
|
||||
require.NoError(t, reportUnpricedAIModels(ctx, logger, db, notifEnq, clk))
|
||||
|
||||
sent := notifEnq.Sent(notificationstest.WithTemplateID(notifications.TemplateAIModelsUnpricedReport))
|
||||
require.Len(t, sent, 1)
|
||||
models := modelsFromPayload(t, sent[0].Data)
|
||||
require.Len(t, models, unpricedAIModelsLimit)
|
||||
require.Equal(t, "most-used-model", models[0]["model"])
|
||||
require.EqualValues(t, unpricedAIModelsLimit+overflow, sent[0].Data["total_count"])
|
||||
require.Equal(t, true, sent[0].Data["truncated"])
|
||||
})
|
||||
|
||||
t.Run("NothingToReport_AdvancesWindow", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx, logger, db, _, notifEnq, clk := setup(t)
|
||||
seedOwner(t, db)
|
||||
now := clk.Now()
|
||||
|
||||
require.NoError(t, reportUnpricedAIModels(ctx, logger, db, notifEnq, clk))
|
||||
require.Empty(t, notifEnq.Sent())
|
||||
|
||||
reportLog, err := db.GetNotificationReportGeneratorLogByTemplate(ctx, notifications.TemplateAIModelsUnpricedReport)
|
||||
require.NoError(t, err)
|
||||
require.True(t, now.Equal(reportLog.LastGeneratedAt))
|
||||
})
|
||||
}
|
||||
|
||||
func seedOwner(t *testing.T, db database.Store) database.User {
|
||||
t.Helper()
|
||||
return dbgen.User(t, db, database.User{
|
||||
RBACRoles: []string{codersdk.RoleOwner},
|
||||
})
|
||||
}
|
||||
|
||||
func seedProvider(t *testing.T, db database.Store, name string, providerType database.AIProviderType) database.AIProvider {
|
||||
t.Helper()
|
||||
return dbgen.AIProvider(t, db, database.AIProvider{
|
||||
Name: name,
|
||||
Type: providerType,
|
||||
})
|
||||
}
|
||||
|
||||
func seedInterception(t *testing.T, db database.Store, initiator database.User, provider database.AIProvider, model string, startedAt time.Time) database.AIBridgeInterception {
|
||||
t.Helper()
|
||||
return dbgen.AIBridgeInterception(t, db, database.InsertAIBridgeInterceptionParams{
|
||||
InitiatorID: initiator.ID,
|
||||
Provider: string(provider.Type),
|
||||
ProviderName: provider.Name,
|
||||
Model: model,
|
||||
StartedAt: startedAt,
|
||||
}, nil)
|
||||
}
|
||||
|
||||
func seedUnpricedModelUsage(t *testing.T, db database.Store, initiator database.User, provider database.AIProvider, model string, createdAt time.Time, inputTokens int64) {
|
||||
t.Helper()
|
||||
interception := seedInterception(t, db, initiator, provider, model, createdAt)
|
||||
dbgen.AIBridgeTokenUsage(t, db, database.InsertAIBridgeTokenUsageParams{
|
||||
InterceptionID: interception.ID,
|
||||
InputTokens: inputTokens,
|
||||
CreatedAt: createdAt,
|
||||
})
|
||||
}
|
||||
|
||||
func seedUnpricedUsage(t *testing.T, db database.Store, providerName string, providerType database.AIProviderType, model string, createdAt time.Time) {
|
||||
t.Helper()
|
||||
seedUnpricedModelUsage(t, db, dbgen.User(t, db, database.User{}), seedProvider(t, db, providerName, providerType), model, createdAt, 100)
|
||||
}
|
||||
|
||||
func seedPrice(ctx context.Context, t *testing.T, db database.Store, provider, model string) {
|
||||
t.Helper()
|
||||
seed, err := json.Marshal([]map[string]any{{
|
||||
"provider": provider,
|
||||
"model": model,
|
||||
"input_price": 3_000_000,
|
||||
"output_price": 15_000_000,
|
||||
"cache_read_price": nil,
|
||||
"cache_write_price": nil,
|
||||
}})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, db.UpsertAIModelPrices(ctx, database.UpsertAIModelPricesParams{
|
||||
Seed: seed,
|
||||
Source: database.AIModelPriceSourceCustom,
|
||||
}))
|
||||
}
|
||||
|
||||
func modelsFromPayload(t *testing.T, data map[string]any) []map[string]any {
|
||||
t.Helper()
|
||||
models, ok := data["models"].([]map[string]any)
|
||||
require.True(t, ok, "models missing from report payload")
|
||||
return models
|
||||
}
|
||||
|
||||
func TestReportUnpricedAIModels_ConcurrentReplicas(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ctx, logger, db, _, notifEnq, clk := setup(t)
|
||||
seedOwner(t, db)
|
||||
initiator := dbgen.User(t, db, database.User{})
|
||||
provider := seedProvider(t, db, "anthropic", database.AIProviderTypeAnthropic)
|
||||
seedUnpricedModelUsage(t, db, initiator, provider, "claude-opus-4-8", clk.Now(), 100)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for range 2 {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
runReport(ctx, logger, db, database.LockIDNotifyUnpricedAIModels, "unpriced AI models",
|
||||
func(tx database.Store) error {
|
||||
return reportUnpricedAIModels(ctx, logger, tx, notifEnq, clk)
|
||||
})
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
require.Len(t, notifEnq.Sent(notificationstest.WithTemplateID(notifications.TemplateAIModelsUnpricedReport)), 1)
|
||||
}
|
||||
Vendored
+105
@@ -0,0 +1,105 @@
|
||||
From: system@coder.com
|
||||
To: bobby@coder.com
|
||||
Subject: Missing AI Model Prices
|
||||
Message-Id: 02ee4935-73be-4fa1-a290-ff9999026b13@blush-whale-48
|
||||
Date: Fri, 11 Oct 2024 09:03:06 +0000
|
||||
Content-Type: multipart/alternative; boundary=bbe61b741255b6098bb6b3c1f41b885773df633cb18d2a3002b68e4bc9c4
|
||||
MIME-Version: 1.0
|
||||
|
||||
--bbe61b741255b6098bb6b3c1f41b885773df633cb18d2a3002b68e4bc9c4
|
||||
Content-Transfer-Encoding: quoted-printable
|
||||
Content-Type: text/plain; charset=UTF-8
|
||||
|
||||
Hi Bobby,
|
||||
|
||||
These models were used in the last week, but they have no price, so their u=
|
||||
sage is missing from AI spend and does not count toward any AI budget. Repo=
|
||||
rted spend is lower than actual, and a user who calls only these models has=
|
||||
no effective limit.
|
||||
|
||||
anthropic/claude-opus-4-8
|
||||
openai/gpt-5.7
|
||||
openrouter/z-ai/glm-5.4
|
||||
|
||||
3 of 15 models with no price are shown, ordered by usage.
|
||||
|
||||
Every Coder release ships with prices for most models, so only the models a=
|
||||
bove need one: see how spend is calculated (https://coder.com/docs/ai-coder=
|
||||
/ai-gateway/cost-controls#how-spend-is-calculated) and how to configure pri=
|
||||
ces (https://coder.com/docs/ai-coder/ai-gateway/cost-controls#configure-mod=
|
||||
el-prices). Prices are not retroactive, so usage recorded before you set a =
|
||||
price stays unpriced.
|
||||
|
||||
|
||||
--bbe61b741255b6098bb6b3c1f41b885773df633cb18d2a3002b68e4bc9c4
|
||||
Content-Transfer-Encoding: quoted-printable
|
||||
Content-Type: text/html; charset=UTF-8
|
||||
|
||||
<!doctype html>
|
||||
<html lang=3D"en">
|
||||
<head>
|
||||
<meta charset=3D"UTF-8" />
|
||||
<meta name=3D"viewport" content=3D"width=3Ddevice-width, initial-scale=
|
||||
=3D1.0" />
|
||||
<title>Missing AI Model Prices</title>
|
||||
</head>
|
||||
<body style=3D"margin: 0; padding: 0; font-family: -apple-system, system-=
|
||||
ui, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen', 'Ubuntu', 'Cantarel=
|
||||
l', 'Fira Sans', 'Droid Sans', 'Helvetica Neue', sans-serif; color: #020617=
|
||||
; background: #f8fafc;">
|
||||
<div style=3D"max-width: 600px; margin: 20px auto; padding: 60px; borde=
|
||||
r: 1px solid #e2e8f0; border-radius: 8px; background-color: #fff; text-alig=
|
||||
n: left; font-size: 14px; line-height: 1.5;">
|
||||
<div style=3D"text-align: center;">
|
||||
<img src=3D"https://coder.com/coder-logo-horizontal.png" alt=3D"Cod=
|
||||
er Logo" style=3D"height: 40px;" />
|
||||
</div>
|
||||
<h1 style=3D"text-align: center; font-size: 24px; font-weight: 400; m=
|
||||
argin: 8px 0 32px; line-height: 1.5;">
|
||||
Missing AI Model Prices
|
||||
</h1>
|
||||
<div style=3D"line-height: 1.5;">
|
||||
<p>Hi Bobby,</p>
|
||||
<p>These models were used in the last week, but they have no price,=
|
||||
so their usage is missing from AI spend and does not count toward any AI b=
|
||||
udget. Reported spend is lower than actual, and a user who calls only these=
|
||||
models has no effective limit.</p>
|
||||
|
||||
<ul>
|
||||
<li>anthropic/claude-opus-4-8<br>
|
||||
</li>
|
||||
<li>openai/gpt-5.7<br>
|
||||
</li>
|
||||
<li>openrouter/z-ai/glm-5.4<br>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
<p>3 of 15 models with no price are shown, ordered by usage.</p>
|
||||
|
||||
<p>Every Coder release ships with prices for most models, so only the model=
|
||||
s above need one: see <a href=3D"https://coder.com/docs/ai-coder/ai-gateway=
|
||||
/cost-controls#how-spend-is-calculated">how spend is calculated</a> and <a =
|
||||
href=3D"https://coder.com/docs/ai-coder/ai-gateway/cost-controls#configure-=
|
||||
model-prices">how to configure prices</a>. Prices are not retroactive, so u=
|
||||
sage recorded before you set a price stays unpriced.</p>
|
||||
</div>
|
||||
<div style=3D"text-align: center; margin-top: 32px;">
|
||||
=20
|
||||
</div>
|
||||
<div style=3D"border-top: 1px solid #e2e8f0; color: #475569; font-siz=
|
||||
e: 12px; margin-top: 64px; padding-top: 24px; line-height: 1.6;">
|
||||
<p>© 2024 Coder. All rights reserved - <a =
|
||||
href=3D"http://test.com" style=3D"color: #2563eb; text-decoration: none;">h=
|
||||
ttp://test.com</a></p>
|
||||
<p><a href=3D"http://test.com/settings/notifications" style=3D"colo=
|
||||
r: #2563eb; text-decoration: none;">Click here to manage your notification =
|
||||
settings</a></p>
|
||||
<p><a href=3D"http://test.com/settings/notifications?disabled=3D1b7=
|
||||
d9fa7-f5a8-4e46-8078-5cf53abfed94" style=3D"color: #2563eb; text-decoration=
|
||||
: none;">Stop receiving emails like this</a></p>
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
--bbe61b741255b6098bb6b3c1f41b885773df633cb18d2a3002b68e4bc9c4--
|
||||
Vendored
+39
@@ -0,0 +1,39 @@
|
||||
{
|
||||
"_version": "1.1",
|
||||
"msg_id": "00000000-0000-0000-0000-000000000000",
|
||||
"payload": {
|
||||
"_version": "1.2",
|
||||
"notification_name": "Missing AI Model Prices",
|
||||
"notification_template_id": "00000000-0000-0000-0000-000000000000",
|
||||
"user_id": "00000000-0000-0000-0000-000000000000",
|
||||
"user_email": "bobby@coder.com",
|
||||
"user_name": "Bobby",
|
||||
"user_username": "bobby",
|
||||
"actions": [],
|
||||
"labels": {},
|
||||
"data": {
|
||||
"models": [
|
||||
{
|
||||
"model": "claude-opus-4-8",
|
||||
"provider": "anthropic"
|
||||
},
|
||||
{
|
||||
"model": "gpt-5.7",
|
||||
"provider": "openai"
|
||||
},
|
||||
{
|
||||
"model": "z-ai/glm-5.4",
|
||||
"provider": "openrouter"
|
||||
}
|
||||
],
|
||||
"report_frequency": "week",
|
||||
"total_count": 15,
|
||||
"truncated": true
|
||||
},
|
||||
"targets": null
|
||||
},
|
||||
"title": "Missing AI Model Prices",
|
||||
"title_markdown": "Missing AI Model Prices",
|
||||
"body": "These models were used in the last week, but they have no price, so their usage is missing from AI spend and does not count toward any AI budget. Reported spend is lower than actual, and a user who calls only these models has no effective limit.\n\nanthropic/claude-opus-4-8\nopenai/gpt-5.7\nopenrouter/z-ai/glm-5.4\n\n3 of 15 models with no price are shown, ordered by usage.\n\nEvery Coder release ships with prices for most models, so only the models above need one: see how spend is calculated (https://coder.com/docs/ai-coder/ai-gateway/cost-controls#how-spend-is-calculated) and how to configure prices (https://coder.com/docs/ai-coder/ai-gateway/cost-controls#configure-model-prices). Prices are not retroactive, so usage recorded before you set a price stays unpriced.",
|
||||
"body_markdown": "These models were used in the last week, but they have no price, so their usage is missing from AI spend and does not count toward any AI budget. Reported spend is lower than actual, and a user who calls only these models has no effective limit.\n\n\n* anthropic/claude-opus-4-8\n* openai/gpt-5.7\n* openrouter/z-ai/glm-5.4\n\n3 of 15 models with no price are shown, ordered by usage.\n\nEvery Coder release ships with prices for most models, so only the models above need one: see [how spend is calculated](https://coder.com/docs/ai-coder/ai-gateway/cost-controls#how-spend-is-calculated) and [how to configure prices](https://coder.com/docs/ai-coder/ai-gateway/cost-controls#configure-model-prices). Prices are not retroactive, so usage recorded before you set a price stays unpriced."
|
||||
}
|
||||
Reference in New Issue
Block a user