From 95328f1ead6bdf275664678a92033a771c8a8db1 Mon Sep 17 00:00:00 2001 From: Susana Ferreira Date: Mon, 17 Aug 2026 14:08:28 +0100 Subject: [PATCH] fix: label unpriced token usage metric by provider name and type (#28210) ## Problem The `provider` label was inconsistent between AI Gateway metrics. Every metric emitted by the gateway labels `provider` with the provider instance name, for example `anthropic-eu`, while `coder_ai_gateway_cost_control_unpriced_token_usage_records_total` used the provider type, for example `anthropic`. The two could not be correlated on `provider`. The metric was also inconsistent with itself: the path where a provider fails to resolve labelled by instance name, and the path where a model has no price labelled by type. The type is still worth exposing, since prices are keyed on `(provider_type, model)` and that is what an operator needs to add a price. ## Changes - Label the metric with `provider` (the instance name, consistent with the other gateway metrics) and add `provider_type` (the configured type the price is keyed on). - Use `unknown` for `provider_type` when the provider does not resolve to a configured type. - Log the unresolved-provider case at `warn` instead of `info`. A missing price is an expected steady state, but a provider that cannot be resolved is not. - Update the metrics docs and the `metricsdocgen` fixture. Closes [AIGOV-574](https://linear.app/codercom/issue/AIGOV-574) > [!NOTE] > Initially generated by Claude Opus 5, modified and reviewed by @ssncferreira --- coderd/aibridgedserver/aibridgedserver_test.go | 8 ++++---- coderd/aibridgedserver/cost.go | 10 +++++++--- coderd/aibridgedserver/metrics.go | 5 +++-- docs/admin/integrations/prometheus.md | 2 +- docs/ai-coder/ai-gateway/cost-controls.md | 9 +++++---- docs/ai-coder/ai-gateway/monitoring.md | 12 ++++++------ scripts/metricsdocgen/metrics | 4 ++-- 7 files changed, 28 insertions(+), 22 deletions(-) diff --git a/coderd/aibridgedserver/aibridgedserver_test.go b/coderd/aibridgedserver/aibridgedserver_test.go index 42915b9589..31fcbddd70 100644 --- a/coderd/aibridgedserver/aibridgedserver_test.go +++ b/coderd/aibridgedserver/aibridgedserver_test.go @@ -1763,7 +1763,7 @@ func TestRecordTokenUsage(t *testing.T) { // A priced model does not increment unpriced_token_usage_records_total. assertMetrics: func(t *testing.T, reg *prometheus.Registry) { require.Nil(t, promhelp.MetricValue(t, reg, "cost_control_unpriced_token_usage_records_total", - prometheus.Labels{"provider": "anthropic", "model": "claude-sonnet-4-6"})) + prometheus.Labels{"provider": "anthropic-eu", "provider_type": "anthropic", "model": "claude-sonnet-4-6"})) }, }, { @@ -1910,7 +1910,7 @@ func TestRecordTokenUsage(t *testing.T) { // A missing price row increments unpriced_token_usage_records_total. assertMetrics: func(t *testing.T, reg *prometheus.Registry) { require.Equal(t, 1, promhelp.CounterValue(t, reg, "cost_control_unpriced_token_usage_records_total", - prometheus.Labels{"provider": "anthropic", "model": "claude-sonnet-4-6"})) + prometheus.Labels{"provider": "anthropic-eu", "provider_type": "anthropic", "model": "claude-sonnet-4-6"})) }, }, { @@ -2159,7 +2159,7 @@ func TestRecordTokenUsage(t *testing.T) { // A missing price row increments unpriced_token_usage_records_total. assertMetrics: func(t *testing.T, reg *prometheus.Registry) { require.Equal(t, 1, promhelp.CounterValue(t, reg, "cost_control_unpriced_token_usage_records_total", - prometheus.Labels{"provider": "anthropic", "model": "claude-sonnet-4-6"})) + prometheus.Labels{"provider": "anthropic-eu", "provider_type": "anthropic", "model": "claude-sonnet-4-6"})) }, }, { @@ -2326,7 +2326,7 @@ func TestRecordTokenUsage(t *testing.T) { // The metric names the provider that failed to resolve. assertMetrics: func(t *testing.T, reg *prometheus.Registry) { require.Equal(t, 1, promhelp.CounterValue(t, reg, "cost_control_unpriced_token_usage_records_total", - prometheus.Labels{"provider": "anthropic-eu", "model": "claude-sonnet-4-6"})) + prometheus.Labels{"provider": "anthropic-eu", "provider_type": "unknown", "model": "claude-sonnet-4-6"})) }, }, { diff --git a/coderd/aibridgedserver/cost.go b/coderd/aibridgedserver/cost.go index 08b84aef6c..1a4b072971 100644 --- a/coderd/aibridgedserver/cost.go +++ b/coderd/aibridgedserver/cost.go @@ -37,6 +37,10 @@ var errTokenUsageOutOfRange = xerrors.New("reported token usage is out of range" // provider-reported token counts. var errCostOutOfRange = xerrors.New("computed cost is out of range") +// unknownProviderType labels a metric whose provider did not resolve to a +// configured type. +const unknownProviderType = "unknown" + // validateTokenUsage rejects an interception whose reported token counts fall // outside [0, maxAllowedTokenUsage]. func validateTokenUsage(in *proto.RecordTokenUsageRequest) error { @@ -109,10 +113,10 @@ func (s *Server) resolveTokenUsageCost(ctx context.Context, intc database.AIBrid switch { case errors.Is(err, sql.ErrNoRows): // Only reachable if the provider was deleted mid-request. - s.logger.Info(ctx, "no configured provider found for interception, recording token usage with NULL cost", + s.logger.Warn(ctx, "no configured provider found for interception, recording token usage with NULL cost", slog.F("provider_name", intc.ProviderName), slog.F("model", intc.Model)) if s.metrics != nil { - s.metrics.UnpricedTokenUsageRecords.WithLabelValues(intc.ProviderName, intc.Model).Inc() + s.metrics.UnpricedTokenUsageRecords.WithLabelValues(intc.ProviderName, unknownProviderType, intc.Model).Inc() } return result, nil case err != nil: @@ -131,7 +135,7 @@ func (s *Server) resolveTokenUsageCost(ctx context.Context, intc database.AIBrid s.logger.Info(ctx, "no price found for model, recording token usage with NULL cost", slog.F("provider", configuredType), slog.F("model", intc.Model)) if s.metrics != nil { - s.metrics.UnpricedTokenUsageRecords.WithLabelValues(configuredType, intc.Model).Inc() + s.metrics.UnpricedTokenUsageRecords.WithLabelValues(intc.ProviderName, configuredType, intc.Model).Inc() } return result, nil case err != nil: diff --git a/coderd/aibridgedserver/metrics.go b/coderd/aibridgedserver/metrics.go index b88b8789a6..c613b2f2f3 100644 --- a/coderd/aibridgedserver/metrics.go +++ b/coderd/aibridgedserver/metrics.go @@ -54,8 +54,9 @@ func NewMetrics(reg prometheus.Registerer) *Metrics { UnpricedTokenUsageRecords: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{ Subsystem: "cost_control", Name: "unpriced_token_usage_records_total", - Help: "The number of recorded AI token-usage records for which no (provider, model) price was found.", - }, []string{"provider", "model"}), + Help: "The number of recorded AI token-usage records for which no (provider_type, model) price was found. " + + "provider is the provider instance name, and provider_type is its configured type.", + }, []string{"provider", "provider_type", "model"}), // Pessimistic cardinality: 3 outcomes, 8 buckets + 3 extra series // (count, sum, +Inf) = up to 33. EnforcementDuration: promauto.With(reg).NewHistogramVec(prometheus.HistogramOpts{ diff --git a/docs/admin/integrations/prometheus.md b/docs/admin/integrations/prometheus.md index 0d580d68eb..5d8551d991 100644 --- a/docs/admin/integrations/prometheus.md +++ b/docs/admin/integrations/prometheus.md @@ -125,7 +125,7 @@ The `coder_ai_gateway_cost_control_*` metrics are exported only by `coderd`. | `coder_ai_gateway_cost_control_blocked_requests_total` | counter | The number of AI requests blocked because the initiator's budget was exceeded. | `group_id` | | `coder_ai_gateway_cost_control_blocked_users` | gauge | The number of users currently over their AI budget. | `group_id` | | `coder_ai_gateway_cost_control_enforcement_duration_seconds` | histogram | The duration of AI budget enforcement checks, in seconds (outcome: allowed, blocked, error). | `outcome` | -| `coder_ai_gateway_cost_control_unpriced_token_usage_records_total` | counter | The number of recorded AI token-usage records for which no (provider, model) price was found. | `model` `provider` | +| `coder_ai_gateway_cost_control_unpriced_token_usage_records_total` | counter | The number of recorded AI token-usage records for which no (provider_type, model) price was found. provider is the provider instance name, and provider_type is its configured type. | `model` `provider_type` `provider` | | `coder_ai_gateway_injected_tool_invocations_total` | counter | The number of times an injected MCP tool was invoked by AI Gateway. | `model` `name` `provider` `server` | | `coder_ai_gateway_interceptions_duration_seconds` | histogram | The total duration of intercepted requests, in seconds. The majority of this time will be the upstream processing of the request. AI Gateway has no control over upstream processing time, so it's just an illustrative metric. | `model` `provider` | | `coder_ai_gateway_interceptions_inflight` | gauge | The number of intercepted requests which are being processed. | `model` `provider` `route` | diff --git a/docs/ai-coder/ai-gateway/cost-controls.md b/docs/ai-coder/ai-gateway/cost-controls.md index 5e95b82644..741d6e5134 100644 --- a/docs/ai-coder/ai-gateway/cost-controls.md +++ b/docs/ai-coder/ai-gateway/cost-controls.md @@ -208,10 +208,11 @@ Replace `` with your Coder minor version, for example `2.36`. > effectively unlimited. Monitor `coder_ai_gateway_cost_control_unpriced_token_usage_records_total`, -labeled by `provider` and `model`, to detect unpriced usage. Any non-zero value -means spend is under-counted. Because the price book ships with the release, a -newly launched model is unpriced until you upgrade Coder or set a price for it -yourself. +labeled by `provider`, `provider_type`, and `model`, to detect unpriced usage. +Use the `(provider_type, model)` tuple to find the price to set. Any non-zero +value means spend is under-counted. Because the price book ships with the +release, a newly launched model is unpriced until you upgrade Coder or set a +price for it yourself. ### Set model prices diff --git a/docs/ai-coder/ai-gateway/monitoring.md b/docs/ai-coder/ai-gateway/monitoring.md index da45b643ee..937cf7360d 100644 --- a/docs/ai-coder/ai-gateway/monitoring.md +++ b/docs/ai-coder/ai-gateway/monitoring.md @@ -54,12 +54,12 @@ Budget enforcement runs in `coderd`. Cost control metrics are exported only from the `coderd` Prometheus listener. Standalone replicas do not export them. -| Metric | Type | Labels | Purpose | -|--------------------------------------------------------------------|-----------|---------------------|------------------------------------------------------------------------------------------| -| `coder_ai_gateway_cost_control_blocked_requests_total` | counter | `group_id` | AI requests blocked because the initiator's budget was exceeded. | -| `coder_ai_gateway_cost_control_blocked_users` | gauge | `group_id` | Users currently over their AI budget. | -| `coder_ai_gateway_cost_control_enforcement_duration_seconds` | histogram | `outcome` | Duration of AI budget enforcement checks. `outcome` is `allowed`, `blocked`, or `error`. | -| `coder_ai_gateway_cost_control_unpriced_token_usage_records_total` | counter | `model`, `provider` | Recorded token-usage records for which no model price was found. | +| Metric | Type | Labels | Purpose | +|--------------------------------------------------------------------|-----------|--------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `coder_ai_gateway_cost_control_blocked_requests_total` | counter | `group_id` | AI requests blocked because the initiator's budget was exceeded. | +| `coder_ai_gateway_cost_control_blocked_users` | gauge | `group_id` | Users currently over their AI budget. | +| `coder_ai_gateway_cost_control_enforcement_duration_seconds` | histogram | `outcome` | Duration of AI budget enforcement checks. `outcome` is `allowed`, `blocked`, or `error`. | +| `coder_ai_gateway_cost_control_unpriced_token_usage_records_total` | counter | `model`, `provider`, `provider_type` | Recorded token-usage records for which no model price was found. `provider` is the provider instance name, and `provider_type` is the configured type the price is keyed on, or `unknown` when the provider could not be resolved. | ### AI Gateway Proxy metrics diff --git a/scripts/metricsdocgen/metrics b/scripts/metricsdocgen/metrics index 755068744c..ccce769144 100644 --- a/scripts/metricsdocgen/metrics +++ b/scripts/metricsdocgen/metrics @@ -166,9 +166,9 @@ coder_ai_gateway_cost_control_enforcement_duration_seconds_bucket{outcome="allow coder_ai_gateway_cost_control_enforcement_duration_seconds_bucket{outcome="allowed",le="+Inf"} 0 coder_ai_gateway_cost_control_enforcement_duration_seconds_sum{outcome="allowed"} 0 coder_ai_gateway_cost_control_enforcement_duration_seconds_count{outcome="allowed"} 0 -# HELP coder_ai_gateway_cost_control_unpriced_token_usage_records_total The number of recorded AI token-usage records for which no (provider, model) price was found. +# HELP coder_ai_gateway_cost_control_unpriced_token_usage_records_total The number of recorded AI token-usage records for which no (provider_type, model) price was found. provider is the provider instance name, and provider_type is its configured type. # TYPE coder_ai_gateway_cost_control_unpriced_token_usage_records_total counter -coder_ai_gateway_cost_control_unpriced_token_usage_records_total{model="gpt-5-nano",provider="openai"} 0 +coder_ai_gateway_cost_control_unpriced_token_usage_records_total{model="gpt-5-nano",provider="openai",provider_type="openai"} 0 # HELP coder_ai_gateway_injected_tool_invocations_total The number of times an injected MCP tool was invoked by AI Gateway. # TYPE coder_ai_gateway_injected_tool_invocations_total counter coder_ai_gateway_injected_tool_invocations_total{model="gpt-5-nano",name="coder_list_templates",provider="openai",server="https://xxx.pit-1.try.coder.app/api/experimental/mcp/http"} 1