diff --git a/aibridge/metrics/metrics.go b/aibridge/metrics/metrics.go index ad75ad4c9c..3b95c56a78 100644 --- a/aibridge/metrics/metrics.go +++ b/aibridge/metrics/metrics.go @@ -68,7 +68,7 @@ func NewMetrics(reg prometheus.Registerer) *Metrics { Name: "duration_seconds", Help: "The total duration of intercepted requests, in seconds. " + "The majority of this time will be the upstream processing of the request. " + - "aibridge has no control over upstream processing time, so it's just an illustrative metric.", + "AI Gateway has no control over upstream processing time, so it's just an illustrative metric.", // TODO: add docs around determining aibridge's *own* latency with distributed traces // once https://github.com/coder/aibridge/issues/26 lands. Buckets: []float64{0.5, 2, 5, 15, 30, 60, 120}, @@ -106,7 +106,7 @@ func NewMetrics(reg prometheus.Registerer) *Metrics { InjectedToolUseCount: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{ Subsystem: "injected_tool_invocations", Name: "total", - Help: "The number of times an injected MCP tool was invoked by aibridge.", + Help: "The number of times an injected MCP tool was invoked by AI Gateway.", }, append(baseLabels, "server", "name")), // Pessimistic cardinality: 3 providers, 5 models, 30 tools = up to 450. NonInjectedToolUseCount: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{ diff --git a/cli/server.go b/cli/server.go index 7210ade859..b521f7b4b6 100644 --- a/cli/server.go +++ b/cli/server.go @@ -1121,7 +1121,10 @@ func (r *RootCmd) Server(newAPI func(context.Context, *coderd.Options) (*coderd. // unconditionally when the bridge feature is enabled by config so // chatd can use it regardless of license entitlement. if vals.AI.BridgeConfig.Enabled.Value() { - aibridgeReg := prometheus.WrapRegistererWithPrefix("coder_aibridged_", coderAPI.PrometheusRegistry) + // TODO(deprecation): Remove "coder_aibridged_" in v2.37. + // See AIGOV-447: + // https://linear.app/codercom/issue/AIGOV-447/remove-legacy-ai-gateway-metric-aliases + aibridgeReg := prometheusmetrics.NewMetricAliasRegisterer(coderAPI.PrometheusRegistry, "coder_ai_gateway_", "coder_aibridged_") aibridgeMetrics := aibridge.NewMetrics(aibridgeReg) aibridgeProviders, _, err := BuildProviders(aibridgeInitCtx, options.Database, vals.AI.BridgeConfig, logger.Named("aibridge.providers"), aibridgeMetrics) if err != nil { diff --git a/coderd/aibridged/metrics.go b/coderd/aibridged/metrics.go index b06a9c067c..1842afec21 100644 --- a/coderd/aibridged/metrics.go +++ b/coderd/aibridged/metrics.go @@ -46,7 +46,7 @@ func NewMetrics(reg prometheus.Registerer) *Metrics { ProvidersLastReloadSuccessTimestampSeconds: factory.NewGauge(prometheus.GaugeOpts{ Name: "providers_last_reload_success_timestamp_seconds", - Help: "Unix timestamp of the last provider reload that successfully refreshed the pool. A gap against coder_aibridged_providers_last_reload_timestamp_seconds means the loop is firing but the refresh function is failing.", + Help: "Unix timestamp of the last provider reload that successfully refreshed the pool. A gap against the providers_last_reload_timestamp_seconds gauge means the loop is firing but the refresh function is failing.", }), } } diff --git a/coderd/prometheusmetrics/metricalias.go b/coderd/prometheusmetrics/metricalias.go new file mode 100644 index 0000000000..97b3068c94 --- /dev/null +++ b/coderd/prometheusmetrics/metricalias.go @@ -0,0 +1,52 @@ +package prometheusmetrics + +import "github.com/prometheus/client_golang/prometheus" + +// metricAliasRegisterer exposes each collector under multiple prefixes. +type metricAliasRegisterer struct { + registerers []prometheus.Registerer +} + +// NewMetricAliasRegisterer exposes collectors under canonicalPrefix and each +// alias prefix. Every exported name reads from the same collector. Alias +// prefixes are typically deprecated names scheduled for removal; see each +// call site for the specific deprecation ticket. +func NewMetricAliasRegisterer(base prometheus.Registerer, canonicalPrefix string, aliasPrefixes ...string) prometheus.Registerer { + prefixes := append([]string{canonicalPrefix}, aliasPrefixes...) + registerers := make([]prometheus.Registerer, 0, len(prefixes)) + for _, prefix := range prefixes { + registerers = append(registerers, prometheus.WrapRegistererWithPrefix(prefix, base)) + } + return &metricAliasRegisterer{registerers: registerers} +} + +// Register registers c under each prefix and rolls back on failure. +func (m *metricAliasRegisterer) Register(c prometheus.Collector) error { + for i, registerer := range m.registerers { + if err := registerer.Register(c); err != nil { + for _, registered := range m.registerers[:i] { + registered.Unregister(c) + } + return err + } + } + return nil +} + +// MustRegister registers collectors and panics on the first failure. +func (m *metricAliasRegisterer) MustRegister(cs ...prometheus.Collector) { + for _, c := range cs { + if err := m.Register(c); err != nil { + panic(err) + } + } +} + +// Unregister removes c from every prefix. +func (m *metricAliasRegisterer) Unregister(c prometheus.Collector) bool { + ok := true + for _, registerer := range m.registerers { + ok = registerer.Unregister(c) && ok + } + return ok +} diff --git a/coderd/prometheusmetrics/metricalias_test.go b/coderd/prometheusmetrics/metricalias_test.go new file mode 100644 index 0000000000..59e981fec1 --- /dev/null +++ b/coderd/prometheusmetrics/metricalias_test.go @@ -0,0 +1,143 @@ +package prometheusmetrics_test + +import ( + "testing" + + "github.com/prometheus/client_golang/prometheus" + io_prometheus_client "github.com/prometheus/client_model/go" + "github.com/stretchr/testify/require" + + "github.com/coder/coder/v2/coderd/prometheusmetrics" +) + +func TestMetricAliasRegisterer(t *testing.T) { + t.Parallel() + + t.Run("EmitsCanonicalAndAliases", func(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + suffix string + register func(prometheus.Registerer) + }{ + { + name: "counter_vec", + suffix: "requests_total", + register: func(reg prometheus.Registerer) { + counter := prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "requests_total", + Help: "Total requests.", + }, []string{"route"}) + reg.MustRegister(counter) + counter.WithLabelValues("/api").Add(3) + }, + }, + { + name: "gauge", + suffix: "inflight_requests", + register: func(reg prometheus.Registerer) { + gauge := prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "inflight_requests", + Help: "Inflight requests.", + }) + reg.MustRegister(gauge) + gauge.Set(7) + }, + }, + { + name: "histogram_vec", + suffix: "request_duration_seconds", + register: func(reg prometheus.Registerer) { + histogram := prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Name: "request_duration_seconds", + Help: "Request duration.", + Buckets: []float64{1, 5}, + }, []string{"route"}) + reg.MustRegister(histogram) + histogram.WithLabelValues("/api").Observe(3) + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + base := prometheus.NewRegistry() + prefixes := []string{"canonical_", "alias_one_", "alias_two_"} + reg := prometheusmetrics.NewMetricAliasRegisterer(base, prefixes[0], prefixes[1:]...) + + tc.register(reg) + + families, err := base.Gather() + require.NoError(t, err) + + canonical := prefixes[0] + tc.suffix + for _, aliasPrefix := range prefixes[1:] { + assertParity(t, families, canonical, aliasPrefix+tc.suffix) + } + require.Len(t, families, len(prefixes)) + }) + } + }) + + t.Run("RegisterRollsBackPartialFailure", func(t *testing.T) { + t.Parallel() + + base := prometheus.NewRegistry() + counter := prometheus.NewCounter(prometheus.CounterOpts{ + Name: "requests_total", + Help: "Total requests.", + }) + prometheus.WrapRegistererWithPrefix("alias_two_", base).MustRegister(counter) + + reg := prometheusmetrics.NewMetricAliasRegisterer(base, "canonical_", "alias_one_", "alias_two_") + err := reg.Register(counter) + require.Error(t, err) + + families, err := base.Gather() + require.NoError(t, err) + require.Len(t, families, 1) + require.Equal(t, "alias_two_requests_total", families[0].GetName()) + }) + + t.Run("Unregister", func(t *testing.T) { + t.Parallel() + + base := prometheus.NewRegistry() + reg := prometheusmetrics.NewMetricAliasRegisterer(base, "canonical_", "alias_one_", "alias_two_") + + counter := prometheus.NewCounter(prometheus.CounterOpts{ + Name: "requests_total", + Help: "Total requests.", + }) + reg.MustRegister(counter) + + require.True(t, reg.Unregister(counter)) + + families, err := base.Gather() + require.NoError(t, err) + require.Empty(t, families) + }) +} + +func assertParity(t *testing.T, families []*io_prometheus_client.MetricFamily, canonical, alias string) { + t.Helper() + canonicalFamily := findMetricFamily(t, families, canonical) + aliasFamily := findMetricFamily(t, families, alias) + require.Equal(t, canonicalFamily.GetType(), aliasFamily.GetType()) + require.Equal(t, canonicalFamily.GetHelp(), aliasFamily.GetHelp()) + require.Equal(t, canonicalFamily.GetMetric(), aliasFamily.GetMetric()) +} + +func findMetricFamily(t *testing.T, families []*io_prometheus_client.MetricFamily, name string) *io_prometheus_client.MetricFamily { + t.Helper() + for _, family := range families { + if family.GetName() == name { + return family + } + } + require.Failf(t, "metric family not found", "missing metric family %q", name) + return nil +} diff --git a/docs/admin/integrations/prometheus.md b/docs/admin/integrations/prometheus.md index 92fbc1d812..8df2126633 100644 --- a/docs/admin/integrations/prometheus.md +++ b/docs/admin/integrations/prometheus.md @@ -104,219 +104,223 @@ deployment. They will always be available from the agent. -| Name | Type | Description | Labels | -|-------------------------------------------------------------------------|-----------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------| -| `agent_boundary_log_proxy_batches_dropped_total` | counter | Total number of boundary log batches dropped before reaching coderd. Reason: buffer_full = the agent's internal buffer is full, meaning boundary is producing logs faster than the agent can forward them to coderd; forward_failed = the agent failed to send the batch to coderd, potentially because coderd is unreachable or the connection was interrupted. | `reason` | -| `agent_boundary_log_proxy_batches_forwarded_total` | counter | Total number of boundary log batches successfully forwarded to coderd. Compare with batches_dropped_total to compute a drop rate. | | -| `agent_boundary_log_proxy_logs_dropped_total` | counter | Total number of individual boundary log entries dropped before reaching coderd. Reason: buffer_full = the agent's internal buffer is full; forward_failed = the agent failed to send the batch to coderd; boundary_channel_full = boundary's internal send channel overflowed, meaning boundary is generating logs faster than it can batch and send them; boundary_batch_full = boundary's outgoing batch buffer overflowed after a failed flush, meaning boundary could not write to the agent's socket. | `reason` | -| `agent_scripts_executed_total` | counter | Total number of scripts executed by the Coder agent. Includes cron scheduled scripts. | `agent_name` `success` `template_name` `username` `workspace_name` | -| `coder_aibridged_circuit_breaker_rejects_total` | counter | Total number of requests rejected due to open circuit breaker. | `endpoint` `model` `provider` | -| `coder_aibridged_circuit_breaker_state` | gauge | Current state of the circuit breaker (0=closed, 0.5=half-open, 1=open). | `endpoint` `model` `provider` | -| `coder_aibridged_circuit_breaker_trips_total` | counter | Total number of times the circuit breaker transitioned to open state. | `endpoint` `model` `provider` | -| `coder_aibridged_injected_tool_invocations_total` | counter | The number of times an injected MCP tool was invoked by aibridge. | `model` `name` `provider` `server` | -| `coder_aibridged_interceptions_duration_seconds` | histogram | The total duration of intercepted requests, in seconds. The majority of this time will be the upstream processing of the request. aibridge has no control over upstream processing time, so it's just an illustrative metric. | `model` `provider` | -| `coder_aibridged_interceptions_inflight` | gauge | The number of intercepted requests which are being processed. | `model` `provider` `route` | -| `coder_aibridged_interceptions_total` | counter | The count of intercepted requests. | `initiator_id` `method` `model` `provider` `route` `status` | -| `coder_aibridged_non_injected_tool_selections_total` | counter | The number of times an AI model selected a tool to be invoked by the client. | `model` `name` `provider` | -| `coder_aibridged_passthrough_total` | counter | The count of requests which were not intercepted but passed through to the upstream. | `method` `provider` `route` | -| `coder_aibridged_prompts_total` | counter | The number of prompts issued by users (initiators). | `initiator_id` `model` `provider` | -| `coder_aibridged_provider_info` | gauge | One series per configured AI provider. Value is always 1; the status label (enabled, disabled, error) carries the alertable signal. | `provider_name` `provider_type` `status` | -| `coder_aibridged_providers_last_reload_success_timestamp_seconds` | gauge | Unix timestamp of the last provider reload that successfully refreshed the pool. A gap against coder_aibridged_providers_last_reload_timestamp_seconds means the loop is firing but the refresh function is failing. | | -| `coder_aibridged_providers_last_reload_timestamp_seconds` | gauge | Unix timestamp of the last provider reload attempt, success or failure. | | -| `coder_aibridged_tokens_total` | counter | The number of tokens used by intercepted requests. | `initiator_id` `model` `provider` `type` | -| `coder_aibridgeproxyd_connect_sessions_total` | counter | Total number of CONNECT sessions established. | `type` | -| `coder_aibridgeproxyd_inflight_mitm_requests` | gauge | Number of MITM requests currently being processed. | `provider` | -| `coder_aibridgeproxyd_mitm_requests_total` | counter | Total number of MITM requests handled by the proxy. | `provider` | -| `coder_aibridgeproxyd_mitm_responses_total` | counter | Total number of MITM responses by HTTP status code class. | `code` `provider` | -| `coder_aibridgeproxyd_provider_info` | gauge | One series per configured AI provider. Value is always 1; the status label (enabled, disabled, error) carries the alertable signal. | `provider_name` `provider_type` `status` | -| `coder_aibridgeproxyd_providers_last_reload_success_timestamp_seconds` | gauge | Unix timestamp of the last provider reload that successfully refreshed the router. A gap against coder_aibridgeproxyd_providers_last_reload_timestamp_seconds means the loop is firing but the refresh function is failing. | | -| `coder_aibridgeproxyd_providers_last_reload_timestamp_seconds` | gauge | Unix timestamp of the last provider reload attempt, success or failure. | | -| `coder_derp_server_accepts_total` | counter | Total DERP connections accepted. | | -| `coder_derp_server_average_queue_duration_ms` | gauge | Average queue duration in milliseconds. | | -| `coder_derp_server_bytes_received_total` | counter | Total bytes received. | | -| `coder_derp_server_bytes_sent_total` | counter | Total bytes sent. | | -| `coder_derp_server_clients` | gauge | Total clients (local + remote). | | -| `coder_derp_server_clients_local` | gauge | Local clients. | | -| `coder_derp_server_clients_remote` | gauge | Remote (mesh) clients. | | -| `coder_derp_server_connections` | gauge | Current DERP connections. | | -| `coder_derp_server_got_ping_total` | counter | Total pings received. | | -| `coder_derp_server_home_connections` | gauge | Current home DERP connections. | | -| `coder_derp_server_home_moves_in_total` | counter | Total home moves in. | | -| `coder_derp_server_home_moves_out_total` | counter | Total home moves out. | | -| `coder_derp_server_packets_dropped_reason_total` | counter | Packets dropped by reason. | `reason` | -| `coder_derp_server_packets_dropped_total` | counter | Total packets dropped. | | -| `coder_derp_server_packets_dropped_type_total` | counter | Packets dropped by type. | `type` | -| `coder_derp_server_packets_forwarded_in_total` | counter | Total packets forwarded in from mesh peers. | | -| `coder_derp_server_packets_forwarded_out_total` | counter | Total packets forwarded out to mesh peers. | | -| `coder_derp_server_packets_received_kind_total` | counter | Packets received by kind. | `kind` | -| `coder_derp_server_packets_received_total` | counter | Total packets received. | | -| `coder_derp_server_packets_sent_total` | counter | Total packets sent. | | -| `coder_derp_server_peer_gone_disconnected_total` | counter | Total peer gone (disconnected) frames sent. | | -| `coder_derp_server_peer_gone_not_here_total` | counter | Total peer gone (not here) frames sent. | | -| `coder_derp_server_sent_pong_total` | counter | Total pongs sent. | | -| `coder_derp_server_unknown_frames_total` | counter | Total unknown frames received. | | -| `coder_derp_server_watchers` | gauge | Current watchers. | | -| `coder_pubsub_connected` | gauge | Whether we are connected (1) or not connected (0) to postgres | | -| `coder_pubsub_current_events` | gauge | The current number of pubsub event channels listened for | | -| `coder_pubsub_current_subscribers` | gauge | The current number of active pubsub subscribers | | -| `coder_pubsub_disconnections_total` | counter | Total number of times we disconnected unexpectedly from postgres | | -| `coder_pubsub_latency_measure_errs_total` | counter | The number of pubsub latency measurement failures | | -| `coder_pubsub_latency_measures_total` | counter | The number of pubsub latency measurements | | -| `coder_pubsub_messages_total` | counter | Total number of messages received from postgres | `size` | -| `coder_pubsub_published_bytes_total` | counter | Total number of bytes successfully published across all publishes | | -| `coder_pubsub_publishes_total` | counter | Total number of calls to Publish | `success` | -| `coder_pubsub_receive_latency_seconds` | gauge | The time taken to receive a message from a pubsub event channel | | -| `coder_pubsub_received_bytes_total` | counter | Total number of bytes received across all messages | | -| `coder_pubsub_send_latency_seconds` | gauge | The time taken to send a message into a pubsub event channel | | -| `coder_pubsub_subscribes_total` | counter | Total number of calls to Subscribe/SubscribeWithErr | `success` | -| `coder_servertailnet_connections_total` | counter | Total number of TCP connections made to workspace agents. | `network` | -| `coder_servertailnet_open_connections` | gauge | Total number of TCP connections currently open to workspace agents. | `network` | -| `coderd_agentapi_metadata_batch_size` | histogram | Total number of metadata entries in each batch, updated before flushes. | | -| `coderd_agentapi_metadata_batch_utilization` | histogram | Number of metadata keys per agent in each batch, updated before flushes. | | -| `coderd_agentapi_metadata_batches_total` | counter | Total number of metadata batches flushed. | `reason` | -| `coderd_agentapi_metadata_dropped_keys_total` | counter | Total number of metadata keys dropped due to capacity limits. | | -| `coderd_agentapi_metadata_flush_duration_seconds` | histogram | Time taken to flush metadata batch to database and pubsub. | `reason` | -| `coderd_agentapi_metadata_flushed_total` | counter | Total number of unique metadatas flushed. | | -| `coderd_agentapi_metadata_publish_errors_total` | counter | Total number of metadata batch pubsub publish calls that have resulted in an error. | | -| `coderd_agents_apps` | gauge | Agent applications with statuses. | `agent_name` `app_name` `health` `username` `workspace_name` | -| `coderd_agents_connection_latencies_seconds` | gauge | Agent connection latencies in seconds. | `agent_name` `derp_region` `preferred` `username` `workspace_name` | -| `coderd_agents_connections` | gauge | Agent connections with statuses. | `agent_name` `lifecycle_state` `status` `tailnet_node` `username` `workspace_name` | -| `coderd_agents_first_connection_seconds` | histogram | Duration from agent creation to first connection in seconds. | `agent_name` `template_name` | -| `coderd_agents_up` | gauge | The number of active agents per workspace. | `template_name` `template_version` `username` `workspace_name` | -| `coderd_agentstats_connection_count` | gauge | The number of established connections by agent | `agent_name` `username` `workspace_name` | -| `coderd_agentstats_connection_median_latency_seconds` | gauge | The median agent connection latency | `agent_name` `username` `workspace_name` | -| `coderd_agentstats_currently_reachable_peers` | gauge | The number of peers (e.g. clients) that are currently reachable over the encrypted network. | `agent_name` `connection_type` `template_name` `username` `workspace_name` | -| `coderd_agentstats_rx_bytes` | gauge | Agent Rx bytes | `agent_name` `username` `workspace_name` | -| `coderd_agentstats_session_count_jetbrains` | gauge | The number of session established by JetBrains | `agent_name` `username` `workspace_name` | -| `coderd_agentstats_session_count_reconnecting_pty` | gauge | The number of session established by reconnecting PTY | `agent_name` `username` `workspace_name` | -| `coderd_agentstats_session_count_ssh` | gauge | The number of session established by SSH | `agent_name` `username` `workspace_name` | -| `coderd_agentstats_session_count_vscode` | gauge | The number of session established by VSCode | `agent_name` `username` `workspace_name` | -| `coderd_agentstats_startup_script_seconds` | gauge | The number of seconds the startup script took to execute. | `agent_name` `success` `template_name` `username` `workspace_name` | -| `coderd_agentstats_tx_bytes` | gauge | Agent Tx bytes | `agent_name` `username` `workspace_name` | -| `coderd_api_active_users_duration_hour` | gauge | The number of users that have been active within the last hour. | | -| `coderd_api_concurrent_requests` | gauge | The number of concurrent API requests. | `method` `path` | -| `coderd_api_concurrent_websockets` | gauge | The total number of concurrent API websockets. | `path` | -| `coderd_api_request_latencies_seconds` | histogram | Latency distribution of requests in seconds. | `method` `path` | -| `coderd_api_requests_processed_total` | counter | The total number of processed API requests | `code` `method` `path` | -| `coderd_api_total_user_count` | gauge | The total number of registered users, partitioned by status. | `status` | -| `coderd_api_websocket_durations_seconds` | histogram | Websocket duration distribution of requests in seconds. | `path` | -| `coderd_api_websocket_probes_total` | counter | WebSocket liveness probe outcomes by route. Compare rate(...{result="ok"}[1m]) against coderd_api_concurrent_websockets to detect unresponsive WebSocket connections. | `path` `result` | -| `coderd_api_workspace_latest_build` | gauge | The current number of workspace builds by status for all non-deleted workspaces. | `status` | -| `coderd_authz_authorize_duration_seconds` | histogram | Duration of the 'Authorize' call in seconds. Only counts calls that succeed. | `allowed` | -| `coderd_authz_prepare_authorize_duration_seconds` | histogram | Duration of the 'PrepareAuthorize' call in seconds. | | -| `coderd_build_info` | gauge | Describes the current build/version of the Coder server. Value is always 1. | `revision` `version` | -| `coderd_chat_auto_archive_records_archived_total` | counter | Total number of chats archived by the auto-archive job (counting both roots and cascaded children). | | -| `coderd_chatd_chats` | gauge | Number of chats being processed, by state. | `state` | -| `coderd_chatd_compaction_total` | counter | Total compaction outcomes (only recorded when compaction was triggered or failed). | `model` `provider` `result` | -| `coderd_chatd_message_count` | histogram | Number of messages in the prompt per LLM request. | `model` `provider` | -| `coderd_chatd_prompt_size_bytes` | histogram | Estimated byte size of the prompt per LLM request. | `model` `provider` | -| `coderd_chatd_steps_total` | counter | Total agentic loop steps across all chats. | `model` `provider` | -| `coderd_chatd_stream_buffer_dropped_total` | counter | Number of chat stream buffer events dropped due to the per-chat buffer cap. | | -| `coderd_chatd_stream_retries_total` | counter | Total LLM stream retries. | `chain_broken` `kind` `model` `provider` | -| `coderd_chatd_tool_errors_total` | counter | Total tool calls that returned an error result. | `model` `provider` `tool_name` | -| `coderd_chatd_tool_result_size_bytes` | histogram | Size in bytes of each tool execution result. | `model` `provider` `tool_name` | -| `coderd_chatd_ttft_seconds` | histogram | Time-to-first-token: wall time from LLM request to first streamed chunk. | `model` `provider` | -| `coderd_db_query_counts_total` | counter | Total number of queries labelled by HTTP route, method, and query name. | `method` `query` `route` | -| `coderd_db_query_latencies_seconds` | histogram | Latency distribution of queries in seconds. | `query` | -| `coderd_db_tx_duration_seconds` | histogram | Duration of transactions in seconds. | `success` `tx_id` | -| `coderd_db_tx_executions_count` | counter | Total count of transactions executed. 'retries' is expected to be 0 for a successful transaction. | `retries` `success` `tx_id` | -| `coderd_dbpurge_iteration_duration_seconds` | histogram | Duration of each dbpurge iteration in seconds. | `success` | -| `coderd_dbpurge_records_purged_total` | counter | Total number of records purged by type. | `record_type` | -| `coderd_experiments` | gauge | Indicates whether each experiment is enabled (1) or not (0) | `experiment` | -| `coderd_insights_applications_usage_seconds` | gauge | The application usage per template. | `application_name` `organization_name` `slug` `template_name` | -| `coderd_insights_parameters` | gauge | The parameter usage per template. | `organization_name` `parameter_name` `parameter_type` `parameter_value` `template_name` | -| `coderd_insights_templates_active_users` | gauge | The number of active users of the template. | `organization_name` `template_name` | -| `coderd_license_active_users` | gauge | The number of active users. | | -| `coderd_license_errors` | gauge | The number of active license errors. | | -| `coderd_license_limit_users` | gauge | The user seats limit based on the active Coder license. | | -| `coderd_license_user_limit_enabled` | gauge | Returns 1 if the current license enforces the user limit. | | -| `coderd_license_warnings` | gauge | The number of active license warnings. | | -| `coderd_lifecycle_autobuild_execution_duration_seconds` | histogram | Duration of each autobuild execution. | | -| `coderd_notifications_dispatcher_send_seconds` | histogram | The time taken to dispatch notifications. | `method` | -| `coderd_notifications_inflight_dispatches` | gauge | The number of dispatch attempts which are currently in progress. | `method` `notification_template_id` | -| `coderd_notifications_pending_updates` | gauge | The number of dispatch attempt results waiting to be flushed to the store. | | -| `coderd_notifications_queued_seconds` | histogram | The time elapsed between a notification being enqueued in the store and retrieved for dispatching (measures the latency of the notifications system). This should generally be within CODER_NOTIFICATIONS_FETCH_INTERVAL seconds; higher values for a sustained period indicates delayed processing and CODER_NOTIFICATIONS_LEASE_COUNT can be increased to accommodate this. | `method` | -| `coderd_notifications_retry_count` | counter | The count of notification dispatch retry attempts. | `method` `notification_template_id` | -| `coderd_notifications_synced_updates_total` | counter | The number of dispatch attempt results flushed to the store. | | -| `coderd_oauth2_external_requests_rate_limit` | gauge | The total number of allowed requests per interval. | `name` `resource` | -| `coderd_oauth2_external_requests_rate_limit_next_reset_unix` | gauge | Unix timestamp for when the next interval starts | `name` `resource` | -| `coderd_oauth2_external_requests_rate_limit_remaining` | gauge | The remaining number of allowed requests in this interval. | `name` `resource` | -| `coderd_oauth2_external_requests_rate_limit_reset_in_seconds` | gauge | Seconds until the next interval | `name` `resource` | -| `coderd_oauth2_external_requests_rate_limit_used` | gauge | The number of requests made in this interval. | `name` `resource` | -| `coderd_oauth2_external_requests_total` | counter | The total number of api calls made to external oauth2 providers. 'status_code' will be 0 if the request failed with no response. | `name` `source` `status_code` | -| `coderd_open_file_refs_current` | gauge | The count of file references currently open in the file cache. Multiple references can be held for the same file. | | -| `coderd_open_file_refs_total` | counter | The total number of file references ever opened in the file cache. The 'hit' label indicates if the file was loaded from the cache. | `hit` | -| `coderd_open_files_current` | gauge | The count of unique files currently open in the file cache. | | -| `coderd_open_files_size_bytes_current` | gauge | The current amount of memory of all files currently open in the file cache. | | -| `coderd_open_files_size_bytes_total` | counter | The total amount of memory ever opened in the file cache. This number never decrements. | | -| `coderd_open_files_total` | counter | The total count of unique files ever opened in the file cache. | | -| `coderd_prebuilds_reconciliation_duration_seconds` | histogram | Duration of each prebuilds reconciliation cycle. | | -| `coderd_prebuilt_workspace_claim_duration_seconds` | histogram | Time to claim a prebuilt workspace by organization, template, and preset. | `organization_name` `preset_name` `template_name` | -| `coderd_prebuilt_workspaces_claimed_total` | counter | Total number of prebuilt workspaces which were claimed by users. Claiming refers to creating a workspace with a preset selected for which eligible prebuilt workspaces are available and one is reassigned to a user. | `organization_name` `preset_name` `template_name` | -| `coderd_prebuilt_workspaces_created_total` | counter | Total number of prebuilt workspaces that have been created to meet the desired instance count of each template preset. | `organization_name` `preset_name` `template_name` | -| `coderd_prebuilt_workspaces_desired` | gauge | Target number of prebuilt workspaces that should be available for each template preset. | `organization_name` `preset_name` `template_name` | -| `coderd_prebuilt_workspaces_eligible` | gauge | Current number of prebuilt workspaces that are eligible to be claimed by users. These are workspaces that have completed their build process with their agent reporting 'ready' status. | `organization_name` `preset_name` `template_name` | -| `coderd_prebuilt_workspaces_failed_total` | counter | Total number of prebuilt workspaces that failed to build. | `organization_name` `preset_name` `template_name` | -| `coderd_prebuilt_workspaces_metrics_last_updated` | gauge | The unix timestamp when the metrics related to prebuilt workspaces were last updated; these metrics are cached. | | -| `coderd_prebuilt_workspaces_preset_hard_limited` | gauge | Indicates whether a given preset has reached the hard failure limit (1 = hard-limited). Metric is omitted otherwise. | `organization_name` `preset_name` `template_name` | -| `coderd_prebuilt_workspaces_preset_validation_failed` | gauge | Indicates whether a given preset has validation failures (1 = validation failed). Metric is omitted otherwise. | `organization_name` `preset_name` `template_name` | -| `coderd_prebuilt_workspaces_reconciliation_paused` | gauge | Indicates whether prebuilds reconciliation is currently paused (1 = paused, 0 = not paused). | | -| `coderd_prebuilt_workspaces_resource_replacements_total` | counter | Total number of prebuilt workspaces whose resource(s) got replaced upon being claimed. In Terraform, drift on immutable attributes results in resource replacement. This represents a worst-case scenario for prebuilt workspaces because the pre-provisioned resource would have been recreated when claiming, thus obviating the point of pre-provisioning. See https://coder.com/docs/admin/templates/extending-templates/prebuilt-workspaces#preventing-resource-replacement | `organization_name` `preset_name` `template_name` | -| `coderd_prebuilt_workspaces_running` | gauge | Current number of prebuilt workspaces that are in a running state. These workspaces have started successfully but may not yet be claimable by users (see coderd_prebuilt_workspaces_eligible). | `organization_name` `preset_name` `template_name` | -| `coderd_prometheusmetrics_agents_execution_seconds` | histogram | Histogram for duration of agents metrics collection in seconds. | | -| `coderd_prometheusmetrics_agentstats_execution_seconds` | histogram | Histogram for duration of agent stats metrics collection in seconds. | | -| `coderd_prometheusmetrics_metrics_aggregator_execution_cleanup_seconds` | histogram | Histogram for duration of metrics aggregator cleanup in seconds. | | -| `coderd_prometheusmetrics_metrics_aggregator_execution_update_seconds` | histogram | Histogram for duration of metrics aggregator update in seconds. | | -| `coderd_prometheusmetrics_metrics_aggregator_store_size` | gauge | The number of metrics stored in the aggregator | | -| `coderd_provisioner_job_queue_wait_seconds` | histogram | Time from job creation to acquisition by a provisioner daemon. | `build_reason` `job_type` `provisioner_type` `transition` | -| `coderd_provisionerd_job_timings_seconds` | histogram | The provisioner job time duration in seconds. | `provisioner` `status` | -| `coderd_provisionerd_jobs_current` | gauge | The number of currently running provisioner jobs. | `provisioner` | -| `coderd_provisionerd_num_daemons` | gauge | The number of provisioner daemons. | | -| `coderd_provisionerd_workspace_build_timings_seconds` | histogram | The time taken for a workspace to build. | `status` `template_name` `template_version` `workspace_transition` | -| `coderd_proxyhealth_health_check_duration_seconds` | histogram | Histogram for duration of proxy health collection in seconds. | | -| `coderd_proxyhealth_health_check_results` | gauge | This endpoint returns a number to indicate the health status. -3 (unknown), -2 (Unreachable), -1 (Unhealthy), 0 (Unregistered), 1 (Healthy) | `proxy_id` | -| `coderd_template_workspace_build_duration_seconds` | histogram | Duration from workspace build creation to agent ready, by template. | `is_prebuild` `organization_name` `status` `template_name` `transition` | -| `coderd_workspace_builds_enqueued_total` | counter | Total number of workspace build enqueue attempts. | `build_reason` `provisioner_type` `status` `transition` | -| `coderd_workspace_builds_total` | counter | The number of workspaces started, updated, or deleted. | `status` `template_name` `template_version` `workspace_name` `workspace_owner` `workspace_transition` | -| `coderd_workspace_creation_duration_seconds` | histogram | Time to create a workspace by organization, template, preset, and type (regular or prebuild). | `organization_name` `preset_name` `template_name` `type` | -| `coderd_workspace_creation_total` | counter | Total regular (non-prebuilt) workspace creations by organization, template, and preset. | `organization_name` `preset_name` `template_name` | -| `coderd_workspace_latest_build_status` | gauge | The current workspace statuses by template, transition, and owner for all non-deleted workspaces. | `status` `template_name` `template_version` `workspace_owner` `workspace_transition` | -| `go_gc_duration_seconds` | summary | A summary of the pause duration of garbage collection cycles. | | -| `go_goroutines` | gauge | Number of goroutines that currently exist. | | -| `go_info` | gauge | Information about the Go environment. | `version` | -| `go_memstats_alloc_bytes` | gauge | Number of bytes allocated and still in use. | | -| `go_memstats_alloc_bytes_total` | counter | Total number of bytes allocated, even if freed. | | -| `go_memstats_buck_hash_sys_bytes` | gauge | Number of bytes used by the profiling bucket hash table. | | -| `go_memstats_frees_total` | counter | Total number of frees. | | -| `go_memstats_gc_sys_bytes` | gauge | Number of bytes used for garbage collection system metadata. | | -| `go_memstats_heap_alloc_bytes` | gauge | Number of heap bytes allocated and still in use. | | -| `go_memstats_heap_idle_bytes` | gauge | Number of heap bytes waiting to be used. | | -| `go_memstats_heap_inuse_bytes` | gauge | Number of heap bytes that are in use. | | -| `go_memstats_heap_objects` | gauge | Number of allocated objects. | | -| `go_memstats_heap_released_bytes` | gauge | Number of heap bytes released to OS. | | -| `go_memstats_heap_sys_bytes` | gauge | Number of heap bytes obtained from system. | | -| `go_memstats_last_gc_time_seconds` | gauge | Number of seconds since 1970 of last garbage collection. | | -| `go_memstats_lookups_total` | counter | Total number of pointer lookups. | | -| `go_memstats_mallocs_total` | counter | Total number of mallocs. | | -| `go_memstats_mcache_inuse_bytes` | gauge | Number of bytes in use by mcache structures. | | -| `go_memstats_mcache_sys_bytes` | gauge | Number of bytes used for mcache structures obtained from system. | | -| `go_memstats_mspan_inuse_bytes` | gauge | Number of bytes in use by mspan structures. | | -| `go_memstats_mspan_sys_bytes` | gauge | Number of bytes used for mspan structures obtained from system. | | -| `go_memstats_next_gc_bytes` | gauge | Number of heap bytes when next garbage collection will take place. | | -| `go_memstats_other_sys_bytes` | gauge | Number of bytes used for other system allocations. | | -| `go_memstats_stack_inuse_bytes` | gauge | Number of bytes in use by the stack allocator. | | -| `go_memstats_stack_sys_bytes` | gauge | Number of bytes obtained from system for stack allocator. | | -| `go_memstats_sys_bytes` | gauge | Number of bytes obtained from system. | | -| `go_threads` | gauge | Number of OS threads created. | | -| `process_cpu_seconds_total` | counter | Total user and system CPU time spent in seconds. | | -| `process_max_fds` | gauge | Maximum number of open file descriptors. | | -| `process_open_fds` | gauge | Number of open file descriptors. | | -| `process_resident_memory_bytes` | gauge | Resident memory size in bytes. | | -| `process_start_time_seconds` | gauge | Start time of the process since unix epoch in seconds. | | -| `process_virtual_memory_bytes` | gauge | Virtual memory size in bytes. | | -| `process_virtual_memory_max_bytes` | gauge | Maximum amount of virtual memory available in bytes. | | -| `promhttp_metric_handler_requests_in_flight` | gauge | Current number of scrapes being served. | | -| `promhttp_metric_handler_requests_total` | counter | Total number of scrapes by HTTP status code. | `code` | +| Name | Type | Description | Labels | +|--------------------------------------------------------------------------|-----------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------| +| `agent_boundary_log_proxy_batches_dropped_total` | counter | Total number of boundary log batches dropped before reaching coderd. Reason: buffer_full = the agent's internal buffer is full, meaning boundary is producing logs faster than the agent can forward them to coderd; forward_failed = the agent failed to send the batch to coderd, potentially because coderd is unreachable or the connection was interrupted. | `reason` | +| `agent_boundary_log_proxy_batches_forwarded_total` | counter | Total number of boundary log batches successfully forwarded to coderd. Compare with batches_dropped_total to compute a drop rate. | | +| `agent_boundary_log_proxy_logs_dropped_total` | counter | Total number of individual boundary log entries dropped before reaching coderd. Reason: buffer_full = the agent's internal buffer is full; forward_failed = the agent failed to send the batch to coderd; boundary_channel_full = boundary's internal send channel overflowed, meaning boundary is generating logs faster than it can batch and send them; boundary_batch_full = boundary's outgoing batch buffer overflowed after a failed flush, meaning boundary could not write to the agent's socket. | `reason` | +| `agent_scripts_executed_total` | counter | Total number of scripts executed by the Coder agent. Includes cron scheduled scripts. | `agent_name` `success` `template_name` `username` `workspace_name` | +| `coder_ai_gateway_circuit_breaker_rejects_total` | counter | Total number of requests rejected due to open circuit breaker. | `endpoint` `model` `provider` | +| `coder_ai_gateway_circuit_breaker_state` | gauge | Current state of the circuit breaker (0=closed, 0.5=half-open, 1=open). | `endpoint` `model` `provider` | +| `coder_ai_gateway_circuit_breaker_trips_total` | counter | Total number of times the circuit breaker transitioned to open state. | `endpoint` `model` `provider` | +| `coder_ai_gateway_injected_tool_invocations_total` | counter | The number of times an injected MCP tool was invoked by AI Gateway. | `model` `name` `provider` `server` | +| `coder_ai_gateway_interceptions_duration_seconds` | histogram | The total duration of intercepted requests, in seconds. The majority of this time will be the upstream processing of the request. AI Gateway has no control over upstream processing time, so it's just an illustrative metric. | `model` `provider` | +| `coder_ai_gateway_interceptions_inflight` | gauge | The number of intercepted requests which are being processed. | `model` `provider` `route` | +| `coder_ai_gateway_interceptions_total` | counter | The count of intercepted requests. | `client` `initiator_id` `method` `model` `provider` `route` `status` | +| `coder_ai_gateway_key_pool_exhaustions_total` | counter | The number of times the key pool was exhausted with no usable key (outcome: rate_limited, auth_failed). | `outcome` `provider` | +| `coder_ai_gateway_key_pool_failover_attempts` | histogram | The number of keys attempted before success or exhaustion, per interception for bridged requests and per request for passthrough requests. | `provider` | +| `coder_ai_gateway_key_pool_state` | gauge | The number of keys currently in each state (state: valid, temporary, permanent). | `provider` `state` | +| `coder_ai_gateway_key_pool_state_transitions_total` | counter | The number of API key state transitions during failover (reason: rate_limited, unauthorized, forbidden). | `provider` `reason` | +| `coder_ai_gateway_non_injected_tool_selections_total` | counter | The number of times an AI model selected a tool to be invoked by the client. | `model` `name` `provider` | +| `coder_ai_gateway_passthrough_total` | counter | The count of requests which were not intercepted but passed through to the upstream. | `method` `provider` `route` | +| `coder_ai_gateway_prompts_total` | counter | The number of prompts issued by users (initiators). | `client` `initiator_id` `model` `provider` | +| `coder_ai_gateway_provider_info` | gauge | One series per configured AI provider. Value is always 1; the status label (enabled, disabled, error) carries the alertable signal. | `provider_name` `provider_type` `status` | +| `coder_ai_gateway_providers_last_reload_success_timestamp_seconds` | gauge | Unix timestamp of the last provider reload that successfully refreshed the pool. A gap against the providers_last_reload_timestamp_seconds gauge means the loop is firing but the refresh function is failing. | | +| `coder_ai_gateway_providers_last_reload_timestamp_seconds` | gauge | Unix timestamp of the last provider reload attempt, success or failure. | | +| `coder_ai_gateway_proxy_connect_sessions_total` | counter | Total number of CONNECT sessions established. | `type` | +| `coder_ai_gateway_proxy_inflight_mitm_requests` | gauge | Number of MITM requests currently being processed. | `provider` | +| `coder_ai_gateway_proxy_mitm_requests_total` | counter | Total number of MITM requests handled by the proxy. | `provider` | +| `coder_ai_gateway_proxy_mitm_responses_total` | counter | Total number of MITM responses by HTTP status code class. | `code` `provider` | +| `coder_ai_gateway_proxy_provider_info` | gauge | One series per configured AI provider. Value is always 1; the status label (enabled, disabled, error) carries the alertable signal. | `provider_name` `provider_type` `status` | +| `coder_ai_gateway_proxy_providers_last_reload_success_timestamp_seconds` | gauge | Unix timestamp of the last provider reload that successfully refreshed the router. A gap against the providers_last_reload_timestamp_seconds gauge means the loop is firing but the refresh function is failing. | | +| `coder_ai_gateway_proxy_providers_last_reload_timestamp_seconds` | gauge | Unix timestamp of the last provider reload attempt, success or failure. | | +| `coder_ai_gateway_tokens_total` | counter | The number of tokens used by intercepted requests. | `client` `initiator_id` `model` `provider` `type` | +| `coder_derp_server_accepts_total` | counter | Total DERP connections accepted. | | +| `coder_derp_server_average_queue_duration_ms` | gauge | Average queue duration in milliseconds. | | +| `coder_derp_server_bytes_received_total` | counter | Total bytes received. | | +| `coder_derp_server_bytes_sent_total` | counter | Total bytes sent. | | +| `coder_derp_server_clients` | gauge | Total clients (local + remote). | | +| `coder_derp_server_clients_local` | gauge | Local clients. | | +| `coder_derp_server_clients_remote` | gauge | Remote (mesh) clients. | | +| `coder_derp_server_connections` | gauge | Current DERP connections. | | +| `coder_derp_server_got_ping_total` | counter | Total pings received. | | +| `coder_derp_server_home_connections` | gauge | Current home DERP connections. | | +| `coder_derp_server_home_moves_in_total` | counter | Total home moves in. | | +| `coder_derp_server_home_moves_out_total` | counter | Total home moves out. | | +| `coder_derp_server_packets_dropped_reason_total` | counter | Packets dropped by reason. | `reason` | +| `coder_derp_server_packets_dropped_total` | counter | Total packets dropped. | | +| `coder_derp_server_packets_dropped_type_total` | counter | Packets dropped by type. | `type` | +| `coder_derp_server_packets_forwarded_in_total` | counter | Total packets forwarded in from mesh peers. | | +| `coder_derp_server_packets_forwarded_out_total` | counter | Total packets forwarded out to mesh peers. | | +| `coder_derp_server_packets_received_kind_total` | counter | Packets received by kind. | `kind` | +| `coder_derp_server_packets_received_total` | counter | Total packets received. | | +| `coder_derp_server_packets_sent_total` | counter | Total packets sent. | | +| `coder_derp_server_peer_gone_disconnected_total` | counter | Total peer gone (disconnected) frames sent. | | +| `coder_derp_server_peer_gone_not_here_total` | counter | Total peer gone (not here) frames sent. | | +| `coder_derp_server_sent_pong_total` | counter | Total pongs sent. | | +| `coder_derp_server_unknown_frames_total` | counter | Total unknown frames received. | | +| `coder_derp_server_watchers` | gauge | Current watchers. | | +| `coder_pubsub_connected` | gauge | Whether we are connected (1) or not connected (0) to postgres | | +| `coder_pubsub_current_events` | gauge | The current number of pubsub event channels listened for | | +| `coder_pubsub_current_subscribers` | gauge | The current number of active pubsub subscribers | | +| `coder_pubsub_disconnections_total` | counter | Total number of times we disconnected unexpectedly from postgres | | +| `coder_pubsub_latency_measure_errs_total` | counter | The number of pubsub latency measurement failures | | +| `coder_pubsub_latency_measures_total` | counter | The number of pubsub latency measurements | | +| `coder_pubsub_messages_total` | counter | Total number of messages received from postgres | `size` | +| `coder_pubsub_published_bytes_total` | counter | Total number of bytes successfully published across all publishes | | +| `coder_pubsub_publishes_total` | counter | Total number of calls to Publish | `success` | +| `coder_pubsub_receive_latency_seconds` | gauge | The time taken to receive a message from a pubsub event channel | | +| `coder_pubsub_received_bytes_total` | counter | Total number of bytes received across all messages | | +| `coder_pubsub_send_latency_seconds` | gauge | The time taken to send a message into a pubsub event channel | | +| `coder_pubsub_subscribes_total` | counter | Total number of calls to Subscribe/SubscribeWithErr | `success` | +| `coder_servertailnet_connections_total` | counter | Total number of TCP connections made to workspace agents. | `network` | +| `coder_servertailnet_open_connections` | gauge | Total number of TCP connections currently open to workspace agents. | `network` | +| `coderd_agentapi_metadata_batch_size` | histogram | Total number of metadata entries in each batch, updated before flushes. | | +| `coderd_agentapi_metadata_batch_utilization` | histogram | Number of metadata keys per agent in each batch, updated before flushes. | | +| `coderd_agentapi_metadata_batches_total` | counter | Total number of metadata batches flushed. | `reason` | +| `coderd_agentapi_metadata_dropped_keys_total` | counter | Total number of metadata keys dropped due to capacity limits. | | +| `coderd_agentapi_metadata_flush_duration_seconds` | histogram | Time taken to flush metadata batch to database and pubsub. | `reason` | +| `coderd_agentapi_metadata_flushed_total` | counter | Total number of unique metadatas flushed. | | +| `coderd_agentapi_metadata_publish_errors_total` | counter | Total number of metadata batch pubsub publish calls that have resulted in an error. | | +| `coderd_agents_apps` | gauge | Agent applications with statuses. | `agent_name` `app_name` `health` `username` `workspace_name` | +| `coderd_agents_connection_latencies_seconds` | gauge | Agent connection latencies in seconds. | `agent_name` `derp_region` `preferred` `username` `workspace_name` | +| `coderd_agents_connections` | gauge | Agent connections with statuses. | `agent_name` `lifecycle_state` `status` `tailnet_node` `username` `workspace_name` | +| `coderd_agents_first_connection_seconds` | histogram | Duration from agent creation to first connection in seconds. | `agent_name` `template_name` | +| `coderd_agents_up` | gauge | The number of active agents per workspace. | `template_name` `template_version` `username` `workspace_name` | +| `coderd_agentstats_connection_count` | gauge | The number of established connections by agent | `agent_name` `username` `workspace_name` | +| `coderd_agentstats_connection_median_latency_seconds` | gauge | The median agent connection latency | `agent_name` `username` `workspace_name` | +| `coderd_agentstats_currently_reachable_peers` | gauge | The number of peers (e.g. clients) that are currently reachable over the encrypted network. | `agent_name` `connection_type` `template_name` `username` `workspace_name` | +| `coderd_agentstats_rx_bytes` | gauge | Agent Rx bytes | `agent_name` `username` `workspace_name` | +| `coderd_agentstats_session_count_jetbrains` | gauge | The number of session established by JetBrains | `agent_name` `username` `workspace_name` | +| `coderd_agentstats_session_count_reconnecting_pty` | gauge | The number of session established by reconnecting PTY | `agent_name` `username` `workspace_name` | +| `coderd_agentstats_session_count_ssh` | gauge | The number of session established by SSH | `agent_name` `username` `workspace_name` | +| `coderd_agentstats_session_count_vscode` | gauge | The number of session established by VSCode | `agent_name` `username` `workspace_name` | +| `coderd_agentstats_startup_script_seconds` | gauge | The number of seconds the startup script took to execute. | `agent_name` `success` `template_name` `username` `workspace_name` | +| `coderd_agentstats_tx_bytes` | gauge | Agent Tx bytes | `agent_name` `username` `workspace_name` | +| `coderd_api_active_users_duration_hour` | gauge | The number of users that have been active within the last hour. | | +| `coderd_api_concurrent_requests` | gauge | The number of concurrent API requests. | `method` `path` | +| `coderd_api_concurrent_websockets` | gauge | The total number of concurrent API websockets. | `path` | +| `coderd_api_request_latencies_seconds` | histogram | Latency distribution of requests in seconds. | `method` `path` | +| `coderd_api_requests_processed_total` | counter | The total number of processed API requests | `code` `method` `path` | +| `coderd_api_total_user_count` | gauge | The total number of registered users, partitioned by status. | `status` | +| `coderd_api_websocket_durations_seconds` | histogram | Websocket duration distribution of requests in seconds. | `path` | +| `coderd_api_websocket_probes_total` | counter | WebSocket liveness probe outcomes by route. Compare rate(...{result="ok"}[1m]) against coderd_api_concurrent_websockets to detect unresponsive WebSocket connections. | `path` `result` | +| `coderd_api_workspace_latest_build` | gauge | The current number of workspace builds by status for all non-deleted workspaces. | `status` | +| `coderd_authz_authorize_duration_seconds` | histogram | Duration of the 'Authorize' call in seconds. Only counts calls that succeed. | `allowed` | +| `coderd_authz_prepare_authorize_duration_seconds` | histogram | Duration of the 'PrepareAuthorize' call in seconds. | | +| `coderd_build_info` | gauge | Describes the current build/version of the Coder server. Value is always 1. | `revision` `version` | +| `coderd_chat_auto_archive_records_archived_total` | counter | Total number of chats archived by the auto-archive job (counting both roots and cascaded children). | | +| `coderd_chatd_chats` | gauge | Number of chats being processed, by state. | `state` | +| `coderd_chatd_compaction_total` | counter | Total compaction outcomes (only recorded when compaction was triggered or failed). | `model` `provider` `result` | +| `coderd_chatd_message_count` | histogram | Number of messages in the prompt per LLM request. | `model` `provider` | +| `coderd_chatd_prompt_size_bytes` | histogram | Estimated byte size of the prompt per LLM request. | `model` `provider` | +| `coderd_chatd_steps_total` | counter | Total agentic loop steps across all chats. | `model` `provider` | +| `coderd_chatd_stream_buffer_dropped_total` | counter | Number of chat stream buffer events dropped due to the per-chat buffer cap. | | +| `coderd_chatd_stream_retries_total` | counter | Total LLM stream retries. | `chain_broken` `kind` `model` `provider` | +| `coderd_chatd_tool_errors_total` | counter | Total tool calls that returned an error result. | `model` `provider` `tool_name` | +| `coderd_chatd_tool_result_size_bytes` | histogram | Size in bytes of each tool execution result. | `model` `provider` `tool_name` | +| `coderd_chatd_ttft_seconds` | histogram | Time-to-first-token: wall time from LLM request to first streamed chunk. | `model` `provider` | +| `coderd_db_query_counts_total` | counter | Total number of queries labelled by HTTP route, method, and query name. | `method` `query` `route` | +| `coderd_db_query_latencies_seconds` | histogram | Latency distribution of queries in seconds. | `query` | +| `coderd_db_tx_duration_seconds` | histogram | Duration of transactions in seconds. | `success` `tx_id` | +| `coderd_db_tx_executions_count` | counter | Total count of transactions executed. 'retries' is expected to be 0 for a successful transaction. | `retries` `success` `tx_id` | +| `coderd_dbpurge_iteration_duration_seconds` | histogram | Duration of each dbpurge iteration in seconds. | `success` | +| `coderd_dbpurge_records_purged_total` | counter | Total number of records purged by type. | `record_type` | +| `coderd_experiments` | gauge | Indicates whether each experiment is enabled (1) or not (0) | `experiment` | +| `coderd_insights_applications_usage_seconds` | gauge | The application usage per template. | `application_name` `organization_name` `slug` `template_name` | +| `coderd_insights_parameters` | gauge | The parameter usage per template. | `organization_name` `parameter_name` `parameter_type` `parameter_value` `template_name` | +| `coderd_insights_templates_active_users` | gauge | The number of active users of the template. | `organization_name` `template_name` | +| `coderd_license_active_users` | gauge | The number of active users. | | +| `coderd_license_errors` | gauge | The number of active license errors. | | +| `coderd_license_limit_users` | gauge | The user seats limit based on the active Coder license. | | +| `coderd_license_user_limit_enabled` | gauge | Returns 1 if the current license enforces the user limit. | | +| `coderd_license_warnings` | gauge | The number of active license warnings. | | +| `coderd_lifecycle_autobuild_execution_duration_seconds` | histogram | Duration of each autobuild execution. | | +| `coderd_notifications_dispatcher_send_seconds` | histogram | The time taken to dispatch notifications. | `method` | +| `coderd_notifications_inflight_dispatches` | gauge | The number of dispatch attempts which are currently in progress. | `method` `notification_template_id` | +| `coderd_notifications_pending_updates` | gauge | The number of dispatch attempt results waiting to be flushed to the store. | | +| `coderd_notifications_queued_seconds` | histogram | The time elapsed between a notification being enqueued in the store and retrieved for dispatching (measures the latency of the notifications system). This should generally be within CODER_NOTIFICATIONS_FETCH_INTERVAL seconds; higher values for a sustained period indicates delayed processing and CODER_NOTIFICATIONS_LEASE_COUNT can be increased to accommodate this. | `method` | +| `coderd_notifications_retry_count` | counter | The count of notification dispatch retry attempts. | `method` `notification_template_id` | +| `coderd_notifications_synced_updates_total` | counter | The number of dispatch attempt results flushed to the store. | | +| `coderd_oauth2_external_requests_rate_limit` | gauge | The total number of allowed requests per interval. | `name` `resource` | +| `coderd_oauth2_external_requests_rate_limit_next_reset_unix` | gauge | Unix timestamp for when the next interval starts | `name` `resource` | +| `coderd_oauth2_external_requests_rate_limit_remaining` | gauge | The remaining number of allowed requests in this interval. | `name` `resource` | +| `coderd_oauth2_external_requests_rate_limit_reset_in_seconds` | gauge | Seconds until the next interval | `name` `resource` | +| `coderd_oauth2_external_requests_rate_limit_used` | gauge | The number of requests made in this interval. | `name` `resource` | +| `coderd_oauth2_external_requests_total` | counter | The total number of api calls made to external oauth2 providers. 'status_code' will be 0 if the request failed with no response. | `name` `source` `status_code` | +| `coderd_open_file_refs_current` | gauge | The count of file references currently open in the file cache. Multiple references can be held for the same file. | | +| `coderd_open_file_refs_total` | counter | The total number of file references ever opened in the file cache. The 'hit' label indicates if the file was loaded from the cache. | `hit` | +| `coderd_open_files_current` | gauge | The count of unique files currently open in the file cache. | | +| `coderd_open_files_size_bytes_current` | gauge | The current amount of memory of all files currently open in the file cache. | | +| `coderd_open_files_size_bytes_total` | counter | The total amount of memory ever opened in the file cache. This number never decrements. | | +| `coderd_open_files_total` | counter | The total count of unique files ever opened in the file cache. | | +| `coderd_prebuilds_reconciliation_duration_seconds` | histogram | Duration of each prebuilds reconciliation cycle. | | +| `coderd_prebuilt_workspace_claim_duration_seconds` | histogram | Time to claim a prebuilt workspace by organization, template, and preset. | `organization_name` `preset_name` `template_name` | +| `coderd_prebuilt_workspaces_claimed_total` | counter | Total number of prebuilt workspaces which were claimed by users. Claiming refers to creating a workspace with a preset selected for which eligible prebuilt workspaces are available and one is reassigned to a user. | `organization_name` `preset_name` `template_name` | +| `coderd_prebuilt_workspaces_created_total` | counter | Total number of prebuilt workspaces that have been created to meet the desired instance count of each template preset. | `organization_name` `preset_name` `template_name` | +| `coderd_prebuilt_workspaces_desired` | gauge | Target number of prebuilt workspaces that should be available for each template preset. | `organization_name` `preset_name` `template_name` | +| `coderd_prebuilt_workspaces_eligible` | gauge | Current number of prebuilt workspaces that are eligible to be claimed by users. These are workspaces that have completed their build process with their agent reporting 'ready' status. | `organization_name` `preset_name` `template_name` | +| `coderd_prebuilt_workspaces_failed_total` | counter | Total number of prebuilt workspaces that failed to build. | `organization_name` `preset_name` `template_name` | +| `coderd_prebuilt_workspaces_metrics_last_updated` | gauge | The unix timestamp when the metrics related to prebuilt workspaces were last updated; these metrics are cached. | | +| `coderd_prebuilt_workspaces_preset_hard_limited` | gauge | Indicates whether a given preset has reached the hard failure limit (1 = hard-limited). Metric is omitted otherwise. | `organization_name` `preset_name` `template_name` | +| `coderd_prebuilt_workspaces_preset_validation_failed` | gauge | Indicates whether a given preset has validation failures (1 = validation failed). Metric is omitted otherwise. | `organization_name` `preset_name` `template_name` | +| `coderd_prebuilt_workspaces_reconciliation_paused` | gauge | Indicates whether prebuilds reconciliation is currently paused (1 = paused, 0 = not paused). | | +| `coderd_prebuilt_workspaces_resource_replacements_total` | counter | Total number of prebuilt workspaces whose resource(s) got replaced upon being claimed. In Terraform, drift on immutable attributes results in resource replacement. This represents a worst-case scenario for prebuilt workspaces because the pre-provisioned resource would have been recreated when claiming, thus obviating the point of pre-provisioning. See https://coder.com/docs/admin/templates/extending-templates/prebuilt-workspaces#preventing-resource-replacement | `organization_name` `preset_name` `template_name` | +| `coderd_prebuilt_workspaces_running` | gauge | Current number of prebuilt workspaces that are in a running state. These workspaces have started successfully but may not yet be claimable by users (see coderd_prebuilt_workspaces_eligible). | `organization_name` `preset_name` `template_name` | +| `coderd_prometheusmetrics_agents_execution_seconds` | histogram | Histogram for duration of agents metrics collection in seconds. | | +| `coderd_prometheusmetrics_agentstats_execution_seconds` | histogram | Histogram for duration of agent stats metrics collection in seconds. | | +| `coderd_prometheusmetrics_metrics_aggregator_execution_cleanup_seconds` | histogram | Histogram for duration of metrics aggregator cleanup in seconds. | | +| `coderd_prometheusmetrics_metrics_aggregator_execution_update_seconds` | histogram | Histogram for duration of metrics aggregator update in seconds. | | +| `coderd_prometheusmetrics_metrics_aggregator_store_size` | gauge | The number of metrics stored in the aggregator | | +| `coderd_provisioner_job_queue_wait_seconds` | histogram | Time from job creation to acquisition by a provisioner daemon. | `build_reason` `job_type` `provisioner_type` `transition` | +| `coderd_provisionerd_job_timings_seconds` | histogram | The provisioner job time duration in seconds. | `provisioner` `status` | +| `coderd_provisionerd_jobs_current` | gauge | The number of currently running provisioner jobs. | `provisioner` | +| `coderd_provisionerd_num_daemons` | gauge | The number of provisioner daemons. | | +| `coderd_provisionerd_workspace_build_timings_seconds` | histogram | The time taken for a workspace to build. | `status` `template_name` `template_version` `workspace_transition` | +| `coderd_proxyhealth_health_check_duration_seconds` | histogram | Histogram for duration of proxy health collection in seconds. | | +| `coderd_proxyhealth_health_check_results` | gauge | This endpoint returns a number to indicate the health status. -3 (unknown), -2 (Unreachable), -1 (Unhealthy), 0 (Unregistered), 1 (Healthy) | `proxy_id` | +| `coderd_template_workspace_build_duration_seconds` | histogram | Duration from workspace build creation to agent ready, by template. | `is_prebuild` `organization_name` `status` `template_name` `transition` | +| `coderd_workspace_builds_enqueued_total` | counter | Total number of workspace build enqueue attempts. | `build_reason` `provisioner_type` `status` `transition` | +| `coderd_workspace_builds_total` | counter | The number of workspaces started, updated, or deleted. | `status` `template_name` `template_version` `workspace_name` `workspace_owner` `workspace_transition` | +| `coderd_workspace_creation_duration_seconds` | histogram | Time to create a workspace by organization, template, preset, and type (regular or prebuild). | `organization_name` `preset_name` `template_name` `type` | +| `coderd_workspace_creation_total` | counter | Total regular (non-prebuilt) workspace creations by organization, template, and preset. | `organization_name` `preset_name` `template_name` | +| `coderd_workspace_latest_build_status` | gauge | The current workspace statuses by template, transition, and owner for all non-deleted workspaces. | `status` `template_name` `template_version` `workspace_owner` `workspace_transition` | +| `go_gc_duration_seconds` | summary | A summary of the pause duration of garbage collection cycles. | | +| `go_goroutines` | gauge | Number of goroutines that currently exist. | | +| `go_info` | gauge | Information about the Go environment. | `version` | +| `go_memstats_alloc_bytes` | gauge | Number of bytes allocated and still in use. | | +| `go_memstats_alloc_bytes_total` | counter | Total number of bytes allocated, even if freed. | | +| `go_memstats_buck_hash_sys_bytes` | gauge | Number of bytes used by the profiling bucket hash table. | | +| `go_memstats_frees_total` | counter | Total number of frees. | | +| `go_memstats_gc_sys_bytes` | gauge | Number of bytes used for garbage collection system metadata. | | +| `go_memstats_heap_alloc_bytes` | gauge | Number of heap bytes allocated and still in use. | | +| `go_memstats_heap_idle_bytes` | gauge | Number of heap bytes waiting to be used. | | +| `go_memstats_heap_inuse_bytes` | gauge | Number of heap bytes that are in use. | | +| `go_memstats_heap_objects` | gauge | Number of allocated objects. | | +| `go_memstats_heap_released_bytes` | gauge | Number of heap bytes released to OS. | | +| `go_memstats_heap_sys_bytes` | gauge | Number of heap bytes obtained from system. | | +| `go_memstats_last_gc_time_seconds` | gauge | Number of seconds since 1970 of last garbage collection. | | +| `go_memstats_lookups_total` | counter | Total number of pointer lookups. | | +| `go_memstats_mallocs_total` | counter | Total number of mallocs. | | +| `go_memstats_mcache_inuse_bytes` | gauge | Number of bytes in use by mcache structures. | | +| `go_memstats_mcache_sys_bytes` | gauge | Number of bytes used for mcache structures obtained from system. | | +| `go_memstats_mspan_inuse_bytes` | gauge | Number of bytes in use by mspan structures. | | +| `go_memstats_mspan_sys_bytes` | gauge | Number of bytes used for mspan structures obtained from system. | | +| `go_memstats_next_gc_bytes` | gauge | Number of heap bytes when next garbage collection will take place. | | +| `go_memstats_other_sys_bytes` | gauge | Number of bytes used for other system allocations. | | +| `go_memstats_stack_inuse_bytes` | gauge | Number of bytes in use by the stack allocator. | | +| `go_memstats_stack_sys_bytes` | gauge | Number of bytes obtained from system for stack allocator. | | +| `go_memstats_sys_bytes` | gauge | Number of bytes obtained from system. | | +| `go_threads` | gauge | Number of OS threads created. | | +| `process_cpu_seconds_total` | counter | Total user and system CPU time spent in seconds. | | +| `process_max_fds` | gauge | Maximum number of open file descriptors. | | +| `process_open_fds` | gauge | Number of open file descriptors. | | +| `process_resident_memory_bytes` | gauge | Resident memory size in bytes. | | +| `process_start_time_seconds` | gauge | Start time of the process since unix epoch in seconds. | | +| `process_virtual_memory_bytes` | gauge | Virtual memory size in bytes. | | +| `process_virtual_memory_max_bytes` | gauge | Maximum amount of virtual memory available in bytes. | | +| `promhttp_metric_handler_requests_in_flight` | gauge | Current number of scrapes being served. | | +| `promhttp_metric_handler_requests_total` | counter | Total number of scrapes by HTTP status code. | `code` | diff --git a/docs/ai-coder/ai-gateway/monitoring.md b/docs/ai-coder/ai-gateway/monitoring.md index f5d59c908d..a9b04d99ad 100644 --- a/docs/ai-coder/ai-gateway/monitoring.md +++ b/docs/ai-coder/ai-gateway/monitoring.md @@ -17,42 +17,69 @@ These logs and metrics can be used to determine usage patterns, track costs, and ## Provider metrics -`aibridged` (the in-process daemon) and `aibridgeproxyd` (the external +AI Gateway (the in-process daemon) and AI Gateway Proxy (the external proxy) each export Prometheus metrics describing the configured provider pool and its reload loop. See [Provider Configuration](./providers.md) for the lifecycle these metrics describe. -| Metric | Type | Labels | Purpose | -|------------------------------------------------------------------------|---------|--------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------| -| `coder_aibridged_provider_info` | gauge | `provider_name`, `provider_type`, `status` | One series per configured provider. Value is always `1`; the `status` label (`enabled`, `disabled`, `error`) carries the alertable signal. | -| `coder_aibridged_providers_last_reload_timestamp_seconds` | gauge | | Unix timestamp of the last reload attempt, success or failure. | -| `coder_aibridged_providers_last_reload_success_timestamp_seconds` | gauge | | Unix timestamp of the last reload that successfully refreshed the pool. | -| `coder_aibridgeproxyd_provider_info` | gauge | `provider_name`, `provider_type`, `status` | Same shape as `aibridged_provider_info` but reported by the external proxy. | -| `coder_aibridgeproxyd_providers_last_reload_timestamp_seconds` | gauge | | Last reload attempt timestamp in `aibridgeproxyd`. | -| `coder_aibridgeproxyd_providers_last_reload_success_timestamp_seconds` | gauge | | Last successful reload timestamp in `aibridgeproxyd`. | -| `coder_aibridgeproxyd_connect_sessions_total` | counter | `type` (`mitm`, `tunneled`) | CONNECT sessions established by the proxy. | -| `coder_aibridgeproxyd_mitm_requests_total` | counter | `provider` | MITM requests handled. | -| `coder_aibridgeproxyd_inflight_mitm_requests` | gauge | `provider` | In-flight MITM requests. | -| `coder_aibridgeproxyd_mitm_responses_total` | counter | `code`, `provider` | MITM responses by HTTP status code. | +| Metric | Type | Labels | Purpose | +|--------------------------------------------------------------------------|---------|--------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------| +| `coder_ai_gateway_provider_info` | gauge | `provider_name`, `provider_type`, `status` | One series per configured provider. Value is always `1`; the `status` label (`enabled`, `disabled`, `error`) carries the alertable signal. | +| `coder_ai_gateway_providers_last_reload_timestamp_seconds` | gauge | | Unix timestamp of the last reload attempt, success or failure. | +| `coder_ai_gateway_providers_last_reload_success_timestamp_seconds` | gauge | | Unix timestamp of the last reload that successfully refreshed the pool. | +| `coder_ai_gateway_proxy_provider_info` | gauge | `provider_name`, `provider_type`, `status` | Same shape as `coder_ai_gateway_provider_info` but reported by the external proxy. | +| `coder_ai_gateway_proxy_providers_last_reload_timestamp_seconds` | gauge | | Last reload attempt timestamp in the external proxy. | +| `coder_ai_gateway_proxy_providers_last_reload_success_timestamp_seconds` | gauge | | Last successful reload timestamp in the external proxy. | +| `coder_ai_gateway_proxy_connect_sessions_total` | counter | `type` (`mitm`, `tunneled`) | CONNECT sessions established by the proxy. | +| `coder_ai_gateway_proxy_mitm_requests_total` | counter | `provider` | MITM requests handled. | +| `coder_ai_gateway_proxy_inflight_mitm_requests` | gauge | `provider` | In-flight MITM requests. | +| `coder_ai_gateway_proxy_mitm_responses_total` | counter | `code`, `provider` | MITM responses by HTTP status code. | + +> [!IMPORTANT] +> The AI Gateway metric prefixes were renamed: `coder_aibridged_*` became +> `coder_ai_gateway_*` and `coder_aibridgeproxyd_*` became +> `coder_ai_gateway_proxy_*`. This rename covers every AI Gateway metric, +> including the interception, token, prompt, tool, and circuit-breaker counters +> listed in the [Prometheus reference](../../admin/integrations/prometheus.md). +> The legacy `coder_aibridged_*` and `coder_aibridgeproxyd_*` names are still +> emitted with identical values during the v2.35 and v2.36 deprecation window. +> They are planned for removal in v2.37. Migrate dashboards and alerts to the new +> names now. Do not relabel new names back to old names while legacy names are +> still emitted, because that creates duplicate legacy series in the same scrape. +> After legacy names are removed, use `metric_relabel_configs` only if you need a +> temporary compatibility bridge for dashboards that still use the old names: +> +> ```yaml +> metric_relabel_configs: +> # Proxy rule must come first; the gateway regex below also matches proxy metrics. +> - source_labels: [__name__] +> regex: 'coder_ai_gateway_proxy_(.*)' +> target_label: __name__ +> replacement: 'coder_aibridgeproxyd_${1}' +> - source_labels: [__name__] +> regex: 'coder_ai_gateway_(.*)' +> target_label: __name__ +> replacement: 'coder_aibridged_${1}' +> ``` ### Suggested alerts Alert on any provider entering a non-`enabled` status: ```promql -sum by (provider_name, status) (coder_aibridged_provider_info{status!="enabled"}) > 0 +sum by (provider_name, status) (coder_ai_gateway_provider_info{status!="enabled"}) > 0 ``` Alert when the reload loop is firing but failing to refresh the pool for longer than a few minutes: ```promql -(coder_aibridged_providers_last_reload_timestamp_seconds - - coder_aibridged_providers_last_reload_success_timestamp_seconds) > 300 +(coder_ai_gateway_providers_last_reload_timestamp_seconds + - coder_ai_gateway_providers_last_reload_success_timestamp_seconds) > 300 ``` -Repeat the same query against `coder_aibridgeproxyd_*` if you run the +Repeat the same query against `coder_ai_gateway_proxy_*` if you run the external proxy. ## Structured Logging diff --git a/docs/ai-coder/ai-gateway/providers.md b/docs/ai-coder/ai-gateway/providers.md index d93d942293..ee928198fc 100644 --- a/docs/ai-coder/ai-gateway/providers.md +++ b/docs/ai-coder/ai-gateway/providers.md @@ -175,11 +175,11 @@ Provider configuration changes take effect automatically, without restarting `coderd`. AI Gateway records the timestamp of each reload attempt and each successful reload, exposed as Prometheus metrics: -- `coder_aibridged_providers_last_reload_timestamp_seconds` -- `coder_aibridged_providers_last_reload_success_timestamp_seconds` +- `coder_ai_gateway_providers_last_reload_timestamp_seconds` +- `coder_ai_gateway_providers_last_reload_success_timestamp_seconds` If you run the [external proxy](./ai-gateway-proxy/index.md), it exposes -the same pair under the `coder_aibridgeproxyd_` prefix. +the same pair under the `coder_ai_gateway_proxy_` prefix. A growing gap between the attempt and success timestamps means reloads are firing but failing to apply. Alert on that gap rather than on a diff --git a/enterprise/aibridgeproxyd/metrics.go b/enterprise/aibridgeproxyd/metrics.go index ccfd334aa7..4f169434d3 100644 --- a/enterprise/aibridgeproxyd/metrics.go +++ b/enterprise/aibridgeproxyd/metrics.go @@ -86,7 +86,7 @@ func NewMetrics(reg prometheus.Registerer) *Metrics { ProvidersLastReloadSuccessTimestampSeconds: factory.NewGauge(prometheus.GaugeOpts{ Name: "providers_last_reload_success_timestamp_seconds", - Help: "Unix timestamp of the last provider reload that successfully refreshed the router. A gap against coder_aibridgeproxyd_providers_last_reload_timestamp_seconds means the loop is firing but the refresh function is failing.", + Help: "Unix timestamp of the last provider reload that successfully refreshed the router. A gap against the providers_last_reload_timestamp_seconds gauge means the loop is firing but the refresh function is failing.", }), } } diff --git a/enterprise/cli/aibridgeproxyd.go b/enterprise/cli/aibridgeproxyd.go index 08641f5769..bb86f1210d 100644 --- a/enterprise/cli/aibridgeproxyd.go +++ b/enterprise/cli/aibridgeproxyd.go @@ -9,7 +9,6 @@ import ( "path/filepath" "strings" - "github.com/prometheus/client_golang/prometheus" "golang.org/x/xerrors" "cdr.dev/slog/v3" @@ -17,6 +16,7 @@ import ( "github.com/coder/coder/v2/coderd/aibridged" "github.com/coder/coder/v2/coderd/database" "github.com/coder/coder/v2/coderd/database/dbauthz" + "github.com/coder/coder/v2/coderd/prometheusmetrics" "github.com/coder/coder/v2/enterprise/aibridgeproxyd" "github.com/coder/coder/v2/enterprise/coderd" ) @@ -45,7 +45,10 @@ func newAIBridgeProxyDaemon(coderAPI *coderd.API) (io.Closer, error) { logger := coderAPI.Logger.Named("aibridgeproxyd") - reg := prometheus.WrapRegistererWithPrefix("coder_aibridgeproxyd_", coderAPI.PrometheusRegistry) + // TODO(deprecation): Remove "coder_aibridgeproxyd_" in v2.37. + // See AIGOV-447: + // https://linear.app/codercom/issue/AIGOV-447/remove-legacy-ai-gateway-metric-aliases + reg := prometheusmetrics.NewMetricAliasRegisterer(coderAPI.PrometheusRegistry, "coder_ai_gateway_proxy_", "coder_aibridgeproxyd_") metrics := aibridgeproxyd.NewMetrics(reg) var newDumper func(provider, requestID string) aibridgeproxyd.RoundTripDumper diff --git a/scripts/metricsdocgen/README.md b/scripts/metricsdocgen/README.md index 509cd9d9ef..e418d235fd 100644 --- a/scripts/metricsdocgen/README.md +++ b/scripts/metricsdocgen/README.md @@ -18,7 +18,7 @@ Contains metrics that are **not** directly defined in the coder source code: - `go_*`: Go runtime metrics - `process_*`: Process metrics from prometheus/client_golang - `promhttp_*`: Prometheus HTTP handler metrics -- `coder_aibridged_*`: Metrics from external dependencies +- `coder_ai_gateway_*`: AI Gateway metrics are registered through prefixed registerer that the scanner does not resolve. > [!Note] > This file also contains edge cases where metric metadata cannot be accurately extracted by the scanner (e.g., labels determined by runtime logic). diff --git a/scripts/metricsdocgen/metrics b/scripts/metricsdocgen/metrics index 036ac496a1..cc9a009dce 100644 --- a/scripts/metricsdocgen/metrics +++ b/scripts/metricsdocgen/metrics @@ -147,82 +147,105 @@ promhttp_metric_handler_requests_in_flight 1 promhttp_metric_handler_requests_total{code="200"} 2 promhttp_metric_handler_requests_total{code="500"} 0 promhttp_metric_handler_requests_total{code="503"} 0 -# HELP coder_aibridged_injected_tool_invocations_total The number of times an injected MCP tool was invoked by aibridge. -# TYPE coder_aibridged_injected_tool_invocations_total counter -coder_aibridged_injected_tool_invocations_total{model="gpt-5-nano",name="coder_list_templates",provider="openai",server="https://xxx.pit-1.try.coder.app/api/experimental/mcp/http"} 1 -# HELP coder_aibridged_interceptions_duration_seconds The total duration of intercepted requests, in seconds. The majority of this time will be the upstream processing of the request. aibridge has no control over upstream processing time, so it's just an illustrative metric. -# TYPE coder_aibridged_interceptions_duration_seconds histogram -coder_aibridged_interceptions_duration_seconds_bucket{model="gpt-5-nano",provider="openai",le="0.5"} 0 -coder_aibridged_interceptions_duration_seconds_bucket{model="gpt-5-nano",provider="openai",le="2"} 0 -coder_aibridged_interceptions_duration_seconds_bucket{model="gpt-5-nano",provider="openai",le="5"} 3 -coder_aibridged_interceptions_duration_seconds_bucket{model="gpt-5-nano",provider="openai",le="15"} 6 -coder_aibridged_interceptions_duration_seconds_bucket{model="gpt-5-nano",provider="openai",le="30"} 6 -coder_aibridged_interceptions_duration_seconds_bucket{model="gpt-5-nano",provider="openai",le="60"} 6 -coder_aibridged_interceptions_duration_seconds_bucket{model="gpt-5-nano",provider="openai",le="120"} 6 -coder_aibridged_interceptions_duration_seconds_bucket{model="gpt-5-nano",provider="openai",le="+Inf"} 6 -coder_aibridged_interceptions_duration_seconds_sum{model="gpt-5-nano",provider="openai"} 34.120188692 -coder_aibridged_interceptions_duration_seconds_count{model="gpt-5-nano",provider="openai"} 6 -# HELP coder_aibridged_interceptions_inflight The number of intercepted requests which are being processed. -# TYPE coder_aibridged_interceptions_inflight gauge -coder_aibridged_interceptions_inflight{model="gpt-5-nano",provider="openai",route="/v1/chat/completions"} 0 -# HELP coder_aibridged_interceptions_total The count of intercepted requests. -# TYPE coder_aibridged_interceptions_total counter -coder_aibridged_interceptions_total{initiator_id="95f6752b-08cc-4cf1-97f7-c2165e3519c5",method="POST",model="gpt-5-nano",provider="openai",route="/v1/chat/completions",status="completed"} 6 -# HELP coder_aibridged_non_injected_tool_selections_total The number of times an AI model selected a tool to be invoked by the client. -# TYPE coder_aibridged_non_injected_tool_selections_total counter -coder_aibridged_non_injected_tool_selections_total{model="gpt-5-nano",name="read_file",provider="openai"} 2 -# HELP coder_aibridged_prompts_total The number of prompts issued by users (initiators). -# TYPE coder_aibridged_prompts_total counter -coder_aibridged_prompts_total{initiator_id="95f6752b-08cc-4cf1-97f7-c2165e3519c5",model="gpt-5-nano",provider="openai"} 4 -# HELP coder_aibridged_tokens_total The number of tokens used by intercepted requests. -# TYPE coder_aibridged_tokens_total counter -coder_aibridged_tokens_total{initiator_id="95f6752b-08cc-4cf1-97f7-c2165e3519c5",model="gpt-5-nano",provider="openai",type="completion_accepted_prediction"} 0 -coder_aibridged_tokens_total{initiator_id="95f6752b-08cc-4cf1-97f7-c2165e3519c5",model="gpt-5-nano",provider="openai",type="completion_audio"} 0 -coder_aibridged_tokens_total{initiator_id="95f6752b-08cc-4cf1-97f7-c2165e3519c5",model="gpt-5-nano",provider="openai",type="completion_reasoning"} 1664 -coder_aibridged_tokens_total{initiator_id="95f6752b-08cc-4cf1-97f7-c2165e3519c5",model="gpt-5-nano",provider="openai",type="completion_rejected_prediction"} 0 -coder_aibridged_tokens_total{initiator_id="95f6752b-08cc-4cf1-97f7-c2165e3519c5",model="gpt-5-nano",provider="openai",type="input"} 13823 -coder_aibridged_tokens_total{initiator_id="95f6752b-08cc-4cf1-97f7-c2165e3519c5",model="gpt-5-nano",provider="openai",type="output"} 2014 -coder_aibridged_tokens_total{initiator_id="95f6752b-08cc-4cf1-97f7-c2165e3519c5",model="gpt-5-nano",provider="openai",type="prompt_audio"} 0 -coder_aibridged_tokens_total{initiator_id="95f6752b-08cc-4cf1-97f7-c2165e3519c5",model="gpt-5-nano",provider="openai",type="prompt_cached"} 31872 -# HELP coder_aibridged_circuit_breaker_rejects_total Total number of requests rejected due to open circuit breaker. -# TYPE coder_aibridged_circuit_breaker_rejects_total counter -coder_aibridged_circuit_breaker_rejects_total{provider="",endpoint="",model=""} 0 -# HELP coder_aibridged_circuit_breaker_state Current state of the circuit breaker (0=closed, 0.5=half-open, 1=open). -# TYPE coder_aibridged_circuit_breaker_state gauge -coder_aibridged_circuit_breaker_state{provider="",endpoint="",model=""} 0 -# HELP coder_aibridged_circuit_breaker_trips_total Total number of times the circuit breaker transitioned to open state. -# TYPE coder_aibridged_circuit_breaker_trips_total counter -coder_aibridged_circuit_breaker_trips_total{provider="",endpoint="",model=""} 0 -# HELP coder_aibridged_passthrough_total The count of requests which were not intercepted but passed through to the upstream. -# TYPE coder_aibridged_passthrough_total counter -coder_aibridged_passthrough_total{provider="",route="",method=""} 0 -# HELP coder_aibridgeproxyd_connect_sessions_total Total number of CONNECT sessions established. -# TYPE coder_aibridgeproxyd_connect_sessions_total counter -coder_aibridgeproxyd_connect_sessions_total{type=""} 0 -# HELP coder_aibridgeproxyd_inflight_mitm_requests Number of MITM requests currently being processed. -# TYPE coder_aibridgeproxyd_inflight_mitm_requests gauge -coder_aibridgeproxyd_inflight_mitm_requests{provider=""} 0 -# HELP coder_aibridgeproxyd_mitm_requests_total Total number of MITM requests handled by the proxy. -# TYPE coder_aibridgeproxyd_mitm_requests_total counter -coder_aibridgeproxyd_mitm_requests_total{provider=""} 0 -# HELP coder_aibridgeproxyd_mitm_responses_total Total number of MITM responses by HTTP status code class. -# TYPE coder_aibridgeproxyd_mitm_responses_total counter -coder_aibridgeproxyd_mitm_responses_total{code="",provider=""} 0 -# HELP coder_aibridged_provider_info One series per configured AI provider. Value is always 1; the status label (enabled, disabled, error) carries the alertable signal. -# TYPE coder_aibridged_provider_info gauge -coder_aibridged_provider_info{provider_name="",provider_type="",status=""} 0 -# HELP coder_aibridged_providers_last_reload_timestamp_seconds Unix timestamp of the last provider reload attempt, success or failure. -# TYPE coder_aibridged_providers_last_reload_timestamp_seconds gauge -coder_aibridged_providers_last_reload_timestamp_seconds 0 -# HELP coder_aibridged_providers_last_reload_success_timestamp_seconds Unix timestamp of the last provider reload that successfully refreshed the pool. A gap against coder_aibridged_providers_last_reload_timestamp_seconds means the loop is firing but the refresh function is failing. -# TYPE coder_aibridged_providers_last_reload_success_timestamp_seconds gauge -coder_aibridged_providers_last_reload_success_timestamp_seconds 0 -# HELP coder_aibridgeproxyd_provider_info One series per configured AI provider. Value is always 1; the status label (enabled, disabled, error) carries the alertable signal. -# TYPE coder_aibridgeproxyd_provider_info gauge -coder_aibridgeproxyd_provider_info{provider_name="",provider_type="",status=""} 0 -# HELP coder_aibridgeproxyd_providers_last_reload_timestamp_seconds Unix timestamp of the last provider reload attempt, success or failure. -# TYPE coder_aibridgeproxyd_providers_last_reload_timestamp_seconds gauge -coder_aibridgeproxyd_providers_last_reload_timestamp_seconds 0 -# HELP coder_aibridgeproxyd_providers_last_reload_success_timestamp_seconds Unix timestamp of the last provider reload that successfully refreshed the router. A gap against coder_aibridgeproxyd_providers_last_reload_timestamp_seconds means the loop is firing but the refresh function is failing. -# TYPE coder_aibridgeproxyd_providers_last_reload_success_timestamp_seconds gauge -coder_aibridgeproxyd_providers_last_reload_success_timestamp_seconds 0 +# HELP coder_ai_gateway_injected_tool_invocations_total The number of times an injected MCP tool was invoked by AI Gateway. +# TYPE coder_ai_gateway_injected_tool_invocations_total counter +coder_ai_gateway_injected_tool_invocations_total{model="gpt-5-nano",name="coder_list_templates",provider="openai",server="https://xxx.pit-1.try.coder.app/api/experimental/mcp/http"} 1 +# HELP coder_ai_gateway_interceptions_duration_seconds The total duration of intercepted requests, in seconds. The majority of this time will be the upstream processing of the request. AI Gateway has no control over upstream processing time, so it's just an illustrative metric. +# TYPE coder_ai_gateway_interceptions_duration_seconds histogram +coder_ai_gateway_interceptions_duration_seconds_bucket{model="gpt-5-nano",provider="openai",le="0.5"} 0 +coder_ai_gateway_interceptions_duration_seconds_bucket{model="gpt-5-nano",provider="openai",le="2"} 0 +coder_ai_gateway_interceptions_duration_seconds_bucket{model="gpt-5-nano",provider="openai",le="5"} 3 +coder_ai_gateway_interceptions_duration_seconds_bucket{model="gpt-5-nano",provider="openai",le="15"} 6 +coder_ai_gateway_interceptions_duration_seconds_bucket{model="gpt-5-nano",provider="openai",le="30"} 6 +coder_ai_gateway_interceptions_duration_seconds_bucket{model="gpt-5-nano",provider="openai",le="60"} 6 +coder_ai_gateway_interceptions_duration_seconds_bucket{model="gpt-5-nano",provider="openai",le="120"} 6 +coder_ai_gateway_interceptions_duration_seconds_bucket{model="gpt-5-nano",provider="openai",le="+Inf"} 6 +coder_ai_gateway_interceptions_duration_seconds_sum{model="gpt-5-nano",provider="openai"} 34.120188692 +coder_ai_gateway_interceptions_duration_seconds_count{model="gpt-5-nano",provider="openai"} 6 +# HELP coder_ai_gateway_interceptions_inflight The number of intercepted requests which are being processed. +# TYPE coder_ai_gateway_interceptions_inflight gauge +coder_ai_gateway_interceptions_inflight{model="gpt-5-nano",provider="openai",route="/v1/chat/completions"} 0 +# HELP coder_ai_gateway_interceptions_total The count of intercepted requests. +# TYPE coder_ai_gateway_interceptions_total counter +coder_ai_gateway_interceptions_total{client="Codex",initiator_id="95f6752b-08cc-4cf1-97f7-c2165e3519c5",method="POST",model="gpt-5-nano",provider="openai",route="/v1/chat/completions",status="completed"} 6 +# HELP coder_ai_gateway_key_pool_state The number of keys currently in each state (state: valid, temporary, permanent). +# TYPE coder_ai_gateway_key_pool_state gauge +coder_ai_gateway_key_pool_state{provider="openai",state="valid"} 2 +coder_ai_gateway_key_pool_state{provider="openai",state="temporary"} 0 +coder_ai_gateway_key_pool_state{provider="openai",state="permanent"} 0 +# HELP coder_ai_gateway_key_pool_state_transitions_total The number of API key state transitions during failover (reason: rate_limited, unauthorized, forbidden). +# TYPE coder_ai_gateway_key_pool_state_transitions_total counter +coder_ai_gateway_key_pool_state_transitions_total{provider="openai",reason="rate_limited"} 1 +# HELP coder_ai_gateway_key_pool_exhaustions_total The number of times the key pool was exhausted with no usable key (outcome: rate_limited, auth_failed). +# TYPE coder_ai_gateway_key_pool_exhaustions_total counter +coder_ai_gateway_key_pool_exhaustions_total{provider="openai",outcome="rate_limited"} 1 +# HELP coder_ai_gateway_key_pool_failover_attempts The number of keys attempted before success or exhaustion, per interception for bridged requests and per request for passthrough requests. +# TYPE coder_ai_gateway_key_pool_failover_attempts histogram +coder_ai_gateway_key_pool_failover_attempts_bucket{provider="openai",le="1"} 0 +coder_ai_gateway_key_pool_failover_attempts_bucket{provider="openai",le="2"} 1 +coder_ai_gateway_key_pool_failover_attempts_bucket{provider="openai",le="3"} 1 +coder_ai_gateway_key_pool_failover_attempts_bucket{provider="openai",le="4"} 1 +coder_ai_gateway_key_pool_failover_attempts_bucket{provider="openai",le="5"} 1 +coder_ai_gateway_key_pool_failover_attempts_bucket{provider="openai",le="10"} 1 +coder_ai_gateway_key_pool_failover_attempts_bucket{provider="openai",le="25"} 1 +coder_ai_gateway_key_pool_failover_attempts_bucket{provider="openai",le="+Inf"} 1 +coder_ai_gateway_key_pool_failover_attempts_sum{provider="openai"} 2 +coder_ai_gateway_key_pool_failover_attempts_count{provider="openai"} 1 +# HELP coder_ai_gateway_non_injected_tool_selections_total The number of times an AI model selected a tool to be invoked by the client. +# TYPE coder_ai_gateway_non_injected_tool_selections_total counter +coder_ai_gateway_non_injected_tool_selections_total{model="gpt-5-nano",name="read_file",provider="openai"} 2 +# HELP coder_ai_gateway_prompts_total The number of prompts issued by users (initiators). +# TYPE coder_ai_gateway_prompts_total counter +coder_ai_gateway_prompts_total{client="Codex",initiator_id="95f6752b-08cc-4cf1-97f7-c2165e3519c5",model="gpt-5-nano",provider="openai"} 4 +# HELP coder_ai_gateway_tokens_total The number of tokens used by intercepted requests. +# TYPE coder_ai_gateway_tokens_total counter +coder_ai_gateway_tokens_total{client="Codex",initiator_id="95f6752b-08cc-4cf1-97f7-c2165e3519c5",model="gpt-5-nano",provider="openai",type="completion_accepted_prediction"} 0 +coder_ai_gateway_tokens_total{client="Codex",initiator_id="95f6752b-08cc-4cf1-97f7-c2165e3519c5",model="gpt-5-nano",provider="openai",type="completion_audio"} 0 +coder_ai_gateway_tokens_total{client="Codex",initiator_id="95f6752b-08cc-4cf1-97f7-c2165e3519c5",model="gpt-5-nano",provider="openai",type="completion_reasoning"} 1664 +coder_ai_gateway_tokens_total{client="Codex",initiator_id="95f6752b-08cc-4cf1-97f7-c2165e3519c5",model="gpt-5-nano",provider="openai",type="completion_rejected_prediction"} 0 +coder_ai_gateway_tokens_total{client="Codex",initiator_id="95f6752b-08cc-4cf1-97f7-c2165e3519c5",model="gpt-5-nano",provider="openai",type="input"} 13823 +coder_ai_gateway_tokens_total{client="Codex",initiator_id="95f6752b-08cc-4cf1-97f7-c2165e3519c5",model="gpt-5-nano",provider="openai",type="output"} 2014 +coder_ai_gateway_tokens_total{client="Codex",initiator_id="95f6752b-08cc-4cf1-97f7-c2165e3519c5",model="gpt-5-nano",provider="openai",type="prompt_audio"} 0 +coder_ai_gateway_tokens_total{client="Codex",initiator_id="95f6752b-08cc-4cf1-97f7-c2165e3519c5",model="gpt-5-nano",provider="openai",type="prompt_cached"} 31872 +# HELP coder_ai_gateway_circuit_breaker_rejects_total Total number of requests rejected due to open circuit breaker. +# TYPE coder_ai_gateway_circuit_breaker_rejects_total counter +coder_ai_gateway_circuit_breaker_rejects_total{provider="",endpoint="",model=""} 0 +# HELP coder_ai_gateway_circuit_breaker_state Current state of the circuit breaker (0=closed, 0.5=half-open, 1=open). +# TYPE coder_ai_gateway_circuit_breaker_state gauge +coder_ai_gateway_circuit_breaker_state{provider="",endpoint="",model=""} 0 +# HELP coder_ai_gateway_circuit_breaker_trips_total Total number of times the circuit breaker transitioned to open state. +# TYPE coder_ai_gateway_circuit_breaker_trips_total counter +coder_ai_gateway_circuit_breaker_trips_total{provider="",endpoint="",model=""} 0 +# HELP coder_ai_gateway_passthrough_total The count of requests which were not intercepted but passed through to the upstream. +# TYPE coder_ai_gateway_passthrough_total counter +coder_ai_gateway_passthrough_total{provider="",route="",method=""} 0 +# HELP coder_ai_gateway_proxy_connect_sessions_total Total number of CONNECT sessions established. +# TYPE coder_ai_gateway_proxy_connect_sessions_total counter +coder_ai_gateway_proxy_connect_sessions_total{type=""} 0 +# HELP coder_ai_gateway_proxy_inflight_mitm_requests Number of MITM requests currently being processed. +# TYPE coder_ai_gateway_proxy_inflight_mitm_requests gauge +coder_ai_gateway_proxy_inflight_mitm_requests{provider=""} 0 +# HELP coder_ai_gateway_proxy_mitm_requests_total Total number of MITM requests handled by the proxy. +# TYPE coder_ai_gateway_proxy_mitm_requests_total counter +coder_ai_gateway_proxy_mitm_requests_total{provider=""} 0 +# HELP coder_ai_gateway_proxy_mitm_responses_total Total number of MITM responses by HTTP status code class. +# TYPE coder_ai_gateway_proxy_mitm_responses_total counter +coder_ai_gateway_proxy_mitm_responses_total{code="",provider=""} 0 +# HELP coder_ai_gateway_provider_info One series per configured AI provider. Value is always 1; the status label (enabled, disabled, error) carries the alertable signal. +# TYPE coder_ai_gateway_provider_info gauge +coder_ai_gateway_provider_info{provider_name="",provider_type="",status=""} 0 +# HELP coder_ai_gateway_providers_last_reload_timestamp_seconds Unix timestamp of the last provider reload attempt, success or failure. +# TYPE coder_ai_gateway_providers_last_reload_timestamp_seconds gauge +coder_ai_gateway_providers_last_reload_timestamp_seconds 0 +# HELP coder_ai_gateway_providers_last_reload_success_timestamp_seconds Unix timestamp of the last provider reload that successfully refreshed the pool. A gap against the providers_last_reload_timestamp_seconds gauge means the loop is firing but the refresh function is failing. +# TYPE coder_ai_gateway_providers_last_reload_success_timestamp_seconds gauge +coder_ai_gateway_providers_last_reload_success_timestamp_seconds 0 +# HELP coder_ai_gateway_proxy_provider_info One series per configured AI provider. Value is always 1; the status label (enabled, disabled, error) carries the alertable signal. +# TYPE coder_ai_gateway_proxy_provider_info gauge +coder_ai_gateway_proxy_provider_info{provider_name="",provider_type="",status=""} 0 +# HELP coder_ai_gateway_proxy_providers_last_reload_timestamp_seconds Unix timestamp of the last provider reload attempt, success or failure. +# TYPE coder_ai_gateway_proxy_providers_last_reload_timestamp_seconds gauge +coder_ai_gateway_proxy_providers_last_reload_timestamp_seconds 0 +# HELP coder_ai_gateway_proxy_providers_last_reload_success_timestamp_seconds Unix timestamp of the last provider reload that successfully refreshed the router. A gap against the providers_last_reload_timestamp_seconds gauge means the loop is firing but the refresh function is failing. +# TYPE coder_ai_gateway_proxy_providers_last_reload_success_timestamp_seconds gauge +coder_ai_gateway_proxy_providers_last_reload_success_timestamp_seconds 0