From a2ce74f398dc4ad6c12b83e7c02de190422eb768 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Mon, 6 Apr 2026 12:10:57 -0400 Subject: [PATCH] feat: add total_runtime_ms to chat cost analytics endpoints (#24050) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Surface the aggregated `runtime_ms` from `chat_messages` through all four cost analytics queries (summary, per-model, per-chat, per-user). This is the key billing metric for agent compute time. The per-chat breakdown already groups by `root_chat_id`, so subagent runtime is automatically rolled up under the parent chat — no additional query changes needed.
Implementation details **SQL** (`coderd/database/queries/chats.sql`): Added `COALESCE(SUM(cm.runtime_ms), 0)::bigint AS total_runtime_ms` to `GetChatCostSummary`, `GetChatCostPerModel`, `GetChatCostPerChat`, and `GetChatCostPerUser`. **Go SDK** (`codersdk/chats.go`): Added `TotalRuntimeMs int64` to `ChatCostSummary`, `ChatCostModelBreakdown`, `ChatCostChatBreakdown`, and `ChatCostUserRollup`. **Handler** (`coderd/exp_chats.go`): Wired the new field through all converter functions and the response assembly. **Tests** (`coderd/exp_chats_test.go`): Updated fixture to seed non-zero `runtime_ms` values and added assertions for the new field at summary, per-model, and per-chat levels.
> 🤖 Generated by Coder Agents --- coderd/database/queries.sql.go | 24 +++++++++++++++---- coderd/database/queries/chats.sql | 16 +++++++++---- coderd/exp_chats.go | 4 ++++ coderd/exp_chats_test.go | 5 +++- codersdk/chats.go | 4 ++++ site/src/api/typesGenerated.ts | 4 ++++ .../AgentSettingsUsagePageView.stories.tsx | 5 ++++ .../AgentsPage/AgentsPageView.stories.tsx | 4 ++++ .../ChatCostSummaryView.stories.tsx | 3 +++ 9 files changed, 58 insertions(+), 11 deletions(-) diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index a57c3771ef..faf6dd4e9c 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -4530,7 +4530,8 @@ WITH chat_costs AS ( COALESCE(SUM(cm.input_tokens), 0)::bigint AS total_input_tokens, COALESCE(SUM(cm.output_tokens), 0)::bigint AS total_output_tokens, COALESCE(SUM(cm.cache_read_tokens), 0)::bigint AS total_cache_read_tokens, - COALESCE(SUM(cm.cache_creation_tokens), 0)::bigint AS total_cache_creation_tokens + COALESCE(SUM(cm.cache_creation_tokens), 0)::bigint AS total_cache_creation_tokens, + COALESCE(SUM(cm.runtime_ms), 0)::bigint AS total_runtime_ms FROM chat_messages cm JOIN chats c ON c.id = cm.chat_id WHERE c.owner_id = $1::uuid @@ -4547,7 +4548,8 @@ SELECT cc.total_input_tokens, cc.total_output_tokens, cc.total_cache_read_tokens, - cc.total_cache_creation_tokens + cc.total_cache_creation_tokens, + cc.total_runtime_ms FROM chat_costs cc LEFT JOIN chats rc ON rc.id = cc.root_chat_id ORDER BY cc.total_cost_micros DESC @@ -4568,6 +4570,7 @@ type GetChatCostPerChatRow struct { TotalOutputTokens int64 `db:"total_output_tokens" json:"total_output_tokens"` TotalCacheReadTokens int64 `db:"total_cache_read_tokens" json:"total_cache_read_tokens"` TotalCacheCreationTokens int64 `db:"total_cache_creation_tokens" json:"total_cache_creation_tokens"` + TotalRuntimeMs int64 `db:"total_runtime_ms" json:"total_runtime_ms"` } // Per-root-chat cost breakdown for a single user within a date range. @@ -4591,6 +4594,7 @@ func (q *sqlQuerier) GetChatCostPerChat(ctx context.Context, arg GetChatCostPerC &i.TotalOutputTokens, &i.TotalCacheReadTokens, &i.TotalCacheCreationTokens, + &i.TotalRuntimeMs, ); err != nil { return nil, err } @@ -4622,7 +4626,8 @@ SELECT COALESCE(SUM(cm.input_tokens), 0)::bigint AS total_input_tokens, COALESCE(SUM(cm.output_tokens), 0)::bigint AS total_output_tokens, COALESCE(SUM(cm.cache_read_tokens), 0)::bigint AS total_cache_read_tokens, - COALESCE(SUM(cm.cache_creation_tokens), 0)::bigint AS total_cache_creation_tokens + COALESCE(SUM(cm.cache_creation_tokens), 0)::bigint AS total_cache_creation_tokens, + COALESCE(SUM(cm.runtime_ms), 0)::bigint AS total_runtime_ms FROM chat_messages cm JOIN @@ -4657,6 +4662,7 @@ type GetChatCostPerModelRow struct { TotalOutputTokens int64 `db:"total_output_tokens" json:"total_output_tokens"` TotalCacheReadTokens int64 `db:"total_cache_read_tokens" json:"total_cache_read_tokens"` TotalCacheCreationTokens int64 `db:"total_cache_creation_tokens" json:"total_cache_creation_tokens"` + TotalRuntimeMs int64 `db:"total_runtime_ms" json:"total_runtime_ms"` } // Per-model cost breakdown for a single user within a date range. @@ -4681,6 +4687,7 @@ func (q *sqlQuerier) GetChatCostPerModel(ctx context.Context, arg GetChatCostPer &i.TotalOutputTokens, &i.TotalCacheReadTokens, &i.TotalCacheCreationTokens, + &i.TotalRuntimeMs, ); err != nil { return nil, err } @@ -4714,7 +4721,8 @@ WITH chat_cost_users AS ( COALESCE(SUM(cm.input_tokens), 0)::bigint AS total_input_tokens, COALESCE(SUM(cm.output_tokens), 0)::bigint AS total_output_tokens, COALESCE(SUM(cm.cache_read_tokens), 0)::bigint AS total_cache_read_tokens, - COALESCE(SUM(cm.cache_creation_tokens), 0)::bigint AS total_cache_creation_tokens + COALESCE(SUM(cm.cache_creation_tokens), 0)::bigint AS total_cache_creation_tokens, + COALESCE(SUM(cm.runtime_ms), 0)::bigint AS total_runtime_ms FROM chat_messages cm JOIN @@ -4748,6 +4756,7 @@ SELECT total_output_tokens, total_cache_read_tokens, total_cache_creation_tokens, + total_runtime_ms, COUNT(*) OVER()::bigint AS total_count FROM chat_cost_users @@ -4780,6 +4789,7 @@ type GetChatCostPerUserRow struct { TotalOutputTokens int64 `db:"total_output_tokens" json:"total_output_tokens"` TotalCacheReadTokens int64 `db:"total_cache_read_tokens" json:"total_cache_read_tokens"` TotalCacheCreationTokens int64 `db:"total_cache_creation_tokens" json:"total_cache_creation_tokens"` + TotalRuntimeMs int64 `db:"total_runtime_ms" json:"total_runtime_ms"` TotalCount int64 `db:"total_count" json:"total_count"` } @@ -4812,6 +4822,7 @@ func (q *sqlQuerier) GetChatCostPerUser(ctx context.Context, arg GetChatCostPerU &i.TotalOutputTokens, &i.TotalCacheReadTokens, &i.TotalCacheCreationTokens, + &i.TotalRuntimeMs, &i.TotalCount, ); err != nil { return nil, err @@ -4846,7 +4857,8 @@ SELECT COALESCE(SUM(cm.input_tokens), 0)::bigint AS total_input_tokens, COALESCE(SUM(cm.output_tokens), 0)::bigint AS total_output_tokens, COALESCE(SUM(cm.cache_read_tokens), 0)::bigint AS total_cache_read_tokens, - COALESCE(SUM(cm.cache_creation_tokens), 0)::bigint AS total_cache_creation_tokens + COALESCE(SUM(cm.cache_creation_tokens), 0)::bigint AS total_cache_creation_tokens, + COALESCE(SUM(cm.runtime_ms), 0)::bigint AS total_runtime_ms FROM chat_messages cm JOIN @@ -4872,6 +4884,7 @@ type GetChatCostSummaryRow struct { TotalOutputTokens int64 `db:"total_output_tokens" json:"total_output_tokens"` TotalCacheReadTokens int64 `db:"total_cache_read_tokens" json:"total_cache_read_tokens"` TotalCacheCreationTokens int64 `db:"total_cache_creation_tokens" json:"total_cache_creation_tokens"` + TotalRuntimeMs int64 `db:"total_runtime_ms" json:"total_runtime_ms"` } // Aggregate cost summary for a single user within a date range. @@ -4887,6 +4900,7 @@ func (q *sqlQuerier) GetChatCostSummary(ctx context.Context, arg GetChatCostSumm &i.TotalOutputTokens, &i.TotalCacheReadTokens, &i.TotalCacheCreationTokens, + &i.TotalRuntimeMs, ) return i, err } diff --git a/coderd/database/queries/chats.sql b/coderd/database/queries/chats.sql index 95743ab3e9..a7c0bf6740 100644 --- a/coderd/database/queries/chats.sql +++ b/coderd/database/queries/chats.sql @@ -883,7 +883,8 @@ SELECT COALESCE(SUM(cm.input_tokens), 0)::bigint AS total_input_tokens, COALESCE(SUM(cm.output_tokens), 0)::bigint AS total_output_tokens, COALESCE(SUM(cm.cache_read_tokens), 0)::bigint AS total_cache_read_tokens, - COALESCE(SUM(cm.cache_creation_tokens), 0)::bigint AS total_cache_creation_tokens + COALESCE(SUM(cm.cache_creation_tokens), 0)::bigint AS total_cache_creation_tokens, + COALESCE(SUM(cm.runtime_ms), 0)::bigint AS total_runtime_ms FROM chat_messages cm JOIN @@ -913,7 +914,8 @@ SELECT COALESCE(SUM(cm.input_tokens), 0)::bigint AS total_input_tokens, COALESCE(SUM(cm.output_tokens), 0)::bigint AS total_output_tokens, COALESCE(SUM(cm.cache_read_tokens), 0)::bigint AS total_cache_read_tokens, - COALESCE(SUM(cm.cache_creation_tokens), 0)::bigint AS total_cache_creation_tokens + COALESCE(SUM(cm.cache_creation_tokens), 0)::bigint AS total_cache_creation_tokens, + COALESCE(SUM(cm.runtime_ms), 0)::bigint AS total_runtime_ms FROM chat_messages cm JOIN @@ -948,7 +950,8 @@ WITH chat_costs AS ( COALESCE(SUM(cm.input_tokens), 0)::bigint AS total_input_tokens, COALESCE(SUM(cm.output_tokens), 0)::bigint AS total_output_tokens, COALESCE(SUM(cm.cache_read_tokens), 0)::bigint AS total_cache_read_tokens, - COALESCE(SUM(cm.cache_creation_tokens), 0)::bigint AS total_cache_creation_tokens + COALESCE(SUM(cm.cache_creation_tokens), 0)::bigint AS total_cache_creation_tokens, + COALESCE(SUM(cm.runtime_ms), 0)::bigint AS total_runtime_ms FROM chat_messages cm JOIN chats c ON c.id = cm.chat_id WHERE c.owner_id = @owner_id::uuid @@ -965,7 +968,8 @@ SELECT cc.total_input_tokens, cc.total_output_tokens, cc.total_cache_read_tokens, - cc.total_cache_creation_tokens + cc.total_cache_creation_tokens, + cc.total_runtime_ms FROM chat_costs cc LEFT JOIN chats rc ON rc.id = cc.root_chat_id ORDER BY cc.total_cost_micros DESC; @@ -991,7 +995,8 @@ WITH chat_cost_users AS ( COALESCE(SUM(cm.input_tokens), 0)::bigint AS total_input_tokens, COALESCE(SUM(cm.output_tokens), 0)::bigint AS total_output_tokens, COALESCE(SUM(cm.cache_read_tokens), 0)::bigint AS total_cache_read_tokens, - COALESCE(SUM(cm.cache_creation_tokens), 0)::bigint AS total_cache_creation_tokens + COALESCE(SUM(cm.cache_creation_tokens), 0)::bigint AS total_cache_creation_tokens, + COALESCE(SUM(cm.runtime_ms), 0)::bigint AS total_runtime_ms FROM chat_messages cm JOIN @@ -1025,6 +1030,7 @@ SELECT total_output_tokens, total_cache_read_tokens, total_cache_creation_tokens, + total_runtime_ms, COUNT(*) OVER()::bigint AS total_count FROM chat_cost_users diff --git a/coderd/exp_chats.go b/coderd/exp_chats.go index 6d352fdc74..3260ba9eec 100644 --- a/coderd/exp_chats.go +++ b/coderd/exp_chats.go @@ -717,6 +717,7 @@ func (api *API) chatCostSummary(rw http.ResponseWriter, r *http.Request) { TotalOutputTokens: summary.TotalOutputTokens, TotalCacheReadTokens: summary.TotalCacheReadTokens, TotalCacheCreationTokens: summary.TotalCacheCreationTokens, + TotalRuntimeMs: summary.TotalRuntimeMs, ByModel: modelBreakdowns, ByChat: chatBreakdowns, } @@ -3840,6 +3841,7 @@ func convertChatCostModelBreakdown(model database.GetChatCostPerModelRow) coders TotalOutputTokens: model.TotalOutputTokens, TotalCacheReadTokens: model.TotalCacheReadTokens, TotalCacheCreationTokens: model.TotalCacheCreationTokens, + TotalRuntimeMs: model.TotalRuntimeMs, } } @@ -3853,6 +3855,7 @@ func convertChatCostChatBreakdown(chat database.GetChatCostPerChatRow) codersdk. TotalOutputTokens: chat.TotalOutputTokens, TotalCacheReadTokens: chat.TotalCacheReadTokens, TotalCacheCreationTokens: chat.TotalCacheCreationTokens, + TotalRuntimeMs: chat.TotalRuntimeMs, } } @@ -3869,6 +3872,7 @@ func convertChatCostUserRollup(user database.GetChatCostPerUserRow) codersdk.Cha TotalOutputTokens: user.TotalOutputTokens, TotalCacheReadTokens: user.TotalCacheReadTokens, TotalCacheCreationTokens: user.TotalCacheCreationTokens, + TotalRuntimeMs: user.TotalRuntimeMs, } } diff --git a/coderd/exp_chats_test.go b/coderd/exp_chats_test.go index 4e9b23afc8..07df3f2f23 100644 --- a/coderd/exp_chats_test.go +++ b/coderd/exp_chats_test.go @@ -6079,7 +6079,7 @@ func seedChatCostFixture(t *testing.T) chatCostTestFixture { ContextLimit: []int64{0, 0}, Compressed: []bool{false, false}, TotalCostMicros: []int64{500, 500}, - RuntimeMs: []int64{0, 0}, + RuntimeMs: []int64{1500, 2500}, }) require.NoError(t, err) require.Len(t, results, 2) @@ -6112,16 +6112,19 @@ func assertChatCostSummary(t *testing.T, summary codersdk.ChatCostSummary, model require.Equal(t, int64(0), summary.UnpricedMessageCount) require.Equal(t, int64(200), summary.TotalInputTokens) require.Equal(t, int64(100), summary.TotalOutputTokens) + require.Equal(t, int64(4000), summary.TotalRuntimeMs) require.Len(t, summary.ByModel, 1) require.Equal(t, modelConfigID, summary.ByModel[0].ModelConfigID) require.Equal(t, int64(1000), summary.ByModel[0].TotalCostMicros) require.Equal(t, int64(2), summary.ByModel[0].MessageCount) + require.Equal(t, int64(4000), summary.ByModel[0].TotalRuntimeMs) require.Len(t, summary.ByChat, 1) require.Equal(t, chatID, summary.ByChat[0].RootChatID) require.Equal(t, int64(1000), summary.ByChat[0].TotalCostMicros) require.Equal(t, int64(2), summary.ByChat[0].MessageCount) + require.Equal(t, int64(4000), summary.ByChat[0].TotalRuntimeMs) } func TestChatCostSummary(t *testing.T) { diff --git a/codersdk/chats.go b/codersdk/chats.go index 55e5e11662..af69c03084 100644 --- a/codersdk/chats.go +++ b/codersdk/chats.go @@ -974,6 +974,7 @@ type ChatCostSummary struct { TotalOutputTokens int64 `json:"total_output_tokens"` TotalCacheReadTokens int64 `json:"total_cache_read_tokens"` TotalCacheCreationTokens int64 `json:"total_cache_creation_tokens"` + TotalRuntimeMs int64 `json:"total_runtime_ms"` ByModel []ChatCostModelBreakdown `json:"by_model"` ByChat []ChatCostChatBreakdown `json:"by_chat"` UsageLimit *ChatUsageLimitStatus `json:"usage_limit,omitempty"` @@ -991,6 +992,7 @@ type ChatCostModelBreakdown struct { TotalOutputTokens int64 `json:"total_output_tokens"` TotalCacheReadTokens int64 `json:"total_cache_read_tokens"` TotalCacheCreationTokens int64 `json:"total_cache_creation_tokens"` + TotalRuntimeMs int64 `json:"total_runtime_ms"` } // ChatCostChatBreakdown contains per-root-chat cost aggregation. @@ -1003,6 +1005,7 @@ type ChatCostChatBreakdown struct { TotalOutputTokens int64 `json:"total_output_tokens"` TotalCacheReadTokens int64 `json:"total_cache_read_tokens"` TotalCacheCreationTokens int64 `json:"total_cache_creation_tokens"` + TotalRuntimeMs int64 `json:"total_runtime_ms"` } // ChatCostUserRollup contains per-user cost aggregation for admin views. @@ -1018,6 +1021,7 @@ type ChatCostUserRollup struct { TotalOutputTokens int64 `json:"total_output_tokens"` TotalCacheReadTokens int64 `json:"total_cache_read_tokens"` TotalCacheCreationTokens int64 `json:"total_cache_creation_tokens"` + TotalRuntimeMs int64 `json:"total_runtime_ms"` } // ChatCostUsersResponse is the response from the admin chat cost users endpoint. diff --git a/site/src/api/typesGenerated.ts b/site/src/api/typesGenerated.ts index 76b2eefdcd..e14e2a8c26 100644 --- a/site/src/api/typesGenerated.ts +++ b/site/src/api/typesGenerated.ts @@ -1263,6 +1263,7 @@ export interface ChatCostChatBreakdown { readonly total_output_tokens: number; readonly total_cache_read_tokens: number; readonly total_cache_creation_tokens: number; + readonly total_runtime_ms: number; } // From codersdk/chats.go @@ -1280,6 +1281,7 @@ export interface ChatCostModelBreakdown { readonly total_output_tokens: number; readonly total_cache_read_tokens: number; readonly total_cache_creation_tokens: number; + readonly total_runtime_ms: number; } // From codersdk/chats.go @@ -1296,6 +1298,7 @@ export interface ChatCostSummary { readonly total_output_tokens: number; readonly total_cache_read_tokens: number; readonly total_cache_creation_tokens: number; + readonly total_runtime_ms: number; readonly by_model: readonly ChatCostModelBreakdown[]; readonly by_chat: readonly ChatCostChatBreakdown[]; readonly usage_limit?: ChatUsageLimitStatus; @@ -1326,6 +1329,7 @@ export interface ChatCostUserRollup { readonly total_output_tokens: number; readonly total_cache_read_tokens: number; readonly total_cache_creation_tokens: number; + readonly total_runtime_ms: number; } // From codersdk/chats.go diff --git a/site/src/pages/AgentsPage/AgentSettingsUsagePageView.stories.tsx b/site/src/pages/AgentsPage/AgentSettingsUsagePageView.stories.tsx index 8a53102083..67303f8b9e 100644 --- a/site/src/pages/AgentsPage/AgentSettingsUsagePageView.stories.tsx +++ b/site/src/pages/AgentsPage/AgentSettingsUsagePageView.stories.tsx @@ -16,6 +16,7 @@ const mockUsers: TypesGen.ChatCostUserRollup[] = [ total_output_tokens: 300_000, total_cache_read_tokens: 10_000, total_cache_creation_tokens: 5_000, + total_runtime_ms: 0, }, { user_id: "user-2", @@ -29,6 +30,7 @@ const mockUsers: TypesGen.ChatCostUserRollup[] = [ total_output_tokens: 120_000, total_cache_read_tokens: 4_000, total_cache_creation_tokens: 2_000, + total_runtime_ms: 0, }, ]; @@ -65,6 +67,7 @@ const mockCostSummary: TypesGen.ChatCostSummary = { total_output_tokens: 300_000, total_cache_read_tokens: 10_000, total_cache_creation_tokens: 5_000, + total_runtime_ms: 0, by_model: [ { model_config_id: "model-1", @@ -77,6 +80,7 @@ const mockCostSummary: TypesGen.ChatCostSummary = { total_output_tokens: 250_000, total_cache_read_tokens: 8_000, total_cache_creation_tokens: 4_000, + total_runtime_ms: 0, }, ], by_chat: [ @@ -89,6 +93,7 @@ const mockCostSummary: TypesGen.ChatCostSummary = { total_output_tokens: 120_000, total_cache_read_tokens: 3_000, total_cache_creation_tokens: 1_500, + total_runtime_ms: 0, }, ], }; diff --git a/site/src/pages/AgentsPage/AgentsPageView.stories.tsx b/site/src/pages/AgentsPage/AgentsPageView.stories.tsx index 1092329e19..e7c37e801a 100644 --- a/site/src/pages/AgentsPage/AgentsPageView.stories.tsx +++ b/site/src/pages/AgentsPage/AgentsPageView.stories.tsx @@ -70,6 +70,7 @@ const mockAnalyticsSummary: TypesGen.ChatCostSummary = { total_output_tokens: 654_321, total_cache_read_tokens: 9_876, total_cache_creation_tokens: 5_432, + total_runtime_ms: 0, by_model: [ { model_config_id: defaultModelConfigID, @@ -82,6 +83,7 @@ const mockAnalyticsSummary: TypesGen.ChatCostSummary = { total_output_tokens: 200_000, total_cache_read_tokens: 7_654, total_cache_creation_tokens: 3_210, + total_runtime_ms: 0, }, ], by_chat: [ @@ -94,6 +96,7 @@ const mockAnalyticsSummary: TypesGen.ChatCostSummary = { total_output_tokens: 80_000, total_cache_read_tokens: 4_321, total_cache_creation_tokens: 1_234, + total_runtime_ms: 0, }, ], }; @@ -115,6 +118,7 @@ const mockUsageUsers: TypesGen.ChatCostUsersResponse = { total_output_tokens: 45_000, total_cache_read_tokens: 6_789, total_cache_creation_tokens: 2_468, + total_runtime_ms: 0, }, ], }; diff --git a/site/src/pages/AgentsPage/components/ChatCostSummaryView.stories.tsx b/site/src/pages/AgentsPage/components/ChatCostSummaryView.stories.tsx index e08be17284..e37eeff262 100644 --- a/site/src/pages/AgentsPage/components/ChatCostSummaryView.stories.tsx +++ b/site/src/pages/AgentsPage/components/ChatCostSummaryView.stories.tsx @@ -15,6 +15,7 @@ const buildSummary = ( total_output_tokens: 654_321, total_cache_read_tokens: 9_876, total_cache_creation_tokens: 5_432, + total_runtime_ms: 0, by_model: [ { model_config_id: "model-config-1", @@ -27,6 +28,7 @@ const buildSummary = ( total_output_tokens: 200_000, total_cache_read_tokens: 7_654, total_cache_creation_tokens: 3_210, + total_runtime_ms: 0, }, ], by_chat: [ @@ -39,6 +41,7 @@ const buildSummary = ( total_output_tokens: 80_000, total_cache_read_tokens: 4_321, total_cache_creation_tokens: 1_234, + total_runtime_ms: 0, }, ], ...overrides,