opensource/coder
3
0
Fork 0
mirror of https://github.com/coder/coder.git synced 2026-09-24 15:04:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: report per-request Anthropic usage in chat token accounting (#27966)

Browse Source
This commit is contained in:
Michael Suchacz
2026-08-12 12:52:22 +02:00
committed by GitHub
parent ac11ae52d1
commit 88e113554a
4 changed files with 78 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
coderd/x/chatd/chatd_test.go
+39
View File
@@ -6872,6 +6872,45 @@ func TestActiveServer_AnthropicUsageMatchesFinalDelta(t *testing.T) {
require.Equal(t, sql.NullInt64{Int64: 150, Valid: true}, last.CacheReadTokens)
}
func TestActiveServer_AnthropicPersistsPerRequestUsage(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitLong)
db, ps := dbtestutil.NewDB(t)
anthropicURL := chattest.NewAnthropic(t, func(_ *chattest.AnthropicRequest) chattest.AnthropicResponse {
return chattest.AnthropicStreamingResponse(chattest.AnthropicTextChunksWithMessageUsages(
chattest.AnthropicUsage{
InputTokens: 2,
CacheReadInputTokens: 139956,
CacheCreationInputTokens: 7770,
},
chattest.AnthropicUsage{
InputTokens: 4,
CacheReadInputTokens: 287682,
CacheCreationInputTokens: 15556,
OutputTokens: 4996,
},
"cached response",
)...)
})
user, org, model := seedAnthropicChatDependencies(t, db, anthropicURL)
server := newActiveTestServer(t, db, ps, func(cfg *chatd.Config) {
cfg.AIBridgeTransportFactory = chatAIGatewayTransportFactoryPointer(chattest.NewMockAIBridgeTransport(t, anthropicURL, chattest.WithPreservePath()))
})
chat := createChatThroughServer(ctx, t, db, server, org.ID, user.ID, model.ID, "hello")
waitForChatStatus(ctx, t, db, chat.ID, database.ChatStatusWaiting)
messages := chatMessages(ctx, t, db, chat.ID)
last := messages[len(messages)-1]
require.Equal(t, database.ChatMessageRoleAssistant, last.Role)
require.Equal(t, sql.NullInt64{Int64: 139956, Valid: true}, last.CacheReadTokens)
require.Equal(t, sql.NullInt64{Int64: 7770, Valid: true}, last.CacheCreationTokens)
require.Equal(t, sql.NullInt64{Int64: 2, Valid: true}, last.InputTokens)
require.Equal(t, sql.NullInt64{Int64: 4996, Valid: true}, last.OutputTokens)
require.Equal(t, sql.NullInt64{Int64: 4998, Valid: true}, last.TotalTokens)
}
func TestActiveServer_ChatTurnDebugRunRecordsStreamStep(t *testing.T) {
t.Parallel()
coderd/x/chatd/chattest/anthropic.go
+32 -11
View File
@@ -375,6 +375,16 @@ func AnthropicTextChunks(deltas ...string) []AnthropicChunk {
// the initial input and cache token counts, and the final message_delta
// carries the output token count.
func AnthropicTextChunksWithCacheUsage(usage AnthropicUsage, deltas ...string) []AnthropicChunk {
return AnthropicTextChunksWithMessageUsages(
usage,
AnthropicUsage{OutputTokens: usage.OutputTokens},
deltas...,
)
}
// AnthropicTextChunksWithMessageUsages creates a streaming response with
// independent message_start and message_delta usage.
func AnthropicTextChunksWithMessageUsages(messageStartUsage, messageDeltaUsage AnthropicUsage, deltas ...string) []AnthropicChunk {
if len(deltas) == 0 {
return nil
}
@@ -383,13 +393,26 @@ func AnthropicTextChunksWithCacheUsage(usage AnthropicUsage, deltas ...string) [
model := "claude-3-opus-20240229"
messageUsage := map[string]int{
"input_tokens": usage.InputTokens,
"input_tokens": messageStartUsage.InputTokens,
}
if usage.CacheCreationInputTokens != 0 {
messageUsage["cache_creation_input_tokens"] = usage.CacheCreationInputTokens
if messageStartUsage.CacheCreationInputTokens != 0 {
messageUsage["cache_creation_input_tokens"] = messageStartUsage.CacheCreationInputTokens
}
if usage.CacheReadInputTokens != 0 {
messageUsage["cache_read_input_tokens"] = usage.CacheReadInputTokens
if messageStartUsage.CacheReadInputTokens != 0 {
messageUsage["cache_read_input_tokens"] = messageStartUsage.CacheReadInputTokens
}
deltaUsage := map[string]int{
"output_tokens": messageDeltaUsage.OutputTokens,
}
if messageDeltaUsage.InputTokens != 0 {
deltaUsage["input_tokens"] = messageDeltaUsage.InputTokens
}
if messageDeltaUsage.CacheCreationInputTokens != 0 {
deltaUsage["cache_creation_input_tokens"] = messageDeltaUsage.CacheCreationInputTokens
}
if messageDeltaUsage.CacheReadInputTokens != 0 {
deltaUsage["cache_read_input_tokens"] = messageDeltaUsage.CacheReadInputTokens
}
chunks := []AnthropicChunk{
@@ -432,9 +455,7 @@ func AnthropicTextChunksWithCacheUsage(usage AnthropicUsage, deltas ...string) [
AnthropicChunk{
Type: "message_delta",
StopReason: "end_turn",
UsageMap: map[string]int{
"output_tokens": usage.OutputTokens,
},
UsageMap: deltaUsage,
},
AnthropicChunk{
Type: "message_stop",
@@ -556,6 +577,7 @@ func AnthropicToolCallChunks(toolName string, inputJSONDeltas ...string) []Anthr
Type: "message",
Role: "assistant",
Model: model,
Usage: map[string]int{"input_tokens": 10},
},
},
{
@@ -589,9 +611,8 @@ func AnthropicToolCallChunks(toolName string, inputJSONDeltas ...string) []Anthr
AnthropicChunk{
Type: "message_delta",
StopReason: "tool_use",
Usage: AnthropicUsage{
InputTokens: 10,
OutputTokens: 5,
UsageMap: map[string]int{
"output_tokens": 5,
},
},
AnthropicChunk{
go.mod
+5 -3
View File
@@ -94,10 +94,12 @@ replace github.com/spf13/afero => github.com/aslilac/afero v0.0.0-20250403163713
// 5) coder/fantasy bedrock: mirror the request region when prefixing
// cross-region inference profiles so the model-ID prefix matches the
// region the request is actually signed for.
// 6) go.mod replaces pointing anthropic-sdk-go and openai-go at the
// 6) coder/fantasy#50, report per-request Anthropic usage at Finish so
// multi-iteration requests do not inflate context estimates.
// 7) go.mod replaces pointing anthropic-sdk-go and openai-go at the
// coder forks below.
// See: https://github.com/coder/fantasy/commits/bb10946892ef
replace charm.land/fantasy => github.com/coder/fantasy v0.0.0-20260810175832-bb10946892ef
// See: https://github.com/coder/fantasy/commits/6f8df3735907
replace charm.land/fantasy => github.com/coder/fantasy v0.0.0-20260812075759-6f8df3735907
// coder/coder uses a fork of charmbracelet's fork of the Anthropic Go SDK
// with performance improvements and Bedrock header cleanup.
go.sum
+2 -2
View File
@@ -327,8 +327,8 @@ github.com/coder/bubbletea v1.2.2-0.20241212190825-007a1cdb2c41 h1:SBN/DA63+ZHwu
github.com/coder/bubbletea v1.2.2-0.20241212190825-007a1cdb2c41/go.mod h1:I9ULxr64UaOSUv7hcb3nX4kowodJCVS7vt7VVJk/kW4=
github.com/coder/clistat v1.2.1 h1:P9/10njXMyj5cWzIU5wkRsSy5LVQH49+tcGMsAgWX0w=
github.com/coder/clistat v1.2.1/go.mod h1:m7SC0uj88eEERgvF8Kn6+w6XF21BeSr+15f7GoLAw0A=
github.com/coder/fantasy v0.0.0-20260810175832-bb10946892ef h1:p2FYbdIwvybIUctKHQ3/53+Wta8gO91NLTEuvyuWQj0=
github.com/coder/fantasy v0.0.0-20260810175832-bb10946892ef/go.mod h1:lcL8B/uroFq5YDHNFij2an0yarOtD7C3rmzD31i8oTA=
github.com/coder/fantasy v0.0.0-20260812075759-6f8df3735907 h1:Mirl1lw+0MsndEL0feesyIqtxTdpiKrR8AH4zcA6SZU=
github.com/coder/fantasy v0.0.0-20260812075759-6f8df3735907/go.mod h1:lcL8B/uroFq5YDHNFij2an0yarOtD7C3rmzD31i8oTA=
github.com/coder/flog v1.1.0 h1:kbAes1ai8fIS5OeV+QAnKBQE22ty1jRF/mcAwHpLBa4=
github.com/coder/flog v1.1.0/go.mod h1:UQlQvrkJBvnRGo69Le8E24Tcl5SJleAAR7gYEHzAmdQ=
github.com/coder/go-httpstat v0.0.0-20230801153223-321c88088322 h1:m0lPZjlQ7vdVpRBPKfYIFlmgevoTkBxB10wv6l2gOaU=