mirror of
https://github.com/Wei-Shaw/sub2api.git
synced 2026-10-16 14:57:36 +08:00
Merge pull request #7380 from wucm667/fix/issue-7377-forward-stream-usage
fix(gateway): forward received chat stream usage
This commit is contained in:
@@ -0,0 +1,123 @@
|
||||
//go:build unit
|
||||
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/pkg/apicompat"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestAnthropicChatStreamAuthoritativeUsage(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
for _, adapter := range []string{"anthropic", "native"} {
|
||||
for _, options := range []string{"", `,"stream_options":{"include_usage":true}`, `,"stream_options":{"include_usage":false}`} {
|
||||
for _, tc := range []struct {
|
||||
name, start, delta, repeatDelta string
|
||||
want bool
|
||||
prompt, billableInput, totalInput, output, cached, created int
|
||||
}{
|
||||
{"cache", `,"usage":{"input_tokens":308,"cache_read_input_tokens":241,"cache_creation_input_tokens":17}`, `,"usage":{"output_tokens":49}`, "", true, 566, 308, 566, 49, 241, 17},
|
||||
{"deepseek_partial_cache", "", `,"usage":{"input_tokens":1200,"output_tokens":30,"prompt_cache_hit_tokens":800,"prompt_cache_miss_tokens":400}`, "", true, 1200, 400, 1200, 30, 800, 0},
|
||||
{"openai_partial_cache", "", `,"usage":{"prompt_tokens":1200,"output_tokens":30,"prompt_tokens_details":{"cached_tokens":800}}`, "", true, 1200, 400, 1200, 30, 800, 0},
|
||||
{"repeated_authoritative_cache_bucket", `,"usage":{"input_tokens":1200,"prompt_tokens":1200}`, `,"usage":{"output_tokens":30,"prompt_tokens":1200,"cache_read_input_tokens":800}`, `,"usage":{"output_tokens":30,"prompt_tokens":1200,"cache_read_input_tokens":800}`, true, 1200, 400, 1200, 30, 800, 0},
|
||||
{"independent_input_with_late_cache", `,"usage":{"input_tokens":308,"cache_read_input_tokens":241,"cache_creation_input_tokens":17}`, `,"usage":{"input_tokens":0,"output_tokens":49,"cache_read_input_tokens":300,"cache_creation_input_tokens":17}`, "", true, 625, 308, 625, 49, 300, 17},
|
||||
{"kimi_full_cache", `,"usage":{"input_tokens":173306,"prompt_tokens":173306}`, `,"usage":{"input_tokens":0,"output_tokens":49,"prompt_tokens":173306,"cache_read_input_tokens":173306}`, "", true, 173306, 0, 173306, 49, 173306, 0},
|
||||
{"kimi_full_cache_cached_tokens", `,"usage":{"input_tokens":173306,"prompt_tokens":173306}`, `,"usage":{"input_tokens":0,"output_tokens":49,"prompt_tokens":173306,"cached_tokens":173306}`, "", true, 173306, 0, 173306, 49, 173306, 0},
|
||||
{"kimi_full_cache_prompt_details", `,"usage":{"input_tokens":173306,"prompt_tokens":173306}`, `,"usage":{"input_tokens":0,"output_tokens":49,"prompt_tokens":173306,"prompt_tokens_details":{"cached_tokens":173306}}`, "", true, 173306, 0, 173306, 49, 173306, 0},
|
||||
{"zero_start", `,"usage":{"input_tokens":0,"output_tokens":0}`, "", "", true, 0, 0, 0, 0, 0, 0},
|
||||
{"zero_delta", "", `,"usage":{"input_tokens":0,"output_tokens":0}`, "", true, 0, 0, 0, 0, 0, 0},
|
||||
{"absent", "", "", "", false, 0, 0, 0, 0, 0, 0},
|
||||
{"null", `,"usage":null`, `,"usage":null`, "", false, 0, 0, 0, 0, 0, 0},
|
||||
} {
|
||||
for _, terminal := range []string{"stop", "eof"} {
|
||||
t.Run(adapter+"/"+options+"/"+tc.name+"/"+terminal, func(t *testing.T) {
|
||||
sse := fmt.Sprintf("event: message_start\ndata: {\"type\":\"message_start\",\"message\":{\"id\":\"msg_1\",\"type\":\"message\",\"role\":\"assistant\",\"model\":\"k3\",\"content\":[]%s}}\n\nevent: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"}%s}\n\n", tc.start, tc.delta)
|
||||
if tc.repeatDelta != "" {
|
||||
sse += fmt.Sprintf("event: message_delta\ndata: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"}%s}\n\n", tc.repeatDelta)
|
||||
}
|
||||
if terminal == "stop" {
|
||||
// Repeated terminal events and EOF must not duplicate usage.
|
||||
sse += "event: message_stop\ndata: {\"type\":\"message_stop\"}\n\nevent: message_stop\ndata: {\"type\":\"message_stop\"}\n\n"
|
||||
}
|
||||
upstream := &httpUpstreamRecorder{resp: &http.Response{
|
||||
StatusCode: http.StatusOK,
|
||||
Header: http.Header{"Content-Type": []string{"text/event-stream"}},
|
||||
Body: io.NopCloser(strings.NewReader(sse)),
|
||||
}}
|
||||
body := []byte(`{"model":"k3","messages":[{"role":"user","content":"hi"}],"stream":true` + options + "}")
|
||||
rec := httptest.NewRecorder()
|
||||
c, _ := gin.CreateTestContext(rec)
|
||||
c.Request = httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(string(body)))
|
||||
if adapter == "native" {
|
||||
svc := &OpenAIGatewayService{cfg: rawChatCompletionsTestConfig(), httpUpstream: upstream}
|
||||
result, err := svc.forwardChatCompletionsViaNativeAnthropic(context.Background(), c, nativeAnthropicTestAccount(), body, "")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, tc.totalInput, result.Usage.InputTokens)
|
||||
require.Equal(t, tc.output, result.Usage.OutputTokens)
|
||||
require.Equal(t, tc.cached, result.Usage.CacheReadInputTokens)
|
||||
require.Equal(t, tc.created, result.Usage.CacheCreationInputTokens)
|
||||
} else {
|
||||
svc := &GatewayService{cfg: rawChatCompletionsTestConfig(), httpUpstream: upstream}
|
||||
account := &Account{ID: 1, Platform: PlatformAnthropic, Type: AccountTypeAPIKey, Credentials: map[string]any{"api_key": "sk-test"}}
|
||||
result, err := svc.ForwardAsChatCompletions(context.Background(), c, account, body, nil)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, tc.billableInput, result.Usage.InputTokens)
|
||||
require.Equal(t, tc.output, result.Usage.OutputTokens)
|
||||
require.Equal(t, tc.cached, result.Usage.CacheReadInputTokens)
|
||||
require.Equal(t, tc.created, result.Usage.CacheCreationInputTokens)
|
||||
}
|
||||
count, done := 0, 0
|
||||
var last *apicompat.ChatCompletionsChunk
|
||||
for _, line := range strings.Split(rec.Body.String(), "\n") {
|
||||
if !strings.HasPrefix(line, "data: ") {
|
||||
continue
|
||||
}
|
||||
payload := strings.TrimPrefix(line, "data: ")
|
||||
if payload == "[DONE]" {
|
||||
done++
|
||||
if tc.want {
|
||||
require.NotNil(t, last)
|
||||
require.NotNil(t, last.Usage)
|
||||
}
|
||||
continue
|
||||
}
|
||||
require.Zero(t, done, "chunk after DONE")
|
||||
var chunk apicompat.ChatCompletionsChunk
|
||||
require.NoError(t, json.Unmarshal([]byte(payload), &chunk))
|
||||
last = &chunk
|
||||
if chunk.Usage == nil {
|
||||
continue
|
||||
}
|
||||
count++
|
||||
require.Empty(t, chunk.Choices)
|
||||
require.Equal(t, tc.prompt, chunk.Usage.PromptTokens)
|
||||
require.Equal(t, tc.output, chunk.Usage.CompletionTokens)
|
||||
require.Equal(t, tc.prompt+tc.output, chunk.Usage.TotalTokens)
|
||||
if tc.cached > 0 {
|
||||
require.NotNil(t, chunk.Usage.PromptTokensDetails)
|
||||
require.Equal(t, tc.cached, chunk.Usage.PromptTokensDetails.CachedTokens)
|
||||
require.Equal(t, tc.created, chunk.Usage.PromptTokensDetails.CacheCreationTokens)
|
||||
}
|
||||
}
|
||||
require.Equal(t, 1, done)
|
||||
if tc.want {
|
||||
require.Equal(t, 1, count)
|
||||
} else {
|
||||
require.Zero(t, count)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -42,7 +42,6 @@ func (s *GatewayService) ForwardAsChatCompletions(
|
||||
}
|
||||
originalModel := ccReq.Model
|
||||
clientStream := ccReq.Stream
|
||||
includeUsage := ccReq.StreamOptions != nil && ccReq.StreamOptions.IncludeUsage
|
||||
|
||||
// 2. Convert CC → Responses → Anthropic (chained conversion)
|
||||
responsesReq, err := apicompat.ChatCompletionsToResponses(&ccReq)
|
||||
@@ -188,7 +187,7 @@ func (s *GatewayService) ForwardAsChatCompletions(
|
||||
var result *ForwardResult
|
||||
var handleErr error
|
||||
if clientStream {
|
||||
result, handleErr = s.handleCCStreamingFromAnthropic(resp, c, originalModel, mappedModel, reasoningEffort, startTime, includeUsage)
|
||||
result, handleErr = s.handleCCStreamingFromAnthropic(resp, c, originalModel, mappedModel, reasoningEffort, startTime)
|
||||
} else {
|
||||
result, handleErr = s.handleCCBufferedFromAnthropic(resp, c, originalModel, mappedModel, reasoningEffort, startTime)
|
||||
}
|
||||
@@ -361,7 +360,6 @@ func (s *GatewayService) handleCCStreamingFromAnthropic(
|
||||
mappedModel string,
|
||||
reasoningEffort *string,
|
||||
startTime time.Time,
|
||||
includeUsage bool,
|
||||
) (*ForwardResult, error) {
|
||||
requestID := resp.Header.Get("x-request-id")
|
||||
|
||||
@@ -379,7 +377,6 @@ func (s *GatewayService) handleCCStreamingFromAnthropic(
|
||||
anthState.Model = originalModel
|
||||
ccState := apicompat.NewResponsesEventToChatState()
|
||||
ccState.Model = originalModel
|
||||
ccState.IncludeUsage = includeUsage
|
||||
|
||||
var usage ClaudeUsage
|
||||
var firstTokenMs *int
|
||||
@@ -445,6 +442,11 @@ func (s *GatewayService) handleCCStreamingFromAnthropic(
|
||||
mergeAnthropicUsage(&usage, event.Message.Usage)
|
||||
}
|
||||
|
||||
// Keep the outward Responses/Chat usage on the same normalized buckets used
|
||||
// for billing, including converter handlers that consume event usage.
|
||||
syncAnthropicResponsesUsage(anthState, usage)
|
||||
normalizeAnthropicEventUsageForResponses(event, usage)
|
||||
|
||||
// Chain: Anthropic event → Responses events → CC chunks
|
||||
responsesEvents := apicompat.AnthropicEventToResponsesEvents(event, anthState)
|
||||
for _, resEvt := range responsesEvents {
|
||||
@@ -479,6 +481,10 @@ func (s *GatewayService) handleCCStreamingFromAnthropic(
|
||||
continue
|
||||
}
|
||||
|
||||
// Forward received usage regardless of the client stream_options.
|
||||
// The intermediate Responses converter synthesizes usage even when absent.
|
||||
ccState.IncludeUsage = ccState.IncludeUsage || anthropicChatStreamHasUsage(&event, payload)
|
||||
|
||||
if processAnthropicEvent(&event) {
|
||||
return resultWithUsage(), nil
|
||||
}
|
||||
@@ -524,3 +530,16 @@ func writeGatewayCCError(c *gin.Context, statusCode int, errType, message string
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// anthropicChatStreamHasUsage distinguishes an explicit zero-valued usage object
|
||||
// from omitted/null usage before the Anthropic→Responses conversion loses that distinction.
|
||||
func anthropicChatStreamHasUsage(event *apicompat.AnthropicStreamEvent, payload string) bool {
|
||||
switch event.Type {
|
||||
case "message_start":
|
||||
return event.Message != nil && gjson.Get(payload, "message.usage").IsObject()
|
||||
case "message_delta":
|
||||
return event.Usage != nil
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
@@ -199,7 +199,7 @@ func TestHandleCCStreamingFromAnthropic_CompactSSEFormat(t *testing.T) {
|
||||
}
|
||||
|
||||
svc := &GatewayService{}
|
||||
result, err := svc.handleCCStreamingFromAnthropic(resp, c, "k3", "k3", nil, time.Now(), true)
|
||||
result, err := svc.handleCCStreamingFromAnthropic(resp, c, "k3", "k3", nil, time.Now())
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, result)
|
||||
require.Equal(t, 21, result.Usage.InputTokens)
|
||||
@@ -236,7 +236,7 @@ func TestHandleCCStreamingFromAnthropic_PreservesMessageStartCacheUsageAndReason
|
||||
}
|
||||
|
||||
svc := &GatewayService{}
|
||||
result, err := svc.handleCCStreamingFromAnthropic(resp, c, "gpt-5", "claude-sonnet-4.5", &reasoningEffort, time.Now(), true)
|
||||
result, err := svc.handleCCStreamingFromAnthropic(resp, c, "gpt-5", "claude-sonnet-4.5", &reasoningEffort, time.Now())
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, result)
|
||||
require.Equal(t, 20, result.Usage.InputTokens)
|
||||
|
||||
@@ -286,22 +286,22 @@ func mergeAnthropicUsage(dst *ClaudeUsage, src apicompat.AnthropicUsage) {
|
||||
return
|
||||
}
|
||||
|
||||
cacheReadTokens := src.CacheReadInputTokens
|
||||
if cacheReadTokens == 0 && src.CachedTokens > 0 {
|
||||
cacheReadTokens = src.CachedTokens
|
||||
}
|
||||
if cacheReadTokens == 0 && src.PromptTokensDetails != nil && src.PromptTokensDetails.CachedTokens > 0 {
|
||||
cacheReadTokens = src.PromptTokensDetails.CachedTokens
|
||||
}
|
||||
if cacheReadTokens == 0 && src.PromptCacheHitTokens != nil {
|
||||
cacheReadTokens = max(*src.PromptCacheHitTokens, 0)
|
||||
}
|
||||
|
||||
// Some Anthropic-compatible providers retain OpenAI-style prompt/cache
|
||||
// fields. Prefer those authoritative totals or hit/miss buckets over the
|
||||
// overloaded input_tokens field. This covers Kimi's changing stream
|
||||
// semantics as well as GLM/DeepSeek cache aliases.
|
||||
if src.PromptTokens > 0 || src.PromptCacheHitTokens != nil || src.PromptCacheMissTokens != nil {
|
||||
cacheReadTokens := src.CacheReadInputTokens
|
||||
if cacheReadTokens == 0 && src.CachedTokens > 0 {
|
||||
cacheReadTokens = src.CachedTokens
|
||||
}
|
||||
if cacheReadTokens == 0 && src.PromptTokensDetails != nil && src.PromptTokensDetails.CachedTokens > 0 {
|
||||
cacheReadTokens = src.PromptTokensDetails.CachedTokens
|
||||
}
|
||||
if cacheReadTokens == 0 && src.PromptCacheHitTokens != nil {
|
||||
cacheReadTokens = max(*src.PromptCacheHitTokens, 0)
|
||||
}
|
||||
|
||||
if src.PromptCacheMissTokens != nil {
|
||||
dst.InputTokens = max(*src.PromptCacheMissTokens, 0)
|
||||
} else {
|
||||
@@ -310,13 +310,16 @@ func mergeAnthropicUsage(dst *ClaudeUsage, src apicompat.AnthropicUsage) {
|
||||
dst.CacheReadInputTokens = cacheReadTokens
|
||||
dst.CacheCreationInputTokens = src.CacheCreationInputTokens
|
||||
} else {
|
||||
// Without an authoritative prompt total or miss bucket, input_tokens is
|
||||
// provider-specific: it may already be the uncached bucket, or it may be
|
||||
// a total from an earlier event. Do not infer a subtraction merely because
|
||||
// a later event contains cache buckets; that would corrupt providers whose
|
||||
// stream uses independent input and cache fields.
|
||||
if src.InputTokens > 0 {
|
||||
dst.InputTokens = src.InputTokens
|
||||
}
|
||||
if src.CacheReadInputTokens > 0 {
|
||||
dst.CacheReadInputTokens = src.CacheReadInputTokens
|
||||
} else if src.CachedTokens > 0 {
|
||||
dst.CacheReadInputTokens = src.CachedTokens
|
||||
if cacheReadTokens > 0 {
|
||||
dst.CacheReadInputTokens = cacheReadTokens
|
||||
}
|
||||
if src.CacheCreationInputTokens > 0 {
|
||||
dst.CacheCreationInputTokens = src.CacheCreationInputTokens
|
||||
@@ -327,6 +330,29 @@ func mergeAnthropicUsage(dst *ClaudeUsage, src apicompat.AnthropicUsage) {
|
||||
}
|
||||
}
|
||||
|
||||
func syncAnthropicResponsesUsage(state *apicompat.AnthropicEventToResponsesState, usage ClaudeUsage) {
|
||||
state.InputTokens = usage.InputTokens
|
||||
state.OutputTokens = usage.OutputTokens
|
||||
state.CacheReadInputTokens = usage.CacheReadInputTokens
|
||||
state.CacheCreationInputTokens = usage.CacheCreationInputTokens
|
||||
}
|
||||
|
||||
func normalizeAnthropicEventUsageForResponses(event *apicompat.AnthropicStreamEvent, usage ClaudeUsage) {
|
||||
normalize := func(dst *apicompat.AnthropicUsage) {
|
||||
if dst == nil {
|
||||
return
|
||||
}
|
||||
dst.InputTokens = usage.InputTokens
|
||||
dst.OutputTokens = usage.OutputTokens
|
||||
dst.CacheReadInputTokens = usage.CacheReadInputTokens
|
||||
dst.CacheCreationInputTokens = usage.CacheCreationInputTokens
|
||||
}
|
||||
normalize(event.Usage)
|
||||
if event.Message != nil {
|
||||
normalize(&event.Message.Usage)
|
||||
}
|
||||
}
|
||||
|
||||
// parseAnthropicSSEField parses an SSE field line in the form "field:value" or "field: value".
|
||||
// According to the SSE spec (https://html.spec.whatwg.org/multipage/server-sent-events.html#event-stream-interpretation),
|
||||
// the space after the colon is optional. This function handles both formats.
|
||||
@@ -558,6 +584,12 @@ func (s *GatewayService) handleResponsesStreamingResponse(
|
||||
mergeAnthropicUsage(&usage, event.Message.Usage)
|
||||
}
|
||||
|
||||
// Keep the terminal Responses usage aligned with the normalized billing
|
||||
// buckets. Normalize the converter input too, so message handlers cannot
|
||||
// restore the provider's overlapping raw input total.
|
||||
syncAnthropicResponsesUsage(state, usage)
|
||||
normalizeAnthropicEventUsageForResponses(event, usage)
|
||||
|
||||
// Convert to Responses events
|
||||
events := apicompat.AnthropicEventToResponsesEvents(event, state)
|
||||
for _, evt := range events {
|
||||
|
||||
@@ -310,6 +310,116 @@ func TestHandleResponsesStreamingResponse_PreservesMessageStartCacheUsage(t *tes
|
||||
require.Contains(t, rec.Body.String(), `response.completed`)
|
||||
}
|
||||
|
||||
func TestHandleResponsesStreamingResponse_NormalizesTerminalUsage(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
startUsage string
|
||||
deltaUsage string
|
||||
wantInput int
|
||||
wantOutput int
|
||||
wantCached int
|
||||
wantCacheCreation int
|
||||
}{
|
||||
{
|
||||
name: "cache_read_input_tokens full cache",
|
||||
startUsage: `"input_tokens":173306,"prompt_tokens":173306`,
|
||||
deltaUsage: `"input_tokens":0,"output_tokens":8,"prompt_tokens":173306,"cache_read_input_tokens":173306`,
|
||||
wantOutput: 8,
|
||||
wantCached: 173306,
|
||||
},
|
||||
{
|
||||
name: "cached_tokens full cache",
|
||||
startUsage: `"input_tokens":173306,"prompt_tokens":173306`,
|
||||
deltaUsage: `"input_tokens":0,"output_tokens":8,"prompt_tokens":173306,"cached_tokens":173306`,
|
||||
wantOutput: 8,
|
||||
wantCached: 173306,
|
||||
},
|
||||
{
|
||||
name: "prompt_tokens_details full cache",
|
||||
startUsage: `"input_tokens":173306,"prompt_tokens":173306`,
|
||||
deltaUsage: `"input_tokens":0,"output_tokens":8,"prompt_tokens":173306,"prompt_tokens_details":{"cached_tokens":173306}`,
|
||||
wantOutput: 8,
|
||||
wantCached: 173306,
|
||||
},
|
||||
{
|
||||
name: "DeepSeek input total with hit and miss buckets",
|
||||
startUsage: `"input_tokens":0`,
|
||||
deltaUsage: `"input_tokens":1200,"output_tokens":30,"prompt_cache_hit_tokens":800,"prompt_cache_miss_tokens":400`,
|
||||
wantInput: 400,
|
||||
wantOutput: 30,
|
||||
wantCached: 800,
|
||||
},
|
||||
{
|
||||
name: "OpenAI prompt total with cached details",
|
||||
startUsage: `"input_tokens":0`,
|
||||
deltaUsage: `"prompt_tokens":1200,"output_tokens":30,"prompt_tokens_details":{"cached_tokens":800}`,
|
||||
wantInput: 400,
|
||||
wantOutput: 30,
|
||||
wantCached: 800,
|
||||
},
|
||||
{
|
||||
name: "cache creation only without prompt total or miss bucket keeps independent input",
|
||||
startUsage: `"input_tokens":1200`,
|
||||
deltaUsage: `"input_tokens":0,"output_tokens":30,"cache_creation_input_tokens":800`,
|
||||
wantInput: 1200,
|
||||
wantOutput: 30,
|
||||
wantCacheCreation: 800,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
for _, terminal := range []string{"message_stop", "eof"} {
|
||||
t.Run(tt.name+"/"+terminal, func(t *testing.T) {
|
||||
rec := httptest.NewRecorder()
|
||||
c, _ := gin.CreateTestContext(rec)
|
||||
lines := []string{
|
||||
`event: message_start`,
|
||||
`data: {"type":"message_start","message":{"id":"msg_usage","type":"message","role":"assistant","content":[],"model":"k3","stop_reason":"","usage":{` + tt.startUsage + `}}}`,
|
||||
``,
|
||||
`event: message_delta`,
|
||||
`data: {"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{` + tt.deltaUsage + `}}`,
|
||||
``,
|
||||
}
|
||||
if terminal == "message_stop" {
|
||||
lines = append(lines, `event: message_stop`, `data: {"type":"message_stop"}`, ``)
|
||||
}
|
||||
resp := &http.Response{Body: io.NopCloser(strings.NewReader(strings.Join(lines, "\n")))}
|
||||
|
||||
result, err := (&GatewayService{}).handleResponsesStreamingResponse(resp, c, "k3", "k3", nil, time.Now(), apicompat.ResponsesClientToolMapping{})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, tt.wantInput, result.Usage.InputTokens)
|
||||
require.Equal(t, tt.wantCached, result.Usage.CacheReadInputTokens)
|
||||
require.Equal(t, tt.wantCacheCreation, result.Usage.CacheCreationInputTokens)
|
||||
|
||||
var completed apicompat.ResponsesStreamEvent
|
||||
for _, line := range strings.Split(rec.Body.String(), "\n") {
|
||||
if !strings.HasPrefix(line, "data: ") {
|
||||
continue
|
||||
}
|
||||
var event apicompat.ResponsesStreamEvent
|
||||
require.NoError(t, json.Unmarshal([]byte(strings.TrimPrefix(line, "data: ")), &event))
|
||||
if event.Type == "response.completed" {
|
||||
completed = event
|
||||
}
|
||||
}
|
||||
require.NotNil(t, completed.Response)
|
||||
require.NotNil(t, completed.Response.Usage)
|
||||
require.Equal(t, tt.wantInput+tt.wantCached+tt.wantCacheCreation, completed.Response.Usage.InputTokens)
|
||||
require.Equal(t, tt.wantOutput, completed.Response.Usage.OutputTokens)
|
||||
require.Equal(t, completed.Response.Usage.InputTokens+tt.wantOutput, completed.Response.Usage.TotalTokens)
|
||||
require.Equal(t, tt.wantCacheCreation, completed.Response.Usage.CacheCreationInputTokens)
|
||||
if tt.wantCached == 0 {
|
||||
require.Nil(t, completed.Response.Usage.InputTokensDetails)
|
||||
} else {
|
||||
require.Equal(t, tt.wantCached, completed.Response.Usage.InputTokensDetails.CachedTokens)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseAnthropicSSEField(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
|
||||
@@ -140,7 +140,7 @@ func TestCCStreamingFromNativeAnthropic_HangTimesOut(t *testing.T) {
|
||||
|
||||
resp, pr, pw := newHangingUpstreamResponse()
|
||||
start := time.Now()
|
||||
res, err := svc.handleCCStreamingFromNativeAnthropic(resp, c, "glm-4.7", "glm-4.7", "glm-4.7", nil, start, true)
|
||||
res, err := svc.handleCCStreamingFromNativeAnthropic(resp, c, "glm-4.7", "glm-4.7", "glm-4.7", nil, start)
|
||||
_ = pw.Close()
|
||||
_ = pr.Close()
|
||||
|
||||
@@ -220,7 +220,7 @@ func TestCCStreamingFromNativeAnthropic_HappyPathStillConverts(t *testing.T) {
|
||||
}()
|
||||
defer func() { _ = pr.Close() }()
|
||||
|
||||
res, err := svc.handleCCStreamingFromNativeAnthropic(resp, c, "glm-4.7", "glm-4.7", "glm-4.7", nil, time.Now(), true)
|
||||
res, err := svc.handleCCStreamingFromNativeAnthropic(resp, c, "glm-4.7", "glm-4.7", "glm-4.7", nil, time.Now())
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
@@ -55,7 +55,6 @@ func (s *OpenAIGatewayService) forwardChatCompletionsViaNativeAnthropic(
|
||||
return nil, fmt.Errorf("missing model in request")
|
||||
}
|
||||
clientStream := ccReq.Stream
|
||||
includeUsage := ccReq.StreamOptions != nil && ccReq.StreamOptions.IncludeUsage
|
||||
|
||||
// 2. Convert CC → Responses → Anthropic (chained conversion)
|
||||
responsesReq, err := apicompat.ChatCompletionsToResponses(&ccReq)
|
||||
@@ -141,7 +140,7 @@ func (s *OpenAIGatewayService) forwardChatCompletionsViaNativeAnthropic(
|
||||
reasoningEffort = ApplyThinkingEnabledFallback(reasoningEffort, forwardedBody, upstreamModel)
|
||||
|
||||
if clientStream {
|
||||
return s.handleCCStreamingFromNativeAnthropic(resp, c, originalModel, billingModel, upstreamModel, reasoningEffort, startTime, includeUsage)
|
||||
return s.handleCCStreamingFromNativeAnthropic(resp, c, originalModel, billingModel, upstreamModel, reasoningEffort, startTime)
|
||||
}
|
||||
return s.handleCCBufferedFromNativeAnthropic(resp, c, originalModel, billingModel, upstreamModel, reasoningEffort, startTime)
|
||||
}
|
||||
@@ -308,7 +307,6 @@ func (s *OpenAIGatewayService) handleCCStreamingFromNativeAnthropic(
|
||||
upstreamModel string,
|
||||
reasoningEffort *string,
|
||||
startTime time.Time,
|
||||
includeUsage bool,
|
||||
) (*OpenAIForwardResult, error) {
|
||||
requestID := resp.Header.Get("x-request-id")
|
||||
|
||||
@@ -325,7 +323,6 @@ func (s *OpenAIGatewayService) handleCCStreamingFromNativeAnthropic(
|
||||
anthState.Model = originalModel
|
||||
ccState := apicompat.NewResponsesEventToChatState()
|
||||
ccState.Model = originalModel
|
||||
ccState.IncludeUsage = includeUsage
|
||||
|
||||
var usage ClaudeUsage
|
||||
var firstTokenMs *int
|
||||
@@ -415,6 +412,11 @@ func (s *OpenAIGatewayService) handleCCStreamingFromNativeAnthropic(
|
||||
mergeAnthropicUsage(&usage, event.Message.Usage)
|
||||
}
|
||||
|
||||
// Keep the outward Responses/Chat usage on the same normalized buckets used
|
||||
// for billing, including converter handlers that consume event usage.
|
||||
syncAnthropicResponsesUsage(anthState, usage)
|
||||
normalizeAnthropicEventUsageForResponses(event, usage)
|
||||
|
||||
// 客户端已断开:跳过转换与写出,继续读上游直到流结束(usage 完整、
|
||||
// 连接及时归还),不再提前 return。
|
||||
if clientDisconnected {
|
||||
@@ -466,6 +468,10 @@ func (s *OpenAIGatewayService) handleCCStreamingFromNativeAnthropic(
|
||||
continue
|
||||
}
|
||||
|
||||
// Forward received usage regardless of the client stream_options.
|
||||
// The intermediate Responses converter synthesizes usage even when absent.
|
||||
ccState.IncludeUsage = ccState.IncludeUsage || anthropicChatStreamHasUsage(&event, payload)
|
||||
|
||||
if processAnthropicEvent(&event) {
|
||||
return resultWithUsage(), nil
|
||||
}
|
||||
|
||||
@@ -11,9 +11,12 @@ package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -109,6 +112,83 @@ func TestResponsesStreamingFromNativeAnthropic_ClientDisconnectDrainsUsage(t *te
|
||||
"output_tokens 必须来自排水读到的末尾 message_delta(断开即弃时会是 1)")
|
||||
}
|
||||
|
||||
func TestResponsesStreamingFromNativeAnthropic_NormalizesTerminalUsage(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
startUsage string
|
||||
deltaUsage string
|
||||
repeatDelta bool
|
||||
wantInput int
|
||||
wantOutput int
|
||||
wantCached int
|
||||
wantCacheCreation int
|
||||
}{
|
||||
{name: "full cache", startUsage: `"input_tokens":1200,"prompt_tokens":1200`, deltaUsage: `"input_tokens":0,"output_tokens":30,"prompt_tokens":1200,"cache_read_input_tokens":1200`, wantOutput: 30, wantCached: 1200},
|
||||
{name: "partial cache", startUsage: `"input_tokens":1200,"prompt_tokens":1200`, deltaUsage: `"input_tokens":0,"output_tokens":30,"prompt_tokens":1200,"cache_read_input_tokens":800`, wantInput: 400, wantOutput: 30, wantCached: 800},
|
||||
{name: "cache creation", startUsage: `"input_tokens":1200,"prompt_tokens":1200`, deltaUsage: `"input_tokens":0,"output_tokens":30,"prompt_tokens":1200,"cache_creation_input_tokens":800`, wantInput: 400, wantOutput: 30, wantCacheCreation: 800},
|
||||
{name: "repeated cumulative cache bucket", startUsage: `"input_tokens":1200,"prompt_tokens":1200`, deltaUsage: `"input_tokens":0,"output_tokens":30,"prompt_tokens":1200,"cache_read_input_tokens":800`, repeatDelta: true, wantInput: 400, wantOutput: 30, wantCached: 800},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
for _, terminal := range []string{"message_stop", "eof"} {
|
||||
t.Run(tt.name+"/"+terminal, func(t *testing.T) {
|
||||
lines := []string{
|
||||
`event: message_start`,
|
||||
`data: {"type":"message_start","message":{"id":"msg_usage","type":"message","role":"assistant","content":[],"model":"k3","stop_reason":"","usage":{` + tt.startUsage + `}}}`,
|
||||
``,
|
||||
`event: message_delta`,
|
||||
`data: {"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{` + tt.deltaUsage + `}}`,
|
||||
``,
|
||||
}
|
||||
if tt.repeatDelta {
|
||||
lines = append(lines, `event: message_delta`, `data: {"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{`+tt.deltaUsage+`}}`, ``)
|
||||
}
|
||||
if terminal == "message_stop" {
|
||||
lines = append(lines, `event: message_stop`, `data: {"type":"message_stop"}`, ``)
|
||||
}
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
c, _ := gin.CreateTestContext(rec)
|
||||
c.Request = httptest.NewRequest(http.MethodPost, "/", nil)
|
||||
resp := &http.Response{StatusCode: http.StatusOK, Header: http.Header{}, Body: io.NopCloser(strings.NewReader(strings.Join(lines, "\n")))}
|
||||
|
||||
result, err := (&OpenAIGatewayService{}).handleResponsesStreamingFromNativeAnthropic(
|
||||
resp, c, "k3", "k3", "k3", nil, time.Now(), apicompat.ResponsesClientToolMapping{})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, tt.wantInput+tt.wantCached+tt.wantCacheCreation, result.Usage.InputTokens)
|
||||
require.Equal(t, tt.wantOutput, result.Usage.OutputTokens)
|
||||
require.Equal(t, tt.wantCached, result.Usage.CacheReadInputTokens)
|
||||
require.Equal(t, tt.wantCacheCreation, result.Usage.CacheCreationInputTokens)
|
||||
|
||||
var completed apicompat.ResponsesStreamEvent
|
||||
for _, line := range strings.Split(rec.Body.String(), "\n") {
|
||||
if !strings.HasPrefix(line, "data: ") {
|
||||
continue
|
||||
}
|
||||
var event apicompat.ResponsesStreamEvent
|
||||
require.NoError(t, json.Unmarshal([]byte(strings.TrimPrefix(line, "data: ")), &event))
|
||||
if event.Type == "response.completed" {
|
||||
completed = event
|
||||
}
|
||||
}
|
||||
require.NotNil(t, completed.Response)
|
||||
require.NotNil(t, completed.Response.Usage)
|
||||
require.Equal(t, tt.wantInput+tt.wantCached+tt.wantCacheCreation, completed.Response.Usage.InputTokens)
|
||||
require.Equal(t, tt.wantOutput, completed.Response.Usage.OutputTokens)
|
||||
require.Equal(t, completed.Response.Usage.InputTokens+tt.wantOutput, completed.Response.Usage.TotalTokens)
|
||||
require.Equal(t, tt.wantCacheCreation, completed.Response.Usage.CacheCreationInputTokens)
|
||||
if tt.wantCached == 0 {
|
||||
require.Nil(t, completed.Response.Usage.InputTokensDetails)
|
||||
} else {
|
||||
require.Equal(t, tt.wantCached, completed.Response.Usage.InputTokensDetails.CachedTokens)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandle403_CNProviderHTMLBodySkipsAccountPenalty(t *testing.T) {
|
||||
for _, platform := range []string{PlatformKimi, PlatformZhipu, PlatformDeepseek, PlatformMiniMax} {
|
||||
repo := &rateLimitAccountRepoStub{}
|
||||
|
||||
@@ -406,6 +406,12 @@ func (s *OpenAIGatewayService) handleResponsesStreamingFromNativeAnthropic(
|
||||
mergeAnthropicUsage(&usage, event.Message.Usage)
|
||||
}
|
||||
|
||||
// Keep terminal Responses usage aligned with the normalized billing
|
||||
// buckets. Normalize converter input too so raw overlapping totals cannot
|
||||
// overwrite the state when message_start/message_delta handlers run.
|
||||
syncAnthropicResponsesUsage(state, usage)
|
||||
normalizeAnthropicEventUsageForResponses(event, usage)
|
||||
|
||||
events := apicompat.AnthropicEventToResponsesEvents(event, state)
|
||||
if clientDisconnected {
|
||||
return
|
||||
|
||||
Reference in New Issue
Block a user