diff --git a/backend/internal/service/openai_gateway_passthrough.go b/backend/internal/service/openai_gateway_passthrough.go index 0a19a9d283..cc0a69503b 100644 --- a/backend/internal/service/openai_gateway_passthrough.go +++ b/backend/internal/service/openai_gateway_passthrough.go @@ -13,6 +13,7 @@ import ( "fmt" "net/http" "sort" + "strconv" "strings" "time" @@ -426,6 +427,73 @@ func shouldFailoverOpenAIPassthroughResponse(statusCode int) bool { } } +func writeOpenAIPassthroughErrorHeaders(dst, src http.Header) { + if dst == nil { + return + } + dst.Set("Content-Type", "application/json; charset=utf-8") + dst.Set("Cache-Control", "no-store") + dst.Del("Retry-After") + if src == nil { + return + } + rawRetryAfter := strings.TrimSpace(src.Get("Retry-After")) + if validOpenAIPassthroughRetryAfter(rawRetryAfter, time.Now()) { + dst.Set("Retry-After", rawRetryAfter) + } +} + +func validOpenAIPassthroughRetryAfter(raw string, now time.Time) bool { + raw = strings.TrimSpace(raw) + if raw == "" { + return false + } + delaySeconds := true + for i := 0; i < len(raw); i++ { + if raw[i] < '0' || raw[i] > '9' { + delaySeconds = false + break + } + } + if delaySeconds { + seconds, err := strconv.ParseUint(raw, 10, 64) + return err == nil && seconds > 0 + } + parsed, err := http.ParseTime(raw) + return err == nil && parsed.After(now) +} + +func writeSanitizedOpenAIPassthroughError(c *gin.Context, upstreamStatus int, upstreamHeaders http.Header) { + if c == nil { + return + } + downstreamStatus := upstreamStatus + message := "Upstream request failed" + switch upstreamStatus { + case http.StatusUnauthorized: + downstreamStatus = http.StatusBadGateway + message = "Upstream authentication failed" + case http.StatusForbidden: + downstreamStatus = http.StatusBadGateway + message = "Upstream access denied" + default: + if upstreamStatus >= http.StatusInternalServerError { + message = "Upstream service temporarily unavailable" + } + } + body, _ := json.Marshal(gin.H{ + "error": gin.H{ + "type": "upstream_error", + "message": message, + }, + }) + if writeOpenAICompactSSEBridge(c, downstreamStatus, body) { + return + } + writeOpenAIPassthroughErrorHeaders(c.Writer.Header(), upstreamHeaders) + c.Data(downstreamStatus, "application/json; charset=utf-8", body) +} + func (s *OpenAIGatewayService) handleFailoverErrorResponsePassthrough( ctx context.Context, resp *http.Response, @@ -478,8 +546,8 @@ func (s *OpenAIGatewayService) handleErrorResponsePassthrough( MarkResponseCommitted(c) body := s.readUpstreamErrorBody(resp) - // cyber_policy:透传账号本就把原始 body 回给客户端(下方 c.Data),此处仅打标记, - // 供 handler 事后写风控/邮件。cyber 是上游网络安全策略拦截,不冷却账号, + // cyber_policy 仍按原始 body 打内部标记,供 handler 事后写风控/邮件;面向客户端的 + // 错误体在下方统一重建。cyber 是上游网络安全策略拦截,不冷却账号, // 故下方跳过 handleOpenAIAccountUpstreamError(避免自定义 temp-unschedulable 规则误冷却)。 cyberHit, cyberCode, cyberMsg := detectOpenAICyberPolicy(body) if cyberHit { @@ -503,8 +571,8 @@ func (s *OpenAIGatewayService) handleErrorResponsePassthrough( } setOpsUpstreamError(c, resp.StatusCode, upstreamMsg, upstreamDetail) logOpenAIInstructionsRequiredDebug(ctx, c, account, resp.StatusCode, upstreamMsg, requestBody, body) - // 透传模式保留原始上游错误响应,但运行态账号状态仍需更新, - // 避免粘性路由继续复用刚被限流的账号。cyber 例外:不冷却账号。 + // 错误体虽不会原样透传,运行态账号状态仍需更新,避免粘性路由继续复用 + // 刚被限流的账号。cyber 例外:不冷却账号。 if !cyberHit { reqModel, _, _ := extractOpenAIRequestMetaFromBody(requestBody) _ = s.handleOpenAIAccountUpstreamError(ctx, account, resp.StatusCode, resp.Header, body, reqModel) @@ -521,18 +589,9 @@ func (s *OpenAIGatewayService) handleErrorResponsePassthrough( Detail: upstreamDetail, UpstreamResponseBody: upstreamDetail, }) + writeSanitizedOpenAIPassthroughError(c, resp.StatusCode, resp.Header) - writeOpenAIPassthroughResponseHeaders(c.Writer.Header(), resp.Header, s.responseHeaderFilter) - contentType := resp.Header.Get("Content-Type") - if contentType == "" { - contentType = "application/json" - } - c.Data(resp.StatusCode, contentType, body) - - if upstreamMsg == "" { - return fmt.Errorf("upstream error: %d", resp.StatusCode) - } - return fmt.Errorf("upstream error: %d message=%s", resp.StatusCode, upstreamMsg) + return fmt.Errorf("upstream error: %d (client response sanitized)", resp.StatusCode) } func isOpenAIPassthroughAllowedRequestHeader(lowerKey string, allowTimeoutHeaders bool) bool { diff --git a/backend/internal/service/openai_oauth_passthrough_test.go b/backend/internal/service/openai_oauth_passthrough_test.go index 0aa536f94f..b1d1d953b5 100644 --- a/backend/internal/service/openai_oauth_passthrough_test.go +++ b/backend/internal/service/openai_oauth_passthrough_test.go @@ -995,7 +995,7 @@ func TestOpenAIGatewayService_OAuthPassthrough_UpstreamErrorIncludesPassthroughF _, err := svc.Forward(context.Background(), c, account, originalBody) require.Error(t, err) - require.True(t, c.Writer.Written(), "非 429/529 的 passthrough 错误应继续原样写回客户端") + require.True(t, c.Writer.Written(), "非 429/529 的 passthrough 错误应直接写回客户端") require.Equal(t, http.StatusBadRequest, rec.Code) // should append an upstream error event with passthrough=true @@ -1008,6 +1008,252 @@ func TestOpenAIGatewayService_OAuthPassthrough_UpstreamErrorIncludesPassthroughF require.Equal(t, "http_error", arr[len(arr)-1].Kind) } +func TestOpenAIGatewayService_APIKeyPassthrough_RebuildsUpstreamErrors(t *testing.T) { + gin.SetMode(gin.TestMode) + + tests := []struct { + name string + statusCode int + contentType string + responseBody string + retryAfter string + wantStatus int + wantMessage string + wantRetryAfter string + }{ + { + name: "upstream forbidden is reported as gateway failure", + statusCode: http.StatusForbidden, + contentType: "text/html; charset=UTF-8", + responseBody: `secret-upstream.example denied the request`, + retryAfter: "17", + wantStatus: http.StatusBadGateway, + wantMessage: "Upstream access denied", + wantRetryAfter: "17", + }, + { + name: "upstream unauthorized is reported as gateway failure", + statusCode: http.StatusUnauthorized, + contentType: "application/json", + responseBody: `{"error":{"message":"invalid secret-upstream.example token","type":"authentication_error","code":"invalid_api_key","param":"api_key"},"rate_limit":{"remaining":0}}`, + wantStatus: http.StatusBadGateway, + wantMessage: "Upstream authentication failed", + }, + { + name: "html 5xx", + statusCode: http.StatusBadGateway, + contentType: "text/html; charset=UTF-8", + responseBody: `secret-upstream.example | 502: Bad gateway`, + wantStatus: http.StatusBadGateway, + wantMessage: "Upstream service temporarily unavailable", + }, + { + name: "structured 5xx", + statusCode: http.StatusInternalServerError, + contentType: "application/json", + responseBody: `{"error":{"message":"secret-upstream.example internal failure"}}`, + wantStatus: http.StatusInternalServerError, + wantMessage: "Upstream service temporarily unavailable", + }, + { + name: "unstructured 4xx", + statusCode: http.StatusBadRequest, + contentType: "text/plain", + responseBody: `proxy secret-upstream.example rejected the request`, + wantStatus: http.StatusBadRequest, + wantMessage: "Upstream request failed", + }, + { + name: "malicious valid json 4xx", + statusCode: http.StatusBadRequest, + contentType: "application/json", + responseBody: `{"error":{"message":"secret-upstream.example invalid parameter","type":"invalid_request_error","code":"upstream_secret_code","param":"private_field","internal_token":"sk-upstream-secret"},"rate_limit":{"remaining":0,"reset":"internal-window"},"debug":{"admin":"root"},"redirect":"https://secret-upstream.example/admin"}`, + retryAfter: "not-a-valid-delay", + wantStatus: http.StatusBadRequest, + wantMessage: "Upstream request failed", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + rec := httptest.NewRecorder() + c, _ := gin.CreateTestContext(rec) + c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", bytes.NewReader(nil)) + c.Request.Header.Set("User-Agent", "codex_cli_rs/0.1.0") + + upstream := &httpUpstreamRecorder{resp: &http.Response{ + StatusCode: tt.statusCode, + Header: http.Header{ + "Content-Type": []string{tt.contentType}, + "Location": []string{"https://secret-upstream.example/admin"}, + "Retry-After": []string{tt.retryAfter}, + "Server": []string{"secret-upstream-proxy"}, + "Set-Cookie": []string{"admin_token=secret"}, + "WWW-Authenticate": []string{`Bearer realm="secret-upstream.example"`}, + "X-Admin-Debug": []string{"internal-route=secret-upstream.example"}, + "X-Codex-Primary-Used-Percent": []string{"99"}, + "x-request-id": []string{"rid-sensitive-upstream"}, + }, + Body: io.NopCloser(strings.NewReader(tt.responseBody)), + }} + svc := &OpenAIGatewayService{ + cfg: &config.Config{Gateway: config.GatewayConfig{ForceCodexCLI: false}}, + httpUpstream: upstream, + } + account := &Account{ + ID: 124, + Name: "sensitive-upstream", + Platform: PlatformOpenAI, + Type: AccountTypeAPIKey, + Concurrency: 1, + Credentials: map[string]any{ + "api_key": "sk-test", + "base_url": "https://secret-upstream.example", + }, + Extra: map[string]any{"openai_passthrough": true}, + Status: StatusActive, + Schedulable: true, + } + requestBody := []byte(`{"model":"gpt-5.2","stream":false,"input":"hello"}`) + + _, err := svc.Forward(context.Background(), c, account, requestBody) + + require.Error(t, err) + require.Equal(t, tt.wantStatus, rec.Code) + opsValue, ok := c.Get(OpsUpstreamErrorsKey) + require.True(t, ok) + opsEvents, ok := opsValue.([]*OpsUpstreamErrorEvent) + require.True(t, ok) + require.NotEmpty(t, opsEvents) + require.Equal(t, tt.statusCode, opsEvents[len(opsEvents)-1].UpstreamStatusCode) + require.Contains(t, rec.Header().Get("Content-Type"), "application/json") + require.Equal(t, tt.wantRetryAfter, rec.Header().Get("Retry-After")) + for _, key := range []string{ + "Location", + "Server", + "Set-Cookie", + "WWW-Authenticate", + "X-Admin-Debug", + "X-Codex-Primary-Used-Percent", + "X-Request-Id", + } { + require.Empty(t, rec.Header().Values(key), "sensitive upstream header %s must be dropped", key) + } + require.Equal(t, "upstream_error", gjson.Get(rec.Body.String(), "error.type").String()) + require.Equal(t, tt.wantMessage, gjson.Get(rec.Body.String(), "error.message").String()) + require.False(t, gjson.Get(rec.Body.String(), "error.code").Exists()) + require.False(t, gjson.Get(rec.Body.String(), "error.param").Exists()) + require.False(t, gjson.Get(rec.Body.String(), "rate_limit").Exists()) + require.NotContains(t, rec.Body.String(), "secret-upstream.example") + require.NotContains(t, rec.Body.String(), "sk-upstream-secret") + require.NotContains(t, err.Error(), "secret-upstream.example") + }) + } +} + +func TestWriteOpenAIPassthroughErrorHeaders_StrictRetryAfter(t *testing.T) { + now := time.Now().UTC() + tests := []struct { + name string + raw string + want bool + }{ + {name: "positive delay seconds", raw: "17", want: true}, + {name: "fractional delay", raw: "1.5"}, + {name: "scientific notation", raw: "1e3"}, + {name: "explicit plus sign", raw: "+17"}, + {name: "zero", raw: "0"}, + {name: "negative delay", raw: "-1"}, + {name: "uint64 overflow", raw: "18446744073709551616"}, + {name: "future http date", raw: now.Add(time.Hour).Format(http.TimeFormat), want: true}, + {name: "past http date", raw: now.Add(-time.Hour).Format(http.TimeFormat)}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dst := http.Header{"Retry-After": []string{"stale"}} + writeOpenAIPassthroughErrorHeaders(dst, http.Header{"Retry-After": []string{tt.raw}}) + if tt.want { + require.Equal(t, tt.raw, dst.Get("Retry-After")) + } else { + require.Empty(t, dst.Get("Retry-After")) + } + }) + } +} + +func TestOpenAIGatewayService_APIKeyPassthrough_CompactErrorBeforeKeepaliveIsSingleJSON(t *testing.T) { + gin.SetMode(gin.TestMode) + rec := httptest.NewRecorder() + c, _ := gin.CreateTestContext(rec) + c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses/compact", bytes.NewReader(nil)) + MarkOpenAICompactClientStream(c) + stop := StartOpenAICompactSSEKeepalive(c, time.Hour) + defer stop() + + svc := &OpenAIGatewayService{ + cfg: &config.Config{Gateway: config.GatewayConfig{ForceCodexCLI: false}}, + httpUpstream: &httpUpstreamRecorder{resp: &http.Response{ + StatusCode: http.StatusBadRequest, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(strings.NewReader(`{"error":{"message":"secret-upstream.example invalid request"}}`)), + }}, + } + account := &Account{ + ID: 125, Platform: PlatformOpenAI, Type: AccountTypeAPIKey, Concurrency: 1, + Credentials: map[string]any{"api_key": "sk-test", "base_url": "https://secret-upstream.example"}, + Extra: map[string]any{"openai_passthrough": true}, Status: StatusActive, Schedulable: true, + } + + _, err := svc.Forward(context.Background(), c, account, []byte(`{"model":"gpt-5.2","input":"hello"}`)) + + require.Error(t, err) + require.Equal(t, http.StatusBadRequest, rec.Code) + require.True(t, gjson.Valid(rec.Body.String())) + require.Equal(t, "upstream_error", gjson.Get(rec.Body.String(), "error.type").String()) + require.NotContains(t, rec.Body.String(), "event:") + require.NotContains(t, rec.Body.String(), ": keepalive") + require.NotContains(t, rec.Body.String(), "secret-upstream.example") +} + +func TestOpenAIGatewayService_APIKeyPassthrough_CompactErrorAfterKeepaliveIsFailedSSE(t *testing.T) { + gin.SetMode(gin.TestMode) + rec := httptest.NewRecorder() + c, _ := gin.CreateTestContext(rec) + c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses/compact", bytes.NewReader(nil)) + MarkOpenAICompactClientStream(c) + stop := StartOpenAICompactSSEKeepalive(c, keepaliveTestInterval) + defer stop() + waitForKeepaliveBeats() + + svc := &OpenAIGatewayService{ + cfg: &config.Config{Gateway: config.GatewayConfig{ForceCodexCLI: false}}, + httpUpstream: &httpUpstreamRecorder{resp: &http.Response{ + StatusCode: http.StatusBadRequest, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(strings.NewReader(`{"error":{"message":"secret-upstream.example invalid request"}}`)), + }}, + } + account := &Account{ + ID: 126, Platform: PlatformOpenAI, Type: AccountTypeAPIKey, Concurrency: 1, + Credentials: map[string]any{"api_key": "sk-test", "base_url": "https://secret-upstream.example"}, + Extra: map[string]any{"openai_passthrough": true}, Status: StatusActive, Schedulable: true, + } + + _, err := svc.Forward(context.Background(), c, account, []byte(`{"model":"gpt-5.2","input":"hello"}`)) + + require.Error(t, err) + require.Equal(t, http.StatusOK, rec.Code) + require.Contains(t, rec.Result().Header.Get("Content-Type"), "text/event-stream") + events := parseCompactBridgeSSE(t, stripKeepaliveComments(rec.Body.String())) + require.Len(t, events, 1) + require.Equal(t, "response.failed", events[0][0]) + require.Equal(t, "failed", gjson.Get(events[0][1], "response.status").String()) + require.Equal(t, "upstream_error", gjson.Get(events[0][1], "response.error.code").String()) + require.Equal(t, "Upstream request failed", gjson.Get(events[0][1], "response.error.message").String()) + require.NotContains(t, rec.Body.String(), "secret-upstream.example") +} + func TestOpenAIGatewayService_OpenAIPassthrough_RetryableStatusesTriggerFailover(t *testing.T) { gin.SetMode(gin.TestMode) originalBody := []byte(`{"model":"gpt-5.2","stream":false,"instructions":"local-test-instructions","input":[{"type":"text","text":"hi"}]}`)