fix(gateway): Anthropic/Bedrock 传输层错误转 failover + 持久故障临时摘除账号

上游 Do/DoWithTLS 返回非 HTTP 错误(代理/DNS/TCP/TLS)时,Anthropic 五条转发路径
(forward/apikey 透传/Bedrock/ForwardAsResponses/ForwardAsChatCompletions)原先
直接向客户端写 502 且不换号,单账号网络故障期间该账号仍持续被调度,请求全量失败。

对齐 OpenAI 侧 handleOpenAIUpstreamTransportError 的既有语义:
- 传输层错误统一返回 *UpstreamFailoverError(502),由 handler failover 循环换号,
  耗尽后才向客户端渲染错误;service 不再写响应
- 持久性故障(connection refused/no route/DNS not found/代理认证失败)额外临时
  摘除账号 10 分钟(仅写库,Anthropic 调度以持久化状态为准,无内存快路径)
- context.Canceled 保持原样返回:不换号也不摘账号
- 分类器改为平台无关命名(classifyUpstreamTransportError)供两侧共用
- Ops 错误事件保留各路径原有字段(UpstreamURL/DurationMs/Passthrough)
This commit is contained in:
feeeei
2026-08-26 22:57:06 +08:00
parent efb46db0a9
commit 44003d7f6b
10 changed files with 290 additions and 104 deletions
@@ -1235,8 +1235,12 @@ func TestGatewayService_AnthropicAPIKeyPassthrough_ForwardDirect_UpstreamRequest
result, err := svc.forwardAnthropicAPIKeyPassthrough(context.Background(), c, account, []byte(`{"model":"x"}`), "x", "x", false, time.Now())
require.Nil(t, result)
require.Error(t, err)
require.Contains(t, err.Error(), "upstream request failed")
require.Equal(t, http.StatusBadGateway, rec.Code)
var failoverErr *UpstreamFailoverError
require.ErrorAs(t, err, &failoverErr)
require.Equal(t, http.StatusBadGateway, failoverErr.StatusCode)
require.True(t, failoverErr.ShouldRetryNextAccount())
// 传输层错误交给 handler failoverservice 不得写响应。
require.False(t, c.Writer.Written())
}
func TestGatewayService_AnthropicAPIKeyPassthrough_ForwardDirect_EmptyResponseBody(t *testing.T) {
@@ -114,29 +114,10 @@ func (s *GatewayService) forwardAnthropicAPIKeyPassthroughWithInput(
if resp != nil && resp.Body != nil {
_ = resp.Body.Close()
}
if !errors.Is(err, context.Canceled) {
scheduleOllamaCloudUsageActivity(s.deferredService, account)
}
safeErr := sanitizeUpstreamErrorMessage(err.Error())
setOpsUpstreamError(c, 0, safeErr, "")
appendOpsUpstreamError(c, OpsUpstreamErrorEvent{
Platform: account.Platform,
AccountID: account.ID,
AccountName: account.Name,
UpstreamStatusCode: 0,
UpstreamURL: safeUpstreamURL(upstreamReq.URL.String()),
Passthrough: true,
Kind: "request_error",
Message: safeErr,
return nil, s.handleUpstreamTransportError(ctx, c, account, err, OpsUpstreamErrorEvent{
UpstreamURL: safeUpstreamURL(upstreamReq.URL.String()),
Passthrough: true,
})
c.JSON(http.StatusBadGateway, gin.H{
"type": "error",
"error": gin.H{
"type": "upstream_error",
"message": "Upstream request failed",
},
})
return nil, fmt.Errorf("upstream request failed: %s", safeErr)
}
// 透传分支禁止 400 请求体降级重试(该重试会改写请求体)
+2 -18
View File
@@ -201,25 +201,9 @@ func (s *GatewayService) executeBedrockUpstream(
if resp != nil && resp.Body != nil {
_ = resp.Body.Close()
}
safeErr := sanitizeUpstreamErrorMessage(err.Error())
setOpsUpstreamError(c, 0, safeErr, "")
appendOpsUpstreamError(c, OpsUpstreamErrorEvent{
Platform: account.Platform,
AccountID: account.ID,
AccountName: account.Name,
UpstreamStatusCode: 0,
UpstreamURL: safeUpstreamURL(upstreamReq.URL.String()),
Kind: "request_error",
Message: safeErr,
return nil, s.handleUpstreamTransportError(ctx, c, account, err, OpsUpstreamErrorEvent{
UpstreamURL: safeUpstreamURL(upstreamReq.URL.String()),
})
c.JSON(http.StatusBadGateway, gin.H{
"type": "error",
"error": gin.H{
"type": "upstream_error",
"message": "Upstream request failed",
},
})
return nil, fmt.Errorf("upstream request failed: %s", safeErr)
}
if resp.StatusCode >= 400 && resp.StatusCode != 400 && s.shouldRetryUpstreamError(account, resp.StatusCode) {
+2 -23
View File
@@ -392,30 +392,9 @@ func (s *GatewayService) Forward(ctx context.Context, c *gin.Context, account *A
if resp != nil && resp.Body != nil {
_ = resp.Body.Close()
}
// Transport attempt left local validation; count Ollama Cloud activity.
if !errors.Is(err, context.Canceled) {
scheduleOllamaCloudUsageActivity(s.deferredService, account)
}
// Ensure the client receives an error response (handlers assume Forward writes on non-failover errors).
safeErr := sanitizeUpstreamErrorMessage(err.Error())
setOpsUpstreamError(c, 0, safeErr, "")
appendOpsUpstreamError(c, OpsUpstreamErrorEvent{
Platform: account.Platform,
AccountID: account.ID,
AccountName: account.Name,
UpstreamStatusCode: 0,
UpstreamURL: safeUpstreamURL(upstreamReq.URL.String()),
Kind: "request_error",
Message: safeErr,
return nil, s.handleUpstreamTransportError(ctx, c, account, err, OpsUpstreamErrorEvent{
UpstreamURL: safeUpstreamURL(upstreamReq.URL.String()),
})
c.JSON(http.StatusBadGateway, gin.H{
"type": "error",
"error": gin.H{
"type": "upstream_error",
"message": "Upstream request failed",
},
})
return nil, fmt.Errorf("upstream request failed: %s", safeErr)
}
// 优先检测thinking block签名错误(400)并重试一次
@@ -131,18 +131,9 @@ func (s *GatewayService) ForwardAsChatCompletions(
if resp != nil && resp.Body != nil {
_ = resp.Body.Close()
}
safeErr := sanitizeUpstreamErrorMessage(err.Error())
setOpsUpstreamError(c, 0, safeErr, "")
appendOpsUpstreamError(c, OpsUpstreamErrorEvent{
Platform: account.Platform,
AccountID: account.ID,
AccountName: account.Name,
UpstreamStatusCode: 0,
Kind: "request_error",
Message: safeErr,
return nil, s.handleUpstreamTransportError(ctx, c, account, err, OpsUpstreamErrorEvent{
UpstreamURL: safeUpstreamURL(upstreamReq.URL.String()),
})
writeGatewayCCError(c, http.StatusBadGateway, "server_error", "Upstream request failed")
return nil, fmt.Errorf("upstream request failed: %s", safeErr)
}
defer func() { _ = resp.Body.Close() }()
@@ -144,18 +144,9 @@ func (s *GatewayService) ForwardAsResponses(
if resp != nil && resp.Body != nil {
_ = resp.Body.Close()
}
safeErr := sanitizeUpstreamErrorMessage(err.Error())
setOpsUpstreamError(c, 0, safeErr, "")
appendOpsUpstreamError(c, OpsUpstreamErrorEvent{
Platform: account.Platform,
AccountID: account.ID,
AccountName: account.Name,
UpstreamStatusCode: 0,
Kind: "request_error",
Message: safeErr,
return nil, s.handleUpstreamTransportError(ctx, c, account, err, OpsUpstreamErrorEvent{
UpstreamURL: safeUpstreamURL(upstreamReq.URL.String()),
})
writeResponsesError(c, http.StatusBadGateway, "server_error", "Upstream request failed")
return nil, fmt.Errorf("upstream request failed: %s", safeErr)
}
defer func() { _ = resp.Body.Close() }()
@@ -0,0 +1,106 @@
package service
import (
"context"
"errors"
"net/http"
"time"
"github.com/Wei-Shaw/sub2api/internal/pkg/logger"
"github.com/gin-gonic/gin"
"go.uber.org/zap"
)
// gatewayTransportErrorTempUnschedDuration is how long an account is temporarily
// unscheduled after a durable transport failure (matches the OpenAI-side
// openAITransportErrorTempUnschedDuration).
const gatewayTransportErrorTempUnschedDuration = 10 * time.Minute
// gatewayTransportFailoverBody is the Anthropic-format error body attached to
// the failover error for a transport-level failure. Kept identical to the
// legacy inline 502 body so the client-visible payload is unchanged if
// failover is ultimately exhausted.
var gatewayTransportFailoverBody = []byte(`{"type":"error","error":{"type":"upstream_error","message":"Upstream request failed"}}`)
// handleUpstreamTransportError handles a transport-level upstream failure on
// the Anthropic/Bedrock forward paths (Do/DoWithTLS returned a non-HTTP error:
// proxy / DNS / TCP / TLS). It:
// 1. records the failure in Ops error logs (status 0, kind=request_error) —
// the caller passes path-specific fields (UpstreamURL, Passthrough) via
// event; identity and classification fields are filled here;
// 2. for durable faults (expired/rejected proxy creds, dead proxy,
// DNS/routing) temporarily unschedules the account and logs a stable warn
// event that alert rules can key on;
// 3. returns an error that is *UpstreamFailoverError (so the handler fails
// over to a healthy account) for all non-canceled errors, or the original
// error for context.Canceled (client gone — no failover, no eviction).
//
// It deliberately does NOT write to the response: the handler owns the
// response (failover, or a protocol-correct error once failover is exhausted).
func (s *GatewayService) handleUpstreamTransportError(ctx context.Context, c *gin.Context, account *Account, err error, event OpsUpstreamErrorEvent) error {
safeErr := sanitizeUpstreamErrorMessage(err.Error())
setOpsUpstreamError(c, 0, safeErr, "")
event.Platform = account.Platform
event.AccountID = account.ID
event.AccountName = account.Name
event.UpstreamStatusCode = 0
event.Kind = "request_error"
event.Message = safeErr
appendOpsUpstreamError(c, event)
// Client disconnected: do NOT fail over to another account and do NOT
// evict this one — the upstream never had a chance to exhibit a fault.
if errors.Is(err, context.Canceled) || (errors.Is(err, context.DeadlineExceeded) && errors.Is(ctx.Err(), context.DeadlineExceeded)) {
return err
}
// Transport attempt left local validation; count Ollama Cloud activity.
scheduleOllamaCloudUsageActivity(s.deferredService, account)
if classifyUpstreamTransportError(err).Persistent {
s.tempUnscheduleTransportError(ctx, account, safeErr)
}
return &UpstreamFailoverError{
StatusCode: http.StatusBadGateway,
ResponseBody: gatewayTransportFailoverBody,
}
}
// tempUnscheduleTransportError marks an account temporarily unschedulable
// after a durable transport failure. Unlike the OpenAI side there is no
// in-memory scheduler block on this path: the Anthropic/Bedrock scheduler
// reads the persisted temp-unschedulable state, so the DB write is the single
// source of truth (same as tempUnscheduleGoogleConfigError /
// tempUnscheduleEmptyResponse).
//
// Log semantics:
// - "gateway.account_temp_unscheduled_transport" — DB write succeeded.
// - "gateway.account_temp_unschedule_transport_failed" — DB write attempted
// but returned an error (the account remains schedulable).
func (s *GatewayService) tempUnscheduleTransportError(ctx context.Context, account *Account, safeErr string) {
if s == nil || account == nil || s.accountRepo == nil {
return
}
until := time.Now().Add(gatewayTransportErrorTempUnschedDuration)
reason := "upstream transport error (proxy/network): " + safeErr
bgCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), openAIAccountStateUpdateTimeout)
defer cancel()
if err := s.accountRepo.SetTempUnschedulable(bgCtx, account.ID, until, reason); err != nil {
logger.L().With(zap.String("component", "service.gateway")).Warn(
"gateway.account_temp_unschedule_transport_failed",
zap.Int64("account_id", account.ID),
zap.Error(err),
)
return
}
logger.L().With(zap.String("component", "service.gateway")).Warn(
"gateway.account_temp_unscheduled_transport",
zap.Int64("account_id", account.ID),
zap.String("account_name", account.Name),
zap.String("platform", account.Platform),
zap.Time("until", until),
zap.String("reason", reason),
)
}
@@ -0,0 +1,150 @@
//go:build unit
package service
import (
"context"
"errors"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/gin-gonic/gin"
)
type transportTempUnschedRepoStub struct {
AccountRepository
calls int
lastID int64
lastUntil time.Time
lastReason string
}
func (r *transportTempUnschedRepoStub) SetTempUnschedulable(_ context.Context, id int64, until time.Time, reason string) error {
r.calls++
r.lastID = id
r.lastUntil = until
r.lastReason = reason
return nil
}
func newTransportErrorTestGin(t *testing.T) *gin.Context {
t.Helper()
gin.SetMode(gin.TestMode)
c, _ := gin.CreateTestContext(httptest.NewRecorder())
c.Request = httptest.NewRequest(http.MethodPost, "/v1/messages", nil)
return c
}
// TestHandleUpstreamTransportError_TransientFailsOverWithoutEviction pins the
// contract for transient transport blips (EOF / connection reset): the request
// fails over to another account, the current account stays schedulable, and
// nothing is written to the response (the handler owns it).
func TestHandleUpstreamTransportError_TransientFailsOverWithoutEviction(t *testing.T) {
repo := &transportTempUnschedRepoStub{}
s := &GatewayService{accountRepo: repo}
c := newTransportErrorTestGin(t)
account := &Account{ID: 149, Name: "acc", Platform: PlatformAnthropic}
err := s.handleUpstreamTransportError(context.Background(), c, account,
errors.New(`Post "http://upstream/v1/messages?beta=true": EOF`), OpsUpstreamErrorEvent{})
var failoverErr *UpstreamFailoverError
if !errors.As(err, &failoverErr) {
t.Fatalf("expected *UpstreamFailoverError, got %T: %v", err, err)
}
if failoverErr.StatusCode != http.StatusBadGateway {
t.Fatalf("StatusCode = %d, want 502", failoverErr.StatusCode)
}
if string(failoverErr.ResponseBody) != string(gatewayTransportFailoverBody) {
t.Fatalf("ResponseBody = %s, want legacy 502 body", failoverErr.ResponseBody)
}
if !failoverErr.ShouldRetryNextAccount() {
t.Fatal("transient transport error must allow retrying the next account")
}
if repo.calls != 0 {
t.Fatalf("SetTempUnschedulable called %d times for a transient error, want 0", repo.calls)
}
if c.Writer.Written() {
t.Fatal("handler owns the response; service must not write on transport failover")
}
}
// TestHandleUpstreamTransportError_PersistentEvictsAccount pins the contract
// for durable faults (dead endpoint / DNS / proxy credentials): fail over AND
// temporarily unschedule the account for the transport cooldown.
func TestHandleUpstreamTransportError_PersistentEvictsAccount(t *testing.T) {
repo := &transportTempUnschedRepoStub{}
s := &GatewayService{accountRepo: repo}
c := newTransportErrorTestGin(t)
account := &Account{ID: 149, Name: "acc", Platform: PlatformAnthropic}
before := time.Now()
err := s.handleUpstreamTransportError(context.Background(), c, account,
errors.New(`dial tcp 1.2.3.4:443: connect: connection refused`), OpsUpstreamErrorEvent{})
var failoverErr *UpstreamFailoverError
if !errors.As(err, &failoverErr) {
t.Fatalf("expected *UpstreamFailoverError, got %T: %v", err, err)
}
if repo.calls != 1 {
t.Fatalf("SetTempUnschedulable called %d times for a persistent error, want 1", repo.calls)
}
if repo.lastID != account.ID {
t.Fatalf("unscheduled account = %d, want %d", repo.lastID, account.ID)
}
wantUntil := before.Add(gatewayTransportErrorTempUnschedDuration)
if repo.lastUntil.Before(wantUntil.Add(-time.Minute)) || repo.lastUntil.After(wantUntil.Add(time.Minute)) {
t.Fatalf("until = %v, want ~%v", repo.lastUntil, wantUntil)
}
if !strings.HasPrefix(repo.lastReason, "upstream transport error (proxy/network): ") {
t.Fatalf("reason = %q, want transport-error prefix", repo.lastReason)
}
}
// TestHandleUpstreamTransportError_ClientCanceledNoFailover pins that a
// canceled client neither fails over nor evicts: the upstream never had a
// chance to exhibit a fault.
func TestHandleUpstreamTransportError_ClientCanceledNoFailover(t *testing.T) {
repo := &transportTempUnschedRepoStub{}
s := &GatewayService{accountRepo: repo}
c := newTransportErrorTestGin(t)
account := &Account{ID: 149, Name: "acc", Platform: PlatformAnthropic}
inErr := context.Canceled
err := s.handleUpstreamTransportError(context.Background(), c, account, inErr, OpsUpstreamErrorEvent{})
var failoverErr *UpstreamFailoverError
if errors.As(err, &failoverErr) {
t.Fatal("canceled client must not fail over to another account")
}
if !errors.Is(err, context.Canceled) {
t.Fatalf("err = %v, want context.Canceled passthrough", err)
}
if repo.calls != 0 {
t.Fatalf("SetTempUnschedulable called %d times on client cancel, want 0", repo.calls)
}
}
// TestHandleUpstreamTransportError_UpstreamDeadlineStillFailsOver pins that an
// upstream-side timeout (request context still alive) is treated as a
// transient fault: fail over, no eviction.
func TestHandleUpstreamTransportError_UpstreamDeadlineStillFailsOver(t *testing.T) {
repo := &transportTempUnschedRepoStub{}
s := &GatewayService{accountRepo: repo}
c := newTransportErrorTestGin(t)
account := &Account{ID: 149, Name: "acc", Platform: PlatformAnthropic}
err := s.handleUpstreamTransportError(context.Background(), c, account,
context.DeadlineExceeded, OpsUpstreamErrorEvent{})
var failoverErr *UpstreamFailoverError
if !errors.As(err, &failoverErr) {
t.Fatalf("upstream deadline with live request context must fail over, got %T: %v", err, err)
}
if repo.calls != 0 {
t.Fatalf("SetTempUnschedulable called %d times for upstream deadline, want 0", repo.calls)
}
}
@@ -24,10 +24,10 @@ const openAITransportErrorTempUnschedDuration = 10 * time.Minute
// ultimately exhausted.
var openAITransportFailoverBody = []byte(`{"error":{"type":"upstream_error","message":"Upstream request failed"}}`)
// openAITransportErrorClass describes how to react to a transport-level upstream
// upstreamTransportErrorClass describes how to react to a transport-level upstream
// failure — i.e. the HTTP round-trip never completed (proxy / DNS / TCP / TLS
// error, no HTTP status code received).
type openAITransportErrorClass struct {
type upstreamTransportErrorClass struct {
// Persistent marks failures where retrying the same proxy/account is
// pointless: expired or rejected proxy credentials, a dead proxy endpoint,
// or DNS/routing failure. Such accounts should be temporarily unscheduled
@@ -35,12 +35,12 @@ type openAITransportErrorClass struct {
Persistent bool
}
// openAIPersistentTransportErrorMarkers are substrings (matched case-insensitively
// persistentUpstreamTransportErrorMarkers are substrings (matched case-insensitively
// against the raw transport error) that indicate a durable proxy/network fault.
// Matched signals are intentionally specific failure *reasons*, not the operation
// (e.g. we match "connection refused", not "proxyconnect") so that a transient
// failure of the same operation (a proxy timeout) is NOT misclassified as durable.
var openAIPersistentTransportErrorMarkers = []string{
var persistentUpstreamTransportErrorMarkers = []string{
"authentication failed", // SOCKS5 RFC1929 / proxy credentials rejected (expired account)
"proxy authentication required", // HTTP proxy 407
"connection refused", // proxy/upstream endpoint down
@@ -49,7 +49,7 @@ var openAIPersistentTransportErrorMarkers = []string{
"no such host", // DNS resolution failure (bad/expired proxy hostname)
}
// classifyOpenAITransportError decides whether a transport-level upstream error
// classifyUpstreamTransportError decides whether a transport-level upstream error
// is durable (Persistent — evict the account + alert) or a transient blip
// (fail over to a healthy account but keep this one schedulable).
//
@@ -65,30 +65,30 @@ var openAIPersistentTransportErrorMarkers = []string{
// The network-layer string markers ("connection refused", "no route to host",
// "network is unreachable", "no such host") are kept as a cross-platform safety
// net even though the typed checks should cover them on modern Go+Linux.
func classifyOpenAITransportError(err error) openAITransportErrorClass {
func classifyUpstreamTransportError(err error) upstreamTransportErrorClass {
if err == nil {
return openAITransportErrorClass{}
return upstreamTransportErrorClass{}
}
// — Typed checks (preferred) ——————————————————————————————————————————————
if errors.Is(err, syscall.ECONNREFUSED) ||
errors.Is(err, syscall.EHOSTUNREACH) ||
errors.Is(err, syscall.ENETUNREACH) {
return openAITransportErrorClass{Persistent: true}
return upstreamTransportErrorClass{Persistent: true}
}
var dnsErr *net.DNSError
if errors.As(err, &dnsErr) && dnsErr.IsNotFound {
return openAITransportErrorClass{Persistent: true}
return upstreamTransportErrorClass{Persistent: true}
}
// — String-marker fallback ————————————————————————————————————————————————
msg := strings.ToLower(err.Error())
for _, marker := range openAIPersistentTransportErrorMarkers {
for _, marker := range persistentUpstreamTransportErrorMarkers {
if strings.Contains(msg, marker) {
return openAITransportErrorClass{Persistent: true}
return upstreamTransportErrorClass{Persistent: true}
}
}
return openAITransportErrorClass{}
return upstreamTransportErrorClass{}
}
// handleOpenAIUpstreamTransportError handles a transport-level upstream failure
@@ -135,7 +135,7 @@ func (s *OpenAIGatewayService) handleOpenAIUpstreamTransportError(ctx context.Co
return err
}
if classifyOpenAITransportError(err).Persistent {
if classifyUpstreamTransportError(err).Persistent {
s.tempUnscheduleOpenAITransportError(ctx, account, safeErr)
}
@@ -11,13 +11,13 @@ import (
"testing"
)
// TestClassifyOpenAITransportError pins which transport-level upstream failures
// TestClassifyUpstreamTransportError pins which transport-level upstream failures
// are "persistent" (retrying the same proxy/account is pointless — evict + alert)
// versus "transient" (a blip — fail over to a healthy account but do not evict).
//
// The motivating incident: a SOCKS5 proxy whose credentials expired returned
// `username/password authentication failed`, yet the account kept being scheduled.
func TestClassifyOpenAITransportError(t *testing.T) {
func TestClassifyUpstreamTransportError(t *testing.T) {
cases := []struct {
name string
err error
@@ -76,9 +76,9 @@ func TestClassifyOpenAITransportError(t *testing.T) {
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
got := classifyOpenAITransportError(tc.err).Persistent
got := classifyUpstreamTransportError(tc.err).Persistent
if got != tc.persistent {
t.Fatalf("classifyOpenAITransportError(%q).Persistent = %v, want %v", errString(tc.err), got, tc.persistent)
t.Fatalf("classifyUpstreamTransportError(%q).Persistent = %v, want %v", errString(tc.err), got, tc.persistent)
}
})
}