mirror of
https://github.com/Wei-Shaw/sub2api.git
synced 2026-08-31 01:13:06 +08:00
fix(gateway): Anthropic/Bedrock 传输层错误转 failover + 持久故障临时摘除账号
上游 Do/DoWithTLS 返回非 HTTP 错误(代理/DNS/TCP/TLS)时,Anthropic 五条转发路径 (forward/apikey 透传/Bedrock/ForwardAsResponses/ForwardAsChatCompletions)原先 直接向客户端写 502 且不换号,单账号网络故障期间该账号仍持续被调度,请求全量失败。 对齐 OpenAI 侧 handleOpenAIUpstreamTransportError 的既有语义: - 传输层错误统一返回 *UpstreamFailoverError(502),由 handler failover 循环换号, 耗尽后才向客户端渲染错误;service 不再写响应 - 持久性故障(connection refused/no route/DNS not found/代理认证失败)额外临时 摘除账号 10 分钟(仅写库,Anthropic 调度以持久化状态为准,无内存快路径) - context.Canceled 保持原样返回:不换号也不摘账号 - 分类器改为平台无关命名(classifyUpstreamTransportError)供两侧共用 - Ops 错误事件保留各路径原有字段(UpstreamURL/DurationMs/Passthrough)
This commit is contained in:
@@ -1235,8 +1235,12 @@ func TestGatewayService_AnthropicAPIKeyPassthrough_ForwardDirect_UpstreamRequest
|
||||
result, err := svc.forwardAnthropicAPIKeyPassthrough(context.Background(), c, account, []byte(`{"model":"x"}`), "x", "x", false, time.Now())
|
||||
require.Nil(t, result)
|
||||
require.Error(t, err)
|
||||
require.Contains(t, err.Error(), "upstream request failed")
|
||||
require.Equal(t, http.StatusBadGateway, rec.Code)
|
||||
var failoverErr *UpstreamFailoverError
|
||||
require.ErrorAs(t, err, &failoverErr)
|
||||
require.Equal(t, http.StatusBadGateway, failoverErr.StatusCode)
|
||||
require.True(t, failoverErr.ShouldRetryNextAccount())
|
||||
// 传输层错误交给 handler failover,service 不得写响应。
|
||||
require.False(t, c.Writer.Written())
|
||||
}
|
||||
|
||||
func TestGatewayService_AnthropicAPIKeyPassthrough_ForwardDirect_EmptyResponseBody(t *testing.T) {
|
||||
|
||||
@@ -114,29 +114,10 @@ func (s *GatewayService) forwardAnthropicAPIKeyPassthroughWithInput(
|
||||
if resp != nil && resp.Body != nil {
|
||||
_ = resp.Body.Close()
|
||||
}
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
scheduleOllamaCloudUsageActivity(s.deferredService, account)
|
||||
}
|
||||
safeErr := sanitizeUpstreamErrorMessage(err.Error())
|
||||
setOpsUpstreamError(c, 0, safeErr, "")
|
||||
appendOpsUpstreamError(c, OpsUpstreamErrorEvent{
|
||||
Platform: account.Platform,
|
||||
AccountID: account.ID,
|
||||
AccountName: account.Name,
|
||||
UpstreamStatusCode: 0,
|
||||
UpstreamURL: safeUpstreamURL(upstreamReq.URL.String()),
|
||||
Passthrough: true,
|
||||
Kind: "request_error",
|
||||
Message: safeErr,
|
||||
return nil, s.handleUpstreamTransportError(ctx, c, account, err, OpsUpstreamErrorEvent{
|
||||
UpstreamURL: safeUpstreamURL(upstreamReq.URL.String()),
|
||||
Passthrough: true,
|
||||
})
|
||||
c.JSON(http.StatusBadGateway, gin.H{
|
||||
"type": "error",
|
||||
"error": gin.H{
|
||||
"type": "upstream_error",
|
||||
"message": "Upstream request failed",
|
||||
},
|
||||
})
|
||||
return nil, fmt.Errorf("upstream request failed: %s", safeErr)
|
||||
}
|
||||
|
||||
// 透传分支禁止 400 请求体降级重试(该重试会改写请求体)
|
||||
|
||||
@@ -201,25 +201,9 @@ func (s *GatewayService) executeBedrockUpstream(
|
||||
if resp != nil && resp.Body != nil {
|
||||
_ = resp.Body.Close()
|
||||
}
|
||||
safeErr := sanitizeUpstreamErrorMessage(err.Error())
|
||||
setOpsUpstreamError(c, 0, safeErr, "")
|
||||
appendOpsUpstreamError(c, OpsUpstreamErrorEvent{
|
||||
Platform: account.Platform,
|
||||
AccountID: account.ID,
|
||||
AccountName: account.Name,
|
||||
UpstreamStatusCode: 0,
|
||||
UpstreamURL: safeUpstreamURL(upstreamReq.URL.String()),
|
||||
Kind: "request_error",
|
||||
Message: safeErr,
|
||||
return nil, s.handleUpstreamTransportError(ctx, c, account, err, OpsUpstreamErrorEvent{
|
||||
UpstreamURL: safeUpstreamURL(upstreamReq.URL.String()),
|
||||
})
|
||||
c.JSON(http.StatusBadGateway, gin.H{
|
||||
"type": "error",
|
||||
"error": gin.H{
|
||||
"type": "upstream_error",
|
||||
"message": "Upstream request failed",
|
||||
},
|
||||
})
|
||||
return nil, fmt.Errorf("upstream request failed: %s", safeErr)
|
||||
}
|
||||
|
||||
if resp.StatusCode >= 400 && resp.StatusCode != 400 && s.shouldRetryUpstreamError(account, resp.StatusCode) {
|
||||
|
||||
@@ -392,30 +392,9 @@ func (s *GatewayService) Forward(ctx context.Context, c *gin.Context, account *A
|
||||
if resp != nil && resp.Body != nil {
|
||||
_ = resp.Body.Close()
|
||||
}
|
||||
// Transport attempt left local validation; count Ollama Cloud activity.
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
scheduleOllamaCloudUsageActivity(s.deferredService, account)
|
||||
}
|
||||
// Ensure the client receives an error response (handlers assume Forward writes on non-failover errors).
|
||||
safeErr := sanitizeUpstreamErrorMessage(err.Error())
|
||||
setOpsUpstreamError(c, 0, safeErr, "")
|
||||
appendOpsUpstreamError(c, OpsUpstreamErrorEvent{
|
||||
Platform: account.Platform,
|
||||
AccountID: account.ID,
|
||||
AccountName: account.Name,
|
||||
UpstreamStatusCode: 0,
|
||||
UpstreamURL: safeUpstreamURL(upstreamReq.URL.String()),
|
||||
Kind: "request_error",
|
||||
Message: safeErr,
|
||||
return nil, s.handleUpstreamTransportError(ctx, c, account, err, OpsUpstreamErrorEvent{
|
||||
UpstreamURL: safeUpstreamURL(upstreamReq.URL.String()),
|
||||
})
|
||||
c.JSON(http.StatusBadGateway, gin.H{
|
||||
"type": "error",
|
||||
"error": gin.H{
|
||||
"type": "upstream_error",
|
||||
"message": "Upstream request failed",
|
||||
},
|
||||
})
|
||||
return nil, fmt.Errorf("upstream request failed: %s", safeErr)
|
||||
}
|
||||
|
||||
// 优先检测thinking block签名错误(400)并重试一次
|
||||
|
||||
@@ -131,18 +131,9 @@ func (s *GatewayService) ForwardAsChatCompletions(
|
||||
if resp != nil && resp.Body != nil {
|
||||
_ = resp.Body.Close()
|
||||
}
|
||||
safeErr := sanitizeUpstreamErrorMessage(err.Error())
|
||||
setOpsUpstreamError(c, 0, safeErr, "")
|
||||
appendOpsUpstreamError(c, OpsUpstreamErrorEvent{
|
||||
Platform: account.Platform,
|
||||
AccountID: account.ID,
|
||||
AccountName: account.Name,
|
||||
UpstreamStatusCode: 0,
|
||||
Kind: "request_error",
|
||||
Message: safeErr,
|
||||
return nil, s.handleUpstreamTransportError(ctx, c, account, err, OpsUpstreamErrorEvent{
|
||||
UpstreamURL: safeUpstreamURL(upstreamReq.URL.String()),
|
||||
})
|
||||
writeGatewayCCError(c, http.StatusBadGateway, "server_error", "Upstream request failed")
|
||||
return nil, fmt.Errorf("upstream request failed: %s", safeErr)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
|
||||
@@ -144,18 +144,9 @@ func (s *GatewayService) ForwardAsResponses(
|
||||
if resp != nil && resp.Body != nil {
|
||||
_ = resp.Body.Close()
|
||||
}
|
||||
safeErr := sanitizeUpstreamErrorMessage(err.Error())
|
||||
setOpsUpstreamError(c, 0, safeErr, "")
|
||||
appendOpsUpstreamError(c, OpsUpstreamErrorEvent{
|
||||
Platform: account.Platform,
|
||||
AccountID: account.ID,
|
||||
AccountName: account.Name,
|
||||
UpstreamStatusCode: 0,
|
||||
Kind: "request_error",
|
||||
Message: safeErr,
|
||||
return nil, s.handleUpstreamTransportError(ctx, c, account, err, OpsUpstreamErrorEvent{
|
||||
UpstreamURL: safeUpstreamURL(upstreamReq.URL.String()),
|
||||
})
|
||||
writeResponsesError(c, http.StatusBadGateway, "server_error", "Upstream request failed")
|
||||
return nil, fmt.Errorf("upstream request failed: %s", safeErr)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/pkg/logger"
|
||||
"github.com/gin-gonic/gin"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// gatewayTransportErrorTempUnschedDuration is how long an account is temporarily
|
||||
// unscheduled after a durable transport failure (matches the OpenAI-side
|
||||
// openAITransportErrorTempUnschedDuration).
|
||||
const gatewayTransportErrorTempUnschedDuration = 10 * time.Minute
|
||||
|
||||
// gatewayTransportFailoverBody is the Anthropic-format error body attached to
|
||||
// the failover error for a transport-level failure. Kept identical to the
|
||||
// legacy inline 502 body so the client-visible payload is unchanged if
|
||||
// failover is ultimately exhausted.
|
||||
var gatewayTransportFailoverBody = []byte(`{"type":"error","error":{"type":"upstream_error","message":"Upstream request failed"}}`)
|
||||
|
||||
// handleUpstreamTransportError handles a transport-level upstream failure on
|
||||
// the Anthropic/Bedrock forward paths (Do/DoWithTLS returned a non-HTTP error:
|
||||
// proxy / DNS / TCP / TLS). It:
|
||||
// 1. records the failure in Ops error logs (status 0, kind=request_error) —
|
||||
// the caller passes path-specific fields (UpstreamURL, Passthrough) via
|
||||
// event; identity and classification fields are filled here;
|
||||
// 2. for durable faults (expired/rejected proxy creds, dead proxy,
|
||||
// DNS/routing) temporarily unschedules the account and logs a stable warn
|
||||
// event that alert rules can key on;
|
||||
// 3. returns an error that is *UpstreamFailoverError (so the handler fails
|
||||
// over to a healthy account) for all non-canceled errors, or the original
|
||||
// error for context.Canceled (client gone — no failover, no eviction).
|
||||
//
|
||||
// It deliberately does NOT write to the response: the handler owns the
|
||||
// response (failover, or a protocol-correct error once failover is exhausted).
|
||||
func (s *GatewayService) handleUpstreamTransportError(ctx context.Context, c *gin.Context, account *Account, err error, event OpsUpstreamErrorEvent) error {
|
||||
safeErr := sanitizeUpstreamErrorMessage(err.Error())
|
||||
setOpsUpstreamError(c, 0, safeErr, "")
|
||||
event.Platform = account.Platform
|
||||
event.AccountID = account.ID
|
||||
event.AccountName = account.Name
|
||||
event.UpstreamStatusCode = 0
|
||||
event.Kind = "request_error"
|
||||
event.Message = safeErr
|
||||
appendOpsUpstreamError(c, event)
|
||||
|
||||
// Client disconnected: do NOT fail over to another account and do NOT
|
||||
// evict this one — the upstream never had a chance to exhibit a fault.
|
||||
if errors.Is(err, context.Canceled) || (errors.Is(err, context.DeadlineExceeded) && errors.Is(ctx.Err(), context.DeadlineExceeded)) {
|
||||
return err
|
||||
}
|
||||
|
||||
// Transport attempt left local validation; count Ollama Cloud activity.
|
||||
scheduleOllamaCloudUsageActivity(s.deferredService, account)
|
||||
|
||||
if classifyUpstreamTransportError(err).Persistent {
|
||||
s.tempUnscheduleTransportError(ctx, account, safeErr)
|
||||
}
|
||||
|
||||
return &UpstreamFailoverError{
|
||||
StatusCode: http.StatusBadGateway,
|
||||
ResponseBody: gatewayTransportFailoverBody,
|
||||
}
|
||||
}
|
||||
|
||||
// tempUnscheduleTransportError marks an account temporarily unschedulable
|
||||
// after a durable transport failure. Unlike the OpenAI side there is no
|
||||
// in-memory scheduler block on this path: the Anthropic/Bedrock scheduler
|
||||
// reads the persisted temp-unschedulable state, so the DB write is the single
|
||||
// source of truth (same as tempUnscheduleGoogleConfigError /
|
||||
// tempUnscheduleEmptyResponse).
|
||||
//
|
||||
// Log semantics:
|
||||
// - "gateway.account_temp_unscheduled_transport" — DB write succeeded.
|
||||
// - "gateway.account_temp_unschedule_transport_failed" — DB write attempted
|
||||
// but returned an error (the account remains schedulable).
|
||||
func (s *GatewayService) tempUnscheduleTransportError(ctx context.Context, account *Account, safeErr string) {
|
||||
if s == nil || account == nil || s.accountRepo == nil {
|
||||
return
|
||||
}
|
||||
until := time.Now().Add(gatewayTransportErrorTempUnschedDuration)
|
||||
reason := "upstream transport error (proxy/network): " + safeErr
|
||||
|
||||
bgCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), openAIAccountStateUpdateTimeout)
|
||||
defer cancel()
|
||||
if err := s.accountRepo.SetTempUnschedulable(bgCtx, account.ID, until, reason); err != nil {
|
||||
logger.L().With(zap.String("component", "service.gateway")).Warn(
|
||||
"gateway.account_temp_unschedule_transport_failed",
|
||||
zap.Int64("account_id", account.ID),
|
||||
zap.Error(err),
|
||||
)
|
||||
return
|
||||
}
|
||||
logger.L().With(zap.String("component", "service.gateway")).Warn(
|
||||
"gateway.account_temp_unscheduled_transport",
|
||||
zap.Int64("account_id", account.ID),
|
||||
zap.String("account_name", account.Name),
|
||||
zap.String("platform", account.Platform),
|
||||
zap.Time("until", until),
|
||||
zap.String("reason", reason),
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
//go:build unit
|
||||
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
type transportTempUnschedRepoStub struct {
|
||||
AccountRepository
|
||||
calls int
|
||||
lastID int64
|
||||
lastUntil time.Time
|
||||
lastReason string
|
||||
}
|
||||
|
||||
func (r *transportTempUnschedRepoStub) SetTempUnschedulable(_ context.Context, id int64, until time.Time, reason string) error {
|
||||
r.calls++
|
||||
r.lastID = id
|
||||
r.lastUntil = until
|
||||
r.lastReason = reason
|
||||
return nil
|
||||
}
|
||||
|
||||
func newTransportErrorTestGin(t *testing.T) *gin.Context {
|
||||
t.Helper()
|
||||
gin.SetMode(gin.TestMode)
|
||||
c, _ := gin.CreateTestContext(httptest.NewRecorder())
|
||||
c.Request = httptest.NewRequest(http.MethodPost, "/v1/messages", nil)
|
||||
return c
|
||||
}
|
||||
|
||||
// TestHandleUpstreamTransportError_TransientFailsOverWithoutEviction pins the
|
||||
// contract for transient transport blips (EOF / connection reset): the request
|
||||
// fails over to another account, the current account stays schedulable, and
|
||||
// nothing is written to the response (the handler owns it).
|
||||
func TestHandleUpstreamTransportError_TransientFailsOverWithoutEviction(t *testing.T) {
|
||||
repo := &transportTempUnschedRepoStub{}
|
||||
s := &GatewayService{accountRepo: repo}
|
||||
c := newTransportErrorTestGin(t)
|
||||
account := &Account{ID: 149, Name: "acc", Platform: PlatformAnthropic}
|
||||
|
||||
err := s.handleUpstreamTransportError(context.Background(), c, account,
|
||||
errors.New(`Post "http://upstream/v1/messages?beta=true": EOF`), OpsUpstreamErrorEvent{})
|
||||
|
||||
var failoverErr *UpstreamFailoverError
|
||||
if !errors.As(err, &failoverErr) {
|
||||
t.Fatalf("expected *UpstreamFailoverError, got %T: %v", err, err)
|
||||
}
|
||||
if failoverErr.StatusCode != http.StatusBadGateway {
|
||||
t.Fatalf("StatusCode = %d, want 502", failoverErr.StatusCode)
|
||||
}
|
||||
if string(failoverErr.ResponseBody) != string(gatewayTransportFailoverBody) {
|
||||
t.Fatalf("ResponseBody = %s, want legacy 502 body", failoverErr.ResponseBody)
|
||||
}
|
||||
if !failoverErr.ShouldRetryNextAccount() {
|
||||
t.Fatal("transient transport error must allow retrying the next account")
|
||||
}
|
||||
if repo.calls != 0 {
|
||||
t.Fatalf("SetTempUnschedulable called %d times for a transient error, want 0", repo.calls)
|
||||
}
|
||||
if c.Writer.Written() {
|
||||
t.Fatal("handler owns the response; service must not write on transport failover")
|
||||
}
|
||||
}
|
||||
|
||||
// TestHandleUpstreamTransportError_PersistentEvictsAccount pins the contract
|
||||
// for durable faults (dead endpoint / DNS / proxy credentials): fail over AND
|
||||
// temporarily unschedule the account for the transport cooldown.
|
||||
func TestHandleUpstreamTransportError_PersistentEvictsAccount(t *testing.T) {
|
||||
repo := &transportTempUnschedRepoStub{}
|
||||
s := &GatewayService{accountRepo: repo}
|
||||
c := newTransportErrorTestGin(t)
|
||||
account := &Account{ID: 149, Name: "acc", Platform: PlatformAnthropic}
|
||||
|
||||
before := time.Now()
|
||||
err := s.handleUpstreamTransportError(context.Background(), c, account,
|
||||
errors.New(`dial tcp 1.2.3.4:443: connect: connection refused`), OpsUpstreamErrorEvent{})
|
||||
|
||||
var failoverErr *UpstreamFailoverError
|
||||
if !errors.As(err, &failoverErr) {
|
||||
t.Fatalf("expected *UpstreamFailoverError, got %T: %v", err, err)
|
||||
}
|
||||
if repo.calls != 1 {
|
||||
t.Fatalf("SetTempUnschedulable called %d times for a persistent error, want 1", repo.calls)
|
||||
}
|
||||
if repo.lastID != account.ID {
|
||||
t.Fatalf("unscheduled account = %d, want %d", repo.lastID, account.ID)
|
||||
}
|
||||
wantUntil := before.Add(gatewayTransportErrorTempUnschedDuration)
|
||||
if repo.lastUntil.Before(wantUntil.Add(-time.Minute)) || repo.lastUntil.After(wantUntil.Add(time.Minute)) {
|
||||
t.Fatalf("until = %v, want ~%v", repo.lastUntil, wantUntil)
|
||||
}
|
||||
if !strings.HasPrefix(repo.lastReason, "upstream transport error (proxy/network): ") {
|
||||
t.Fatalf("reason = %q, want transport-error prefix", repo.lastReason)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHandleUpstreamTransportError_ClientCanceledNoFailover pins that a
|
||||
// canceled client neither fails over nor evicts: the upstream never had a
|
||||
// chance to exhibit a fault.
|
||||
func TestHandleUpstreamTransportError_ClientCanceledNoFailover(t *testing.T) {
|
||||
repo := &transportTempUnschedRepoStub{}
|
||||
s := &GatewayService{accountRepo: repo}
|
||||
c := newTransportErrorTestGin(t)
|
||||
account := &Account{ID: 149, Name: "acc", Platform: PlatformAnthropic}
|
||||
|
||||
inErr := context.Canceled
|
||||
err := s.handleUpstreamTransportError(context.Background(), c, account, inErr, OpsUpstreamErrorEvent{})
|
||||
|
||||
var failoverErr *UpstreamFailoverError
|
||||
if errors.As(err, &failoverErr) {
|
||||
t.Fatal("canceled client must not fail over to another account")
|
||||
}
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("err = %v, want context.Canceled passthrough", err)
|
||||
}
|
||||
if repo.calls != 0 {
|
||||
t.Fatalf("SetTempUnschedulable called %d times on client cancel, want 0", repo.calls)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHandleUpstreamTransportError_UpstreamDeadlineStillFailsOver pins that an
|
||||
// upstream-side timeout (request context still alive) is treated as a
|
||||
// transient fault: fail over, no eviction.
|
||||
func TestHandleUpstreamTransportError_UpstreamDeadlineStillFailsOver(t *testing.T) {
|
||||
repo := &transportTempUnschedRepoStub{}
|
||||
s := &GatewayService{accountRepo: repo}
|
||||
c := newTransportErrorTestGin(t)
|
||||
account := &Account{ID: 149, Name: "acc", Platform: PlatformAnthropic}
|
||||
|
||||
err := s.handleUpstreamTransportError(context.Background(), c, account,
|
||||
context.DeadlineExceeded, OpsUpstreamErrorEvent{})
|
||||
|
||||
var failoverErr *UpstreamFailoverError
|
||||
if !errors.As(err, &failoverErr) {
|
||||
t.Fatalf("upstream deadline with live request context must fail over, got %T: %v", err, err)
|
||||
}
|
||||
if repo.calls != 0 {
|
||||
t.Fatalf("SetTempUnschedulable called %d times for upstream deadline, want 0", repo.calls)
|
||||
}
|
||||
}
|
||||
@@ -24,10 +24,10 @@ const openAITransportErrorTempUnschedDuration = 10 * time.Minute
|
||||
// ultimately exhausted.
|
||||
var openAITransportFailoverBody = []byte(`{"error":{"type":"upstream_error","message":"Upstream request failed"}}`)
|
||||
|
||||
// openAITransportErrorClass describes how to react to a transport-level upstream
|
||||
// upstreamTransportErrorClass describes how to react to a transport-level upstream
|
||||
// failure — i.e. the HTTP round-trip never completed (proxy / DNS / TCP / TLS
|
||||
// error, no HTTP status code received).
|
||||
type openAITransportErrorClass struct {
|
||||
type upstreamTransportErrorClass struct {
|
||||
// Persistent marks failures where retrying the same proxy/account is
|
||||
// pointless: expired or rejected proxy credentials, a dead proxy endpoint,
|
||||
// or DNS/routing failure. Such accounts should be temporarily unscheduled
|
||||
@@ -35,12 +35,12 @@ type openAITransportErrorClass struct {
|
||||
Persistent bool
|
||||
}
|
||||
|
||||
// openAIPersistentTransportErrorMarkers are substrings (matched case-insensitively
|
||||
// persistentUpstreamTransportErrorMarkers are substrings (matched case-insensitively
|
||||
// against the raw transport error) that indicate a durable proxy/network fault.
|
||||
// Matched signals are intentionally specific failure *reasons*, not the operation
|
||||
// (e.g. we match "connection refused", not "proxyconnect") so that a transient
|
||||
// failure of the same operation (a proxy timeout) is NOT misclassified as durable.
|
||||
var openAIPersistentTransportErrorMarkers = []string{
|
||||
var persistentUpstreamTransportErrorMarkers = []string{
|
||||
"authentication failed", // SOCKS5 RFC1929 / proxy credentials rejected (expired account)
|
||||
"proxy authentication required", // HTTP proxy 407
|
||||
"connection refused", // proxy/upstream endpoint down
|
||||
@@ -49,7 +49,7 @@ var openAIPersistentTransportErrorMarkers = []string{
|
||||
"no such host", // DNS resolution failure (bad/expired proxy hostname)
|
||||
}
|
||||
|
||||
// classifyOpenAITransportError decides whether a transport-level upstream error
|
||||
// classifyUpstreamTransportError decides whether a transport-level upstream error
|
||||
// is durable (Persistent — evict the account + alert) or a transient blip
|
||||
// (fail over to a healthy account but keep this one schedulable).
|
||||
//
|
||||
@@ -65,30 +65,30 @@ var openAIPersistentTransportErrorMarkers = []string{
|
||||
// The network-layer string markers ("connection refused", "no route to host",
|
||||
// "network is unreachable", "no such host") are kept as a cross-platform safety
|
||||
// net even though the typed checks should cover them on modern Go+Linux.
|
||||
func classifyOpenAITransportError(err error) openAITransportErrorClass {
|
||||
func classifyUpstreamTransportError(err error) upstreamTransportErrorClass {
|
||||
if err == nil {
|
||||
return openAITransportErrorClass{}
|
||||
return upstreamTransportErrorClass{}
|
||||
}
|
||||
|
||||
// — Typed checks (preferred) ——————————————————————————————————————————————
|
||||
if errors.Is(err, syscall.ECONNREFUSED) ||
|
||||
errors.Is(err, syscall.EHOSTUNREACH) ||
|
||||
errors.Is(err, syscall.ENETUNREACH) {
|
||||
return openAITransportErrorClass{Persistent: true}
|
||||
return upstreamTransportErrorClass{Persistent: true}
|
||||
}
|
||||
var dnsErr *net.DNSError
|
||||
if errors.As(err, &dnsErr) && dnsErr.IsNotFound {
|
||||
return openAITransportErrorClass{Persistent: true}
|
||||
return upstreamTransportErrorClass{Persistent: true}
|
||||
}
|
||||
|
||||
// — String-marker fallback ————————————————————————————————————————————————
|
||||
msg := strings.ToLower(err.Error())
|
||||
for _, marker := range openAIPersistentTransportErrorMarkers {
|
||||
for _, marker := range persistentUpstreamTransportErrorMarkers {
|
||||
if strings.Contains(msg, marker) {
|
||||
return openAITransportErrorClass{Persistent: true}
|
||||
return upstreamTransportErrorClass{Persistent: true}
|
||||
}
|
||||
}
|
||||
return openAITransportErrorClass{}
|
||||
return upstreamTransportErrorClass{}
|
||||
}
|
||||
|
||||
// handleOpenAIUpstreamTransportError handles a transport-level upstream failure
|
||||
@@ -135,7 +135,7 @@ func (s *OpenAIGatewayService) handleOpenAIUpstreamTransportError(ctx context.Co
|
||||
return err
|
||||
}
|
||||
|
||||
if classifyOpenAITransportError(err).Persistent {
|
||||
if classifyUpstreamTransportError(err).Persistent {
|
||||
s.tempUnscheduleOpenAITransportError(ctx, account, safeErr)
|
||||
}
|
||||
|
||||
|
||||
@@ -11,13 +11,13 @@ import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestClassifyOpenAITransportError pins which transport-level upstream failures
|
||||
// TestClassifyUpstreamTransportError pins which transport-level upstream failures
|
||||
// are "persistent" (retrying the same proxy/account is pointless — evict + alert)
|
||||
// versus "transient" (a blip — fail over to a healthy account but do not evict).
|
||||
//
|
||||
// The motivating incident: a SOCKS5 proxy whose credentials expired returned
|
||||
// `username/password authentication failed`, yet the account kept being scheduled.
|
||||
func TestClassifyOpenAITransportError(t *testing.T) {
|
||||
func TestClassifyUpstreamTransportError(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
err error
|
||||
@@ -76,9 +76,9 @@ func TestClassifyOpenAITransportError(t *testing.T) {
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := classifyOpenAITransportError(tc.err).Persistent
|
||||
got := classifyUpstreamTransportError(tc.err).Persistent
|
||||
if got != tc.persistent {
|
||||
t.Fatalf("classifyOpenAITransportError(%q).Persistent = %v, want %v", errString(tc.err), got, tc.persistent)
|
||||
t.Fatalf("classifyUpstreamTransportError(%q).Persistent = %v, want %v", errString(tc.err), got, tc.persistent)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user