mirror of
https://github.com/Wei-Shaw/sub2api.git
synced 2026-09-21 14:19:18 +08:00
fix(openai): /responses 传输层错误转 failover + 持久故障临时摘除账号
代理/网络等传输层失败（Do/DoWithTLS 返回 error、无 HTTP 状态码，例如 SOCKS5 代理凭据过期返回 "username/password authentication failed"）此前直接写 502 "Upstream request failed" 并返回普通 error：既不 failover 到健康账号， 也不摘除故障账号，导致同一坏账号被反复调度、大量用户持续收到 502。 - 新增 classifyOpenAITransportError：typed-error 优先 (ECONNREFUSED/EHOSTUNREACH/ENETUNREACH、*net.DNSError.IsNotFound) + 字符串兜底（SOCKS5 认证失败无 typed 形式）区分持久 vs 瞬时。 - 新增 handleOpenAIUpstreamTransportError：记录 ops 错误；对持久故障调用 SetTempUnschedulable(10min,DB) + 内存 BlockAccountScheduling 摘除账号并打 稳定 WARN 事件；统一返回 *UpstreamFailoverError 让 handler 切换到健康账号； context.Canceled（客户端断开）不 failover、不摘除。 - /responses 主路径、passthrough、chat-completions 回退三处统一接入共享 helper。 Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
DaydreamCoding
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
2a7c0fbb0f
commit
217f85999c
@@ -142,23 +142,10 @@ func (s *OpenAIGatewayService) forwardResponsesViaRawChatCompletions(
|
||||
}
|
||||
resp, err := s.httpUpstream.Do(upstreamReq, proxyURL, account.ID, account.Concurrency)
|
||||
if err != nil {
|
||||
safeErr := sanitizeUpstreamErrorMessage(err.Error())
|
||||
setOpsUpstreamError(c, 0, safeErr, "")
|
||||
appendOpsUpstreamError(c, OpsUpstreamErrorEvent{
|
||||
Platform: account.Platform,
|
||||
AccountID: account.ID,
|
||||
AccountName: account.Name,
|
||||
UpstreamStatusCode: 0,
|
||||
Kind: "request_error",
|
||||
Message: safeErr,
|
||||
})
|
||||
c.JSON(http.StatusBadGateway, gin.H{
|
||||
"error": gin.H{
|
||||
"type": "upstream_error",
|
||||
"message": "Upstream request failed",
|
||||
},
|
||||
})
|
||||
return nil, fmt.Errorf("upstream request failed: %s", safeErr)
|
||||
// Transport-level failure (proxy/DNS/TCP/TLS — no HTTP response). Convert to
|
||||
// a failover so the handler switches to a healthy account, and temporarily
|
||||
// unschedule the account on durable faults (e.g. rejected proxy credentials).
|
||||
return nil, s.handleOpenAIUpstreamTransportError(ctx, c, account, err, false)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
|
||||
@@ -2982,24 +2982,10 @@ func (s *OpenAIGatewayService) Forward(ctx context.Context, c *gin.Context, acco
|
||||
resp, err := s.httpUpstream.Do(upstreamReq, proxyURL, account.ID, account.Concurrency)
|
||||
SetOpsLatencyMs(c, OpsUpstreamLatencyMsKey, time.Since(upstreamStart).Milliseconds())
|
||||
if err != nil {
|
||||
// Ensure the client receives an error response (handlers assume Forward writes on non-failover errors).
|
||||
safeErr := sanitizeUpstreamErrorMessage(err.Error())
|
||||
setOpsUpstreamError(c, 0, safeErr, "")
|
||||
appendOpsUpstreamError(c, OpsUpstreamErrorEvent{
|
||||
Platform: account.Platform,
|
||||
AccountID: account.ID,
|
||||
AccountName: account.Name,
|
||||
UpstreamStatusCode: 0,
|
||||
Kind: "request_error",
|
||||
Message: safeErr,
|
||||
})
|
||||
c.JSON(http.StatusBadGateway, gin.H{
|
||||
"error": gin.H{
|
||||
"type": "upstream_error",
|
||||
"message": "Upstream request failed",
|
||||
},
|
||||
})
|
||||
return nil, fmt.Errorf("upstream request failed: %s", safeErr)
|
||||
// Transport-level failure (proxy/DNS/TCP/TLS — no HTTP response). Convert to
|
||||
// a failover so the handler switches to a healthy account, and temporarily
|
||||
// unschedule the account on durable faults (e.g. rejected proxy credentials).
|
||||
return nil, s.handleOpenAIUpstreamTransportError(ctx, c, account, err, false)
|
||||
}
|
||||
|
||||
// Handle error response
|
||||
@@ -3283,24 +3269,10 @@ func (s *OpenAIGatewayService) forwardOpenAIPassthrough(
|
||||
resp, err := s.httpUpstream.Do(upstreamReq, proxyURL, account.ID, account.Concurrency)
|
||||
SetOpsLatencyMs(c, OpsUpstreamLatencyMsKey, time.Since(upstreamStart).Milliseconds())
|
||||
if err != nil {
|
||||
safeErr := sanitizeUpstreamErrorMessage(err.Error())
|
||||
setOpsUpstreamError(c, 0, safeErr, "")
|
||||
appendOpsUpstreamError(c, OpsUpstreamErrorEvent{
|
||||
Platform: account.Platform,
|
||||
AccountID: account.ID,
|
||||
AccountName: account.Name,
|
||||
UpstreamStatusCode: 0,
|
||||
Passthrough: true,
|
||||
Kind: "request_error",
|
||||
Message: safeErr,
|
||||
})
|
||||
c.JSON(http.StatusBadGateway, gin.H{
|
||||
"error": gin.H{
|
||||
"type": "upstream_error",
|
||||
"message": "Upstream request failed",
|
||||
},
|
||||
})
|
||||
return nil, fmt.Errorf("upstream request failed: %s", safeErr)
|
||||
// Transport-level failure (proxy/DNS/TCP/TLS — no HTTP response). Convert to
|
||||
// a failover so the handler switches to a healthy account, and temporarily
|
||||
// unschedule the account on durable faults (e.g. rejected proxy credentials).
|
||||
return nil, s.handleOpenAIUpstreamTransportError(ctx, c, account, err, true)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
|
||||
@@ -0,0 +1,193 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/pkg/logger"
|
||||
"github.com/gin-gonic/gin"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
// openAITransportErrorTempUnschedDuration is how long an account is temporarily
|
||||
// unscheduled after a durable transport failure (matches tokenRefreshTempUnschedDuration).
|
||||
const openAITransportErrorTempUnschedDuration = 10 * time.Minute
|
||||
|
||||
// openAITransportFailoverBody is the OpenAI-format error body attached to the
|
||||
// failover error for a transport-level failure. Kept identical to the legacy
|
||||
// inline 502 body so the client-visible payload is unchanged if failover is
|
||||
// ultimately exhausted.
|
||||
var openAITransportFailoverBody = []byte(`{"error":{"type":"upstream_error","message":"Upstream request failed"}}`)
|
||||
|
||||
// openAITransportErrorClass describes how to react to a transport-level upstream
|
||||
// failure — i.e. the HTTP round-trip never completed (proxy / DNS / TCP / TLS
|
||||
// error, no HTTP status code received).
|
||||
type openAITransportErrorClass struct {
|
||||
// Persistent marks failures where retrying the same proxy/account is
|
||||
// pointless: expired or rejected proxy credentials, a dead proxy endpoint,
|
||||
// or DNS/routing failure. Such accounts should be temporarily unscheduled
|
||||
// (and alerted on) instead of being repeatedly scheduled into hard failures.
|
||||
Persistent bool
|
||||
}
|
||||
|
||||
// openAIPersistentTransportErrorMarkers are substrings (matched case-insensitively
|
||||
// against the raw transport error) that indicate a durable proxy/network fault.
|
||||
// Matched signals are intentionally specific failure *reasons*, not the operation
|
||||
// (e.g. we match "connection refused", not "proxyconnect") so that a transient
|
||||
// failure of the same operation (a proxy timeout) is NOT misclassified as durable.
|
||||
var openAIPersistentTransportErrorMarkers = []string{
|
||||
"authentication failed", // SOCKS5 RFC1929 / proxy credentials rejected (expired account)
|
||||
"proxy authentication required", // HTTP proxy 407
|
||||
"connection refused", // proxy/upstream endpoint down
|
||||
"no route to host",
|
||||
"network is unreachable",
|
||||
"no such host", // DNS resolution failure (bad/expired proxy hostname)
|
||||
}
|
||||
|
||||
// classifyOpenAITransportError decides whether a transport-level upstream error
|
||||
// is durable (Persistent — evict the account + alert) or a transient blip
|
||||
// (fail over to a healthy account but keep this one schedulable).
|
||||
//
|
||||
// Motivating incident: a SOCKS5 proxy whose subscription lapsed returned
|
||||
// `username/password authentication failed`; the account was nonetheless
|
||||
// rescheduled on every request, hard-failing users with 502s.
|
||||
//
|
||||
// Classification strategy (mirrors sanitizeStreamError in gateway_service.go):
|
||||
// 1. Typed-error checks first (syscall constants, *net.DNSError) — portable and
|
||||
// unambiguous.
|
||||
// 2. String-marker fallback for errors that have no typed form (e.g. the plain
|
||||
// string returned by golang.org/x/net/proxy for SOCKS5 credential rejection).
|
||||
// The network-layer string markers ("connection refused", "no route to host",
|
||||
// "network is unreachable", "no such host") are kept as a cross-platform safety
|
||||
// net even though the typed checks should cover them on modern Go+Linux.
|
||||
func classifyOpenAITransportError(err error) openAITransportErrorClass {
|
||||
if err == nil {
|
||||
return openAITransportErrorClass{}
|
||||
}
|
||||
|
||||
// — Typed checks (preferred) ——————————————————————————————————————————————
|
||||
if errors.Is(err, syscall.ECONNREFUSED) ||
|
||||
errors.Is(err, syscall.EHOSTUNREACH) ||
|
||||
errors.Is(err, syscall.ENETUNREACH) {
|
||||
return openAITransportErrorClass{Persistent: true}
|
||||
}
|
||||
var dnsErr *net.DNSError
|
||||
if errors.As(err, &dnsErr) && dnsErr.IsNotFound {
|
||||
return openAITransportErrorClass{Persistent: true}
|
||||
}
|
||||
|
||||
// — String-marker fallback ————————————————————————————————————————————————
|
||||
msg := strings.ToLower(err.Error())
|
||||
for _, marker := range openAIPersistentTransportErrorMarkers {
|
||||
if strings.Contains(msg, marker) {
|
||||
return openAITransportErrorClass{Persistent: true}
|
||||
}
|
||||
}
|
||||
return openAITransportErrorClass{}
|
||||
}
|
||||
|
||||
// handleOpenAIUpstreamTransportError handles a transport-level upstream failure
|
||||
// (Do/DoWithTLS returned a non-HTTP error: proxy/DNS/TCP/TLS). It:
|
||||
// 1. records the failure in Ops error logs (status 0, kind=request_error);
|
||||
// 2. for durable faults (expired/rejected proxy creds, dead proxy, DNS/routing)
|
||||
// temporarily unschedules the account (DB + in-memory) and logs a stable
|
||||
// warn event that alert rules can key on;
|
||||
// 3. returns an error that is *UpstreamFailoverError (so the handler fails over
|
||||
// to a healthy account) for all non-canceled errors, or a plain error for
|
||||
// context.Canceled (client gone — no failover, no eviction).
|
||||
//
|
||||
// It deliberately does NOT write to the response: the handler owns the response
|
||||
// (failover, or a protocol-correct error once failover is exhausted).
|
||||
//
|
||||
// passthrough tags the Ops error event for the OpenAI passthrough forward path.
|
||||
func (s *OpenAIGatewayService) handleOpenAIUpstreamTransportError(ctx context.Context, c *gin.Context, account *Account, err error, passthrough bool) error {
|
||||
safeErr := sanitizeUpstreamErrorMessage(err.Error())
|
||||
setOpsUpstreamError(c, 0, safeErr, "")
|
||||
appendOpsUpstreamError(c, OpsUpstreamErrorEvent{
|
||||
Platform: account.Platform,
|
||||
AccountID: account.ID,
|
||||
AccountName: account.Name,
|
||||
UpstreamStatusCode: 0,
|
||||
Passthrough: passthrough,
|
||||
Kind: "request_error",
|
||||
Message: safeErr,
|
||||
})
|
||||
|
||||
// Client disconnected: do NOT fail over to another account and do NOT evict
|
||||
// this one — the upstream never had a chance to exhibit a fault.
|
||||
if errors.Is(err, context.Canceled) {
|
||||
return err
|
||||
}
|
||||
|
||||
if classifyOpenAITransportError(err).Persistent {
|
||||
s.tempUnscheduleOpenAITransportError(ctx, account, safeErr)
|
||||
}
|
||||
|
||||
return &UpstreamFailoverError{
|
||||
StatusCode: http.StatusBadGateway,
|
||||
ResponseBody: openAITransportFailoverBody,
|
||||
}
|
||||
}
|
||||
|
||||
// tempUnscheduleOpenAITransportError marks an account temporarily unschedulable
|
||||
// after a durable transport failure, both persistently (DB, survives restart)
|
||||
// and in-memory (immediate scheduler effect before the DB/account cache propagates).
|
||||
//
|
||||
// Log semantics:
|
||||
// - "openai.account_temp_unscheduled_transport" — emitted ONLY after a
|
||||
// successful DB write (both in-memory + persisted).
|
||||
// - "openai.account_temp_unscheduled_transport_memory_only" — emitted when
|
||||
// accountRepo is nil (in-memory only; no persistence).
|
||||
// - "openai.account_temp_unscheduled_transport_failed" — DB write attempted
|
||||
// but returned an error.
|
||||
func (s *OpenAIGatewayService) tempUnscheduleOpenAITransportError(ctx context.Context, account *Account, safeErr string) {
|
||||
if s == nil || account == nil {
|
||||
return
|
||||
}
|
||||
until := time.Now().Add(openAITransportErrorTempUnschedDuration)
|
||||
reason := "upstream transport error (proxy/network): " + safeErr
|
||||
|
||||
// Immediate in-memory block (honoured by the scheduler at selection time),
|
||||
// effective even if the DB write below fails or the account cache lags.
|
||||
s.BlockAccountScheduling(account, until, "transport_error")
|
||||
|
||||
if s.accountRepo == nil {
|
||||
// No DB configured — block is in-memory only; emit a distinct event so
|
||||
// operators are not misled into thinking the block survived a restart.
|
||||
logger.L().With(zap.String("component", "service.openai_gateway")).Warn(
|
||||
"openai.account_temp_unscheduled_transport_memory_only",
|
||||
zap.Int64("account_id", account.ID),
|
||||
zap.String("account_name", account.Name),
|
||||
zap.String("platform", account.Platform),
|
||||
zap.Time("until", until),
|
||||
zap.String("reason", reason),
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
bgCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), openAIAccountStateUpdateTimeout)
|
||||
defer cancel()
|
||||
if err := s.accountRepo.SetTempUnschedulable(bgCtx, account.ID, until, reason); err != nil {
|
||||
logger.L().With(zap.String("component", "service.openai_gateway")).Warn(
|
||||
"openai.account_temp_unscheduled_transport_failed",
|
||||
zap.Int64("account_id", account.ID),
|
||||
zap.Error(err),
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
// DB write succeeded: both in-memory and persisted.
|
||||
logger.L().With(zap.String("component", "service.openai_gateway")).Warn(
|
||||
"openai.account_temp_unscheduled_transport",
|
||||
zap.Int64("account_id", account.ID),
|
||||
zap.String("account_name", account.Name),
|
||||
zap.String("platform", account.Platform),
|
||||
zap.Time("until", until),
|
||||
zap.String("reason", reason),
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
//go:build unit
|
||||
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// openaiTransportAccountRepoStub records SetTempUnschedulable calls. It embeds the
|
||||
// (nil) AccountRepository interface so any other method call would panic — the
|
||||
// helper under test must only touch SetTempUnschedulable. tempUnschedCall is shared
|
||||
// with antigravity_internal500_penalty_test.go (same package).
|
||||
type openaiTransportAccountRepoStub struct {
|
||||
AccountRepository
|
||||
tempUnschedCalls []tempUnschedCall
|
||||
}
|
||||
|
||||
func (r *openaiTransportAccountRepoStub) SetTempUnschedulable(_ context.Context, id int64, until time.Time, reason string) error {
|
||||
r.tempUnschedCalls = append(r.tempUnschedCalls, tempUnschedCall{accountID: id, until: until, reason: reason})
|
||||
return nil
|
||||
}
|
||||
|
||||
func newOpenAITransportErrTestContext() (*gin.Context, *httptest.ResponseRecorder) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
rec := httptest.NewRecorder()
|
||||
c, _ := gin.CreateTestContext(rec)
|
||||
c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", nil)
|
||||
return c, rec
|
||||
}
|
||||
|
||||
// A durable proxy/credential failure must (a) temporarily unschedule the account
|
||||
// so it stops being hammered, and (b) return a failover error so the handler
|
||||
// switches to a healthy account instead of writing a hard 502 itself.
|
||||
func TestHandleOpenAIUpstreamTransportError_PersistentEvictsAndFailsOver(t *testing.T) {
|
||||
repo := &openaiTransportAccountRepoStub{}
|
||||
svc := &OpenAIGatewayService{accountRepo: repo}
|
||||
account := &Account{ID: 4627, Name: "proxy-expired", Platform: PlatformOpenAI}
|
||||
c, rec := newOpenAITransportErrTestContext()
|
||||
|
||||
before := time.Now()
|
||||
retErr := svc.handleOpenAIUpstreamTransportError(context.Background(), c, account,
|
||||
errors.New(`Post "https://chatgpt.com/backend-api/codex/responses": socks connect tcp 85.255.176.68:12324->chatgpt.com:443: username/password authentication failed`), false)
|
||||
after := time.Now()
|
||||
|
||||
// Failover error (handler will switch accounts), not a direct response.
|
||||
var fo *UpstreamFailoverError
|
||||
require.True(t, errors.As(retErr, &fo), "persistent error must return *UpstreamFailoverError")
|
||||
require.Equal(t, http.StatusBadGateway, fo.StatusCode)
|
||||
|
||||
// Persistent → account temporarily unscheduled for ~10min, reason carries cause.
|
||||
require.Len(t, repo.tempUnschedCalls, 1)
|
||||
require.Equal(t, int64(4627), repo.tempUnschedCalls[0].accountID)
|
||||
require.Contains(t, repo.tempUnschedCalls[0].reason, "authentication failed")
|
||||
require.True(t, repo.tempUnschedCalls[0].until.After(before.Add(openAITransportErrorTempUnschedDuration-time.Second)))
|
||||
require.True(t, repo.tempUnschedCalls[0].until.Before(after.Add(openAITransportErrorTempUnschedDuration+time.Second)))
|
||||
|
||||
// Immediate in-memory effect so subsequent requests skip it before DB/cache catches up.
|
||||
require.True(t, svc.isOpenAIAccountRuntimeBlocked(account))
|
||||
|
||||
// Must NOT write a response body — the handler owns the (failover) response.
|
||||
require.Equal(t, 0, rec.Body.Len())
|
||||
}
|
||||
|
||||
// A transient blip should fail over but must NOT evict the account.
|
||||
func TestHandleOpenAIUpstreamTransportError_TransientFailsOverWithoutEviction(t *testing.T) {
|
||||
repo := &openaiTransportAccountRepoStub{}
|
||||
svc := &OpenAIGatewayService{accountRepo: repo}
|
||||
account := &Account{ID: 99, Name: "flaky", Platform: PlatformOpenAI}
|
||||
c, rec := newOpenAITransportErrTestContext()
|
||||
|
||||
err := svc.handleOpenAIUpstreamTransportError(context.Background(), c, account,
|
||||
errors.New(`Post "https://chatgpt.com/...": context deadline exceeded (Client.Timeout exceeded while awaiting headers)`), false)
|
||||
|
||||
var fo *UpstreamFailoverError
|
||||
require.True(t, errors.As(err, &fo), "transient error must return *UpstreamFailoverError")
|
||||
require.Equal(t, http.StatusBadGateway, fo.StatusCode)
|
||||
|
||||
// Transient → do NOT evict.
|
||||
require.Empty(t, repo.tempUnschedCalls)
|
||||
require.False(t, svc.isOpenAIAccountRuntimeBlocked(account))
|
||||
require.Equal(t, 0, rec.Body.Len())
|
||||
}
|
||||
|
||||
// context.Canceled means the client disconnected — do NOT fail over to another
|
||||
// account and do NOT temporarily evict this one.
|
||||
func TestHandleOpenAIUpstreamTransportError_ContextCanceled_NoFailoverNoEviction(t *testing.T) {
|
||||
repo := &openaiTransportAccountRepoStub{}
|
||||
svc := &OpenAIGatewayService{accountRepo: repo}
|
||||
account := &Account{ID: 77, Name: "healthy", Platform: PlatformOpenAI}
|
||||
c, rec := newOpenAITransportErrTestContext()
|
||||
|
||||
err := svc.handleOpenAIUpstreamTransportError(context.Background(), c, account,
|
||||
context.Canceled, false)
|
||||
|
||||
// Must NOT be a failover error.
|
||||
var fo *UpstreamFailoverError
|
||||
require.False(t, errors.As(err, &fo), "context.Canceled must NOT return *UpstreamFailoverError")
|
||||
require.NotNil(t, err, "must return a non-nil error")
|
||||
|
||||
// Must NOT evict the account.
|
||||
require.Empty(t, repo.tempUnschedCalls, "context.Canceled must not trigger temp-unsched DB write")
|
||||
require.False(t, svc.isOpenAIAccountRuntimeBlocked(account), "context.Canceled must not block account in-memory")
|
||||
|
||||
// Must NOT write a response body.
|
||||
require.Equal(t, 0, rec.Body.Len())
|
||||
}
|
||||
|
||||
// context.Canceled wrapped inside another error must also avoid failover.
|
||||
func TestHandleOpenAIUpstreamTransportError_WrappedContextCanceled_NoFailover(t *testing.T) {
|
||||
repo := &openaiTransportAccountRepoStub{}
|
||||
svc := &OpenAIGatewayService{accountRepo: repo}
|
||||
account := &Account{ID: 78, Name: "healthy2", Platform: PlatformOpenAI}
|
||||
c, _ := newOpenAITransportErrTestContext()
|
||||
|
||||
wrapped := fmt.Errorf("http request failed: %w", context.Canceled)
|
||||
err := svc.handleOpenAIUpstreamTransportError(context.Background(), c, account, wrapped, false)
|
||||
|
||||
var fo *UpstreamFailoverError
|
||||
require.False(t, errors.As(err, &fo), "wrapped context.Canceled must NOT return *UpstreamFailoverError")
|
||||
require.Empty(t, repo.tempUnschedCalls)
|
||||
require.False(t, svc.isOpenAIAccountRuntimeBlocked(account))
|
||||
}
|
||||
|
||||
// When accountRepo is nil (no DB), in-memory block must still happen but the
|
||||
// success log "openai.account_temp_unscheduled_transport" must NOT fire (it
|
||||
// would be misleading: the account is only blocked in memory, not persisted).
|
||||
// We verify the in-memory block occurs and no DB call is made.
|
||||
func TestTempUnscheduleOpenAITransportError_NilAccountRepo_InMemoryBlockOnly(t *testing.T) {
|
||||
// nil accountRepo → no DB write.
|
||||
svc := &OpenAIGatewayService{accountRepo: nil}
|
||||
account := &Account{ID: 55, Name: "no-db", Platform: PlatformOpenAI}
|
||||
|
||||
svc.tempUnscheduleOpenAITransportError(context.Background(), account, "proxy refused")
|
||||
|
||||
// In-memory block must still happen.
|
||||
require.True(t, svc.isOpenAIAccountRuntimeBlocked(account),
|
||||
"in-memory block must apply even when accountRepo is nil")
|
||||
}
|
||||
|
||||
// context.DeadlineExceeded is NOT special-cased — a slow upstream is worth failing over.
|
||||
func TestHandleOpenAIUpstreamTransportError_DeadlineExceeded_StillFailsOver(t *testing.T) {
|
||||
repo := &openaiTransportAccountRepoStub{}
|
||||
svc := &OpenAIGatewayService{accountRepo: repo}
|
||||
account := &Account{ID: 79, Name: "slow", Platform: PlatformOpenAI}
|
||||
c, _ := newOpenAITransportErrTestContext()
|
||||
|
||||
err := svc.handleOpenAIUpstreamTransportError(context.Background(), c, account,
|
||||
context.DeadlineExceeded, false)
|
||||
|
||||
var fo *UpstreamFailoverError
|
||||
require.True(t, errors.As(err, &fo), "context.DeadlineExceeded must still return *UpstreamFailoverError")
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
//go:build unit
|
||||
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net"
|
||||
"os"
|
||||
"syscall"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestClassifyOpenAITransportError pins which transport-level upstream failures
|
||||
// are "persistent" (retrying the same proxy/account is pointless — evict + alert)
|
||||
// versus "transient" (a blip — fail over to a healthy account but do not evict).
|
||||
//
|
||||
// The motivating incident: a SOCKS5 proxy whose credentials expired returned
|
||||
// `username/password authentication failed`, yet the account kept being scheduled.
|
||||
func TestClassifyOpenAITransportError(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
err error
|
||||
persistent bool
|
||||
}{
|
||||
// Durable — config/credential/routing problems. Retrying same proxy won't help.
|
||||
{"socks5 proxy credential rejected", errors.New(`Post "https://chatgpt.com/backend-api/codex/responses": socks connect tcp 85.255.176.68:12324->chatgpt.com:443: username/password authentication failed`), true},
|
||||
{"proxy connection refused", errors.New(`proxyconnect tcp: dial tcp 1.2.3.4:1080: connect: connection refused`), true},
|
||||
{"no route to host", errors.New(`dial tcp 1.2.3.4:443: connect: no route to host`), true},
|
||||
{"dns resolution failure", errors.New(`dial tcp: lookup proxy.example.com: no such host`), true},
|
||||
{"network unreachable", errors.New(`dial tcp 1.2.3.4:443: connect: network is unreachable`), true},
|
||||
|
||||
// Transient — a temporary blip. Fail over, but do NOT evict the account.
|
||||
{"client timeout", errors.New(`Post "https://chatgpt.com/...": context deadline exceeded (Client.Timeout exceeded while awaiting headers)`), false},
|
||||
{"i/o timeout", errors.New(`dial tcp 1.2.3.4:443: i/o timeout`), false},
|
||||
{"connection reset by peer", errors.New(`read tcp 10.0.0.1:5->2.2.2.2:443: read: connection reset by peer`), false},
|
||||
{"unexpected eof", errors.New(`unexpected EOF`), false},
|
||||
{"broken pipe", errors.New(`write tcp 10.0.0.1:5->2.2.2.2:443: write: broken pipe`), false},
|
||||
|
||||
{"nil error", nil, false},
|
||||
|
||||
// ── Typed-error cases ──────────────────────────────────────────────
|
||||
// ECONNREFUSED wrapped in the canonical net.OpError shape Go produces.
|
||||
{
|
||||
"ECONNREFUSED via net.OpError",
|
||||
&net.OpError{
|
||||
Op: "dial",
|
||||
Net: "tcp",
|
||||
Err: &os.SyscallError{Syscall: "connect", Err: syscall.ECONNREFUSED},
|
||||
},
|
||||
true,
|
||||
},
|
||||
// Bare syscall error (errors.Is traverses the chain).
|
||||
{"ECONNREFUSED bare", syscall.ECONNREFUSED, true},
|
||||
{"EHOSTUNREACH bare", syscall.EHOSTUNREACH, true},
|
||||
{"ENETUNREACH bare", syscall.ENETUNREACH, true},
|
||||
|
||||
// *net.DNSError with IsNotFound — permanent DNS lookup failure.
|
||||
{
|
||||
"DNS not found (IsNotFound=true)",
|
||||
&net.DNSError{Err: "no such host", Name: "proxy.example.com", IsNotFound: true},
|
||||
true,
|
||||
},
|
||||
// *net.DNSError with IsNotFound=false — transient DNS timeout (not persistent).
|
||||
{
|
||||
"DNS timeout (IsNotFound=false)",
|
||||
&net.DNSError{Err: "i/o timeout", Name: "proxy.example.com", IsTimeout: true},
|
||||
false,
|
||||
},
|
||||
|
||||
// context.Canceled — client gone; NOT classified as persistent.
|
||||
{"context.Canceled", context.Canceled, false},
|
||||
// context.DeadlineExceeded — slow upstream; NOT persistent.
|
||||
{"context.DeadlineExceeded", context.DeadlineExceeded, false},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got := classifyOpenAITransportError(tc.err).Persistent
|
||||
if got != tc.persistent {
|
||||
t.Fatalf("classifyOpenAITransportError(%q).Persistent = %v, want %v", errString(tc.err), got, tc.persistent)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func errString(err error) string {
|
||||
if err == nil {
|
||||
return "<nil>"
|
||||
}
|
||||
return err.Error()
|
||||
}
|
||||
Reference in New Issue
Block a user