mirror of
https://github.com/Wei-Shaw/sub2api.git
synced 2026-09-21 05:45:26 +08:00
339 lines
12 KiB
Go
339 lines
12 KiB
Go
package service
|
|
|
|
import (
|
|
"context"
|
|
"log/slog"
|
|
"net/http"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
const (
|
|
openAIAccountStateUpdateTimeout = 5 * time.Second
|
|
openAIOAuth429FallbackCooldown = 5 * time.Second
|
|
openAIStopSchedulingBridgeCooldown = 2 * time.Minute
|
|
openAIOAuth429StormWindow = 10 * time.Second
|
|
openAIOAuth429StormThreshold = 20
|
|
openAIOAuth429StormMaxAccountSwitches = 1
|
|
)
|
|
|
|
// OpenAIOAuth429FailoverState tracks the request-local follow-up budget after
|
|
// the first Grok OAuth 429. Once that 429 occurs, exactly one different account
|
|
// may be attempted; any failure from that follow-up account ends failover.
|
|
type OpenAIOAuth429FailoverState struct {
|
|
grokOAuth429FollowupPending bool
|
|
}
|
|
|
|
func openAIAccountStateContext(ctx context.Context) (context.Context, context.CancelFunc) {
|
|
base := context.Background()
|
|
if ctx != nil {
|
|
base = context.WithoutCancel(ctx)
|
|
}
|
|
return context.WithTimeout(base, openAIAccountStateUpdateTimeout)
|
|
}
|
|
|
|
func isOpenAIOAuthAccount(account *Account) bool {
|
|
return account != nil && account.Platform == PlatformOpenAI && account.Type == AccountTypeOAuth
|
|
}
|
|
|
|
func isGrokOAuthAccount(account *Account) bool {
|
|
return account != nil && account.Platform == PlatformGrok && account.Type == AccountTypeOAuth
|
|
}
|
|
|
|
func isOpenAIAccount(account *Account) bool {
|
|
return account != nil && (account.Platform == PlatformOpenAI || account.Platform == PlatformGrok)
|
|
}
|
|
|
|
// handleOpenAIAccountUpstreamError expects canonicalModel to be the model used
|
|
// for scheduling after applying account mapping exactly once.
|
|
func (s *OpenAIGatewayService) handleOpenAIAccountUpstreamError(ctx context.Context, account *Account, statusCode int, headers http.Header, responseBody []byte, canonicalModel ...string) bool {
|
|
stateCtx, cancel := openAIAccountStateContext(ctx)
|
|
defer cancel()
|
|
|
|
if account != nil && account.Platform == PlatformOpenAI && isOpenAIContextWindowError("", responseBody) {
|
|
return false
|
|
}
|
|
|
|
if isOpenAIImageRateLimitError(statusCode, responseBody) {
|
|
if s != nil && s.rateLimitService != nil {
|
|
_ = s.rateLimitService.HandleOpenAIImageRateLimit(stateCtx, account, statusCode, headers, responseBody)
|
|
}
|
|
return false
|
|
}
|
|
|
|
if statusCode == http.StatusTooManyRequests {
|
|
s.markOpenAIOAuth429RateLimited(stateCtx, account, headers, responseBody)
|
|
}
|
|
if s == nil || account == nil || s.rateLimitService == nil {
|
|
return false
|
|
}
|
|
if len(canonicalModel) > 0 && s.rateLimitService.HandleUpstreamModelNotFound(stateCtx, account, canonicalModel[0], statusCode, responseBody) {
|
|
return true
|
|
}
|
|
shouldDisable := s.rateLimitService.HandleUpstreamError(stateCtx, account, statusCode, headers, responseBody)
|
|
if shouldDisable {
|
|
s.BlockAccountScheduling(account, time.Time{}, "upstream_disable")
|
|
}
|
|
if !shouldDisable && account.Platform == PlatformOpenAI && account.Type == AccountTypeAPIKey && shouldCooldownOpenAITransientUpstreamError(statusCode, responseBody) {
|
|
model := ""
|
|
if len(canonicalModel) > 0 {
|
|
model = canonicalModel[0]
|
|
}
|
|
decision := s.recordOpenAIAccountModelTransientFailure(account, model, time.Now())
|
|
if decision.FailureStreak > 0 {
|
|
slog.Warn("openai_model_transient_state",
|
|
"account_id", account.ID,
|
|
"model", openAIAccountModelTransientModel(model),
|
|
"failure_streak", decision.FailureStreak,
|
|
"cooldown_ms", decision.Cooldown.Milliseconds(),
|
|
"block_scope", "account_model",
|
|
)
|
|
}
|
|
}
|
|
return shouldDisable
|
|
}
|
|
|
|
func shouldCooldownOpenAITransientUpstreamError(statusCode int, responseBody []byte) bool {
|
|
switch statusCode {
|
|
case http.StatusInternalServerError, http.StatusBadGateway, http.StatusServiceUnavailable, http.StatusGatewayTimeout, 520, 521, 522, 523, 524:
|
|
return true
|
|
case http.StatusBadRequest:
|
|
return isOpenAITransientProcessingError(statusCode, "", responseBody)
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func (s *OpenAIGatewayService) markOpenAIOAuth429RateLimited(ctx context.Context, account *Account, headers http.Header, responseBody []byte) {
|
|
if s == nil || !isOpenAIOAuthAccount(account) {
|
|
return
|
|
}
|
|
// Spark 影子：不按 /responses 429 的 global x-codex-* 信号做内存运行时熔断(同 handle429,外审第8轮 P1)。
|
|
// 同时避免把 spark 的 429 计入全局 429 storm 计数(recordOpenAIOAuth429),否则会误伤母账号 failover 决策。
|
|
if account.IsShadow() {
|
|
return
|
|
}
|
|
s.recordOpenAIOAuth429()
|
|
|
|
cooldownUntil := time.Now().Add(openAIOAuth429FallbackCooldown)
|
|
if s.rateLimitService != nil {
|
|
if resetAt := s.rateLimitService.calculateOpenAI429ResetTime(headers); resetAt != nil && resetAt.After(time.Now()) {
|
|
cooldownUntil = *resetAt
|
|
} else if resetUnix := parseOpenAIRateLimitResetTime(responseBody); resetUnix != nil {
|
|
if resetAt := time.Unix(*resetUnix, 0); resetAt.After(time.Now()) {
|
|
cooldownUntil = resetAt
|
|
}
|
|
} else if cooldown, ok := s.rateLimitService.get429FallbackCooldown(ctx, account); ok && cooldown > 0 {
|
|
cooldownUntil = time.Now().Add(cooldown)
|
|
}
|
|
}
|
|
s.BlockAccountScheduling(account, cooldownUntil, "429")
|
|
}
|
|
|
|
func (s *OpenAIGatewayService) BlockAccountScheduling(account *Account, until time.Time, reason string) {
|
|
if s == nil || !isOpenAIAccount(account) {
|
|
return
|
|
}
|
|
mu := s.openAIAccountRuntimeBlockLock(account.ID)
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
_, _ = s.blockAccountSchedulingLocked(account, until, reason)
|
|
}
|
|
|
|
func (s *OpenAIGatewayService) openAIAccountRuntimeBlockLock(accountID int64) *sync.Mutex {
|
|
actual, _ := s.openaiAccountRuntimeBlockLocks.LoadOrStore(accountID, &sync.Mutex{})
|
|
mu, ok := actual.(*sync.Mutex)
|
|
if !ok {
|
|
mu = &sync.Mutex{}
|
|
s.openaiAccountRuntimeBlockLocks.Store(accountID, mu)
|
|
}
|
|
return mu
|
|
}
|
|
|
|
func (s *OpenAIGatewayService) blockAccountSchedulingLocked(account *Account, until time.Time, _ string) (uint64, bool) {
|
|
generation := s.openaiAccountRuntimeBlockSequence.Add(1)
|
|
s.openaiAccountRuntimeBlockGeneration.Store(account.ID, generation)
|
|
now := time.Now()
|
|
blockUntil := until
|
|
if blockUntil.IsZero() || !blockUntil.After(now) {
|
|
blockUntil = now.Add(openAIStopSchedulingBridgeCooldown)
|
|
}
|
|
|
|
for {
|
|
current, loaded := s.openaiAccountRuntimeBlockUntil.Load(account.ID)
|
|
if !loaded {
|
|
actual, stored := s.openaiAccountRuntimeBlockUntil.LoadOrStore(account.ID, blockUntil)
|
|
if !stored {
|
|
return generation, true
|
|
}
|
|
current = actual
|
|
}
|
|
|
|
currentUntil, ok := current.(time.Time)
|
|
if !ok || currentUntil.IsZero() {
|
|
if s.openaiAccountRuntimeBlockUntil.CompareAndSwap(account.ID, current, blockUntil) {
|
|
return generation, true
|
|
}
|
|
continue
|
|
}
|
|
if !blockUntil.After(currentUntil) {
|
|
return generation, false
|
|
}
|
|
if s.openaiAccountRuntimeBlockUntil.CompareAndSwap(account.ID, current, blockUntil) {
|
|
return generation, true
|
|
}
|
|
}
|
|
}
|
|
|
|
func (s *OpenAIGatewayService) ClearAccountSchedulingBlock(accountID int64) {
|
|
if s == nil || accountID <= 0 {
|
|
return
|
|
}
|
|
mu := s.openAIAccountRuntimeBlockLock(accountID)
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
s.openaiAccountRuntimeBlockUntil.Delete(accountID)
|
|
s.openaiAccountRuntimeBlockGeneration.Store(accountID, s.openaiAccountRuntimeBlockSequence.Add(1))
|
|
}
|
|
|
|
func (s *OpenAIGatewayService) isOpenAIAccountRuntimeBlocked(account *Account) bool {
|
|
if s == nil || !isOpenAIAccount(account) {
|
|
return false
|
|
}
|
|
mu := s.openAIAccountRuntimeBlockLock(account.ID)
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
value, ok := s.openaiAccountRuntimeBlockUntil.Load(account.ID)
|
|
if !ok {
|
|
return false
|
|
}
|
|
cooldownUntil, ok := value.(time.Time)
|
|
if !ok || cooldownUntil.IsZero() {
|
|
s.openaiAccountRuntimeBlockUntil.Delete(account.ID)
|
|
s.openaiAccountRuntimeBlockGeneration.Store(account.ID, s.openaiAccountRuntimeBlockSequence.Add(1))
|
|
return false
|
|
}
|
|
if time.Now().Before(cooldownUntil) {
|
|
return true
|
|
}
|
|
s.openaiAccountRuntimeBlockUntil.Delete(account.ID)
|
|
s.openaiAccountRuntimeBlockGeneration.Store(account.ID, s.openaiAccountRuntimeBlockSequence.Add(1))
|
|
return false
|
|
}
|
|
|
|
func (s *OpenAIGatewayService) getOpenAIAccountModelTransientState() *openAIAccountModelTransientState {
|
|
if s == nil {
|
|
return nil
|
|
}
|
|
s.openaiModelTransientOnce.Do(func() {
|
|
if s.openaiModelTransient == nil {
|
|
s.openaiModelTransient = newOpenAIAccountModelTransientState(openAIModelTransientDefaultMax)
|
|
}
|
|
})
|
|
return s.openaiModelTransient
|
|
}
|
|
|
|
func canonicalOpenAIAccountSchedulingModel(account *Account, requestedModel string) string {
|
|
model := strings.TrimSpace(requestedModel)
|
|
if account == nil || model == "" {
|
|
return model
|
|
}
|
|
if mapped := strings.TrimSpace(account.GetMappedModel(model)); mapped != "" {
|
|
return mapped
|
|
}
|
|
return model
|
|
}
|
|
|
|
func openAIAccountModelTransientModel(canonicalModel string) string {
|
|
return normalizeOpenAIAccountModelTransientModel(canonicalModel)
|
|
}
|
|
|
|
func (s *OpenAIGatewayService) recordOpenAIAccountModelTransientFailure(account *Account, canonicalModel string, now time.Time) openAIAccountModelTransientDecision {
|
|
if s == nil || account == nil {
|
|
return openAIAccountModelTransientDecision{}
|
|
}
|
|
state := s.getOpenAIAccountModelTransientState()
|
|
if state == nil {
|
|
return openAIAccountModelTransientDecision{}
|
|
}
|
|
return state.recordFailure(account.ID, openAIAccountModelTransientModel(canonicalModel), now)
|
|
}
|
|
|
|
func (s *OpenAIGatewayService) clearOpenAIAccountModelTransientState(accountID int64, model string) {
|
|
state := s.getOpenAIAccountModelTransientState()
|
|
if state == nil {
|
|
return
|
|
}
|
|
state.recordSuccess(accountID, model)
|
|
}
|
|
|
|
func (s *OpenAIGatewayService) isOpenAIAccountModelRuntimeBlocked(account *Account, requestedModel string) bool {
|
|
if s == nil || account == nil {
|
|
return false
|
|
}
|
|
state := s.getOpenAIAccountModelTransientState()
|
|
if state == nil {
|
|
return false
|
|
}
|
|
canonicalModel := canonicalOpenAIAccountSchedulingModel(account, requestedModel)
|
|
return state.isBlocked(account.ID, openAIAccountModelTransientModel(canonicalModel), time.Now())
|
|
}
|
|
|
|
func (s *OpenAIGatewayService) isOpenAIAccountRequestRuntimeBlocked(account *Account, requestedModel string) bool {
|
|
return s != nil && (s.isOpenAIAccountRuntimeBlocked(account) || s.isOpenAIAccountModelRuntimeBlocked(account, requestedModel))
|
|
}
|
|
|
|
func (s *OpenAIGatewayService) recordOpenAIOAuth429() {
|
|
if s == nil {
|
|
return
|
|
}
|
|
now := time.Now()
|
|
windowStart := s.openaiOAuth429WindowStartUnixNano.Load()
|
|
if windowStart == 0 || now.Sub(time.Unix(0, windowStart)) >= openAIOAuth429StormWindow {
|
|
if s.openaiOAuth429WindowStartUnixNano.CompareAndSwap(windowStart, now.UnixNano()) {
|
|
s.openaiOAuth429WindowCount.Store(1)
|
|
return
|
|
}
|
|
}
|
|
s.openaiOAuth429WindowCount.Add(1)
|
|
}
|
|
|
|
func (s *OpenAIGatewayService) isOpenAIOAuth429Storm() bool {
|
|
if s == nil {
|
|
return false
|
|
}
|
|
windowStart := s.openaiOAuth429WindowStartUnixNano.Load()
|
|
if windowStart == 0 || time.Since(time.Unix(0, windowStart)) >= openAIOAuth429StormWindow {
|
|
return false
|
|
}
|
|
return s.openaiOAuth429WindowCount.Load() >= openAIOAuth429StormThreshold
|
|
}
|
|
|
|
func (s *OpenAIGatewayService) ShouldStopOpenAIOAuth429Failover(account *Account, statusCode int, failedSwitches int, state *OpenAIOAuth429FailoverState) bool {
|
|
if failedSwitches < openAIOAuth429StormMaxAccountSwitches {
|
|
return false
|
|
}
|
|
if state != nil && state.grokOAuth429FollowupPending {
|
|
// The follow-up budget was armed by a Grok OAuth 429. Consume it on
|
|
// any failing follow-up account, even if a mixed pool selected an API-key
|
|
// account next.
|
|
return true
|
|
}
|
|
if isGrokOAuthAccount(account) {
|
|
if state == nil {
|
|
// Preserve the old threshold for callers that have not adopted the
|
|
// request-local state contract yet.
|
|
return statusCode == http.StatusTooManyRequests && failedSwitches >= 2
|
|
}
|
|
if statusCode == http.StatusTooManyRequests {
|
|
state.grokOAuth429FollowupPending = true
|
|
}
|
|
return false
|
|
}
|
|
if statusCode != http.StatusTooManyRequests || !isOpenAIOAuthAccount(account) {
|
|
return false
|
|
}
|
|
return s.isOpenAIOAuth429Storm()
|
|
}
|