From 5aeb03018c1defc8d46e108a4a72fcc2b72ff4fe Mon Sep 17 00:00:00 2001 From: Bestony Date: Mon, 13 Jul 2026 16:29:18 +0800 Subject: [PATCH] fix(scheduler): cool down Codex plan-gated models per account OpenAI OAuth (ChatGPT) accounts deterministically reject plan-gated models with 400 "The 'X' model is not supported when using Codex with a ChatGPT account". Account selection had no capability filtering for this, so the scheduler kept picking the same account for the same model forever; every attempt burned an upstream call and surfaced to clients as a retryable 502, sustaining client retry storms. Treat this 400 like upstream model-not-found: mark the (account, model) pair via SetModelRateLimit (30min cooldown) so IsSchedulableForModelWithContext skips the account for that model during selection, and return true so the in-flight request fails over to another account through the existing UpstreamFailoverError path. --- .../internal/service/model_not_found_error.go | 24 ++++++ .../service/model_not_found_error_test.go | 48 ++++++++++++ backend/internal/service/ratelimit_service.go | 27 +++++-- .../ratelimit_service_model_not_found_test.go | 74 +++++++++++++++++++ 4 files changed, 168 insertions(+), 5 deletions(-) diff --git a/backend/internal/service/model_not_found_error.go b/backend/internal/service/model_not_found_error.go index 910a97d844..de4a004d1e 100644 --- a/backend/internal/service/model_not_found_error.go +++ b/backend/internal/service/model_not_found_error.go @@ -22,6 +22,30 @@ func isModelNotFoundError(statusCode int, body []byte) bool { return isUpstreamModelNotFoundError(statusCode, body) || statusCode == http.StatusNotFound } +// openAICodexPlanGatedModelPhrase matches the deterministic Codex 400 returned +// when a ChatGPT OAuth account's plan cannot serve the requested model, e.g. +// {"detail":"The 'gpt-5.6-sol' model is not supported when using Codex with a ChatGPT account."} +// The phrase is compared against the normalized body (lowercased, "_"/"-" +// folded to spaces), so it also matches the same message embedded in +// error.message-style payloads. +const openAICodexPlanGatedModelPhrase = "model is not supported when using codex" + +// isOpenAICodexPlanGatedModelError reports whether the upstream response is the +// deterministic Codex rejection of a plan-gated model on a ChatGPT account. +// Unlike transient failures, retrying the same account cannot succeed until the +// account's plan changes, so callers should treat it like model-not-found and +// cool the (account, model) pair down instead of re-selecting the account. +func isOpenAICodexPlanGatedModelError(statusCode int, body []byte) bool { + if statusCode != http.StatusBadRequest { + return false + } + normalized := normalizeModelNotFoundBody(body) + if normalized == "" { + return false + } + return strings.Contains(normalized, openAICodexPlanGatedModelPhrase) +} + func containsModelNotFoundKeyword(normalizedBody string) bool { if normalizedBody == "" { return false diff --git a/backend/internal/service/model_not_found_error_test.go b/backend/internal/service/model_not_found_error_test.go index a87340eb55..2f8c83466f 100644 --- a/backend/internal/service/model_not_found_error_test.go +++ b/backend/internal/service/model_not_found_error_test.go @@ -64,3 +64,51 @@ func TestAntigravityModelNotFoundKeepsBare404Fallback(t *testing.T) { t.Fatal("antigravity model-not-found helper should keep bare 404 fallback") } } + +func TestIsOpenAICodexPlanGatedModelError(t *testing.T) { + tests := []struct { + name string + statusCode int + body []byte + want bool + }{ + { + name: "400 codex plan gated detail payload", + statusCode: http.StatusBadRequest, + body: []byte(`{"detail":"The 'gpt-5.6-sol' model is not supported when using Codex with a ChatGPT account."}`), + want: true, + }, + { + name: "400 codex plan gated error message payload", + statusCode: http.StatusBadRequest, + body: []byte(`{"error":{"message":"The 'gpt-5.4' model is not supported when using Codex with a ChatGPT account."}}`), + want: true, + }, + { + name: "400 unrelated invalid request does not match", + statusCode: http.StatusBadRequest, + body: []byte(`{"error":{"message":"Invalid schema for response_format 'agentic_plan'"}}`), + want: false, + }, + { + name: "404 with plan gated message does not match", + statusCode: http.StatusNotFound, + body: []byte(`{"detail":"The 'gpt-5.6-sol' model is not supported when using Codex with a ChatGPT account."}`), + want: false, + }, + { + name: "400 empty body does not match", + statusCode: http.StatusBadRequest, + body: nil, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isOpenAICodexPlanGatedModelError(tt.statusCode, tt.body); got != tt.want { + t.Fatalf("isOpenAICodexPlanGatedModelError() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/backend/internal/service/ratelimit_service.go b/backend/internal/service/ratelimit_service.go index 100b240785..507e02d877 100644 --- a/backend/internal/service/ratelimit_service.go +++ b/backend/internal/service/ratelimit_service.go @@ -2006,9 +2006,19 @@ func parseOpenAIImageTryAgainCooldown(body []byte) time.Duration { const upstreamModelNotFoundCooldown = 30 * time.Minute const upstreamModelNotFoundReason = "upstream_404_model_not_found" +const upstreamCodexPlanGatedModelCooldown = 30 * time.Minute +const upstreamCodexPlanGatedModelReason = "upstream_400_codex_plan_gated_model" const tempUnschedBodyMaxBytes = 64 << 10 const tempUnschedMessageMaxBytes = 2048 +// HandleUpstreamModelNotFound marks the requested model as temporarily +// unavailable on the account when the upstream deterministically reports it +// cannot serve that model: a 404 model-not-found, or the Codex 400 rejecting a +// plan-gated model on a ChatGPT OAuth account. Returning true tells the caller +// to fail the current attempt over to another account; the scheduler skips the +// (account, model) pair via IsSchedulableForModelWithContext until the +// cooldown expires, instead of re-selecting an account that can never serve +// the model. func (s *RateLimitService) HandleUpstreamModelNotFound(ctx context.Context, account *Account, requestedModel string, statusCode int, responseBody []byte) bool { if s == nil || account == nil || s.accountRepo == nil { return false @@ -2016,19 +2026,26 @@ func (s *RateLimitService) HandleUpstreamModelNotFound(ctx context.Context, acco if !account.ShouldHandleErrorCode(statusCode) { return false } - if !isUpstreamModelNotFoundError(statusCode, responseBody) { + var cooldown time.Duration + var reason string + switch { + case isUpstreamModelNotFoundError(statusCode, responseBody): + cooldown, reason = upstreamModelNotFoundCooldown, upstreamModelNotFoundReason + case isOpenAIOAuthAccount(account) && isOpenAICodexPlanGatedModelError(statusCode, responseBody): + cooldown, reason = upstreamCodexPlanGatedModelCooldown, upstreamCodexPlanGatedModelReason + default: return false } modelKey := modelRateLimitKeyForUpstreamModelNotFound(ctx, account, requestedModel) if modelKey == "" { return false } - resetAt := time.Now().Add(upstreamModelNotFoundCooldown) - if err := s.accountRepo.SetModelRateLimit(ctx, account.ID, modelKey, resetAt, upstreamModelNotFoundReason); err != nil { - slog.Warn("upstream_model_not_found_set_model_rate_limit_failed", "account_id", account.ID, "model", modelKey, "error", err) + resetAt := time.Now().Add(cooldown) + if err := s.accountRepo.SetModelRateLimit(ctx, account.ID, modelKey, resetAt, reason); err != nil { + slog.Warn("upstream_model_not_found_set_model_rate_limit_failed", "account_id", account.ID, "model", modelKey, "reason", reason, "error", err) return true } - slog.Info("upstream_model_not_found_model_rate_limited", "account_id", account.ID, "model", modelKey, "reset_at", resetAt) + slog.Info("upstream_model_not_found_model_rate_limited", "account_id", account.ID, "model", modelKey, "reason", reason, "reset_at", resetAt) return true } diff --git a/backend/internal/service/ratelimit_service_model_not_found_test.go b/backend/internal/service/ratelimit_service_model_not_found_test.go index dfd18c5f69..51bd8a607e 100644 --- a/backend/internal/service/ratelimit_service_model_not_found_test.go +++ b/backend/internal/service/ratelimit_service_model_not_found_test.go @@ -125,3 +125,77 @@ func openAIModelNotFoundTempAccount() *Account { }, } } + +func TestRateLimitService_HandleUpstreamError_CodexPlanGatedModelUsesModelRateLimit(t *testing.T) { + repo := &modelNotFoundAccountRepoStub{} + svc := &RateLimitService{accountRepo: repo} + account := openAICodexPlanGatedOAuthAccount() + + handled := svc.HandleUpstreamError( + context.Background(), + account, + http.StatusBadRequest, + http.Header{}, + []byte(`{"detail":"The 'gpt-5.6-sol' model is not supported when using Codex with a ChatGPT account."}`), + "gpt-5.6-sol", + ) + + require.True(t, handled) + require.Zero(t, repo.tempCalls) + require.Len(t, repo.modelRateLimitCalls, 1) + call := repo.modelRateLimitCalls[0] + require.Equal(t, account.ID, call.accountID) + require.Equal(t, "gpt-5.6-sol", call.scope) + require.Equal(t, upstreamCodexPlanGatedModelReason, call.reason) + require.WithinDuration(t, time.Now().Add(upstreamCodexPlanGatedModelCooldown), call.resetAt, 5*time.Second) +} + +func TestRateLimitService_HandleUpstreamError_CodexPlanGatedModelRespectsModelMapping(t *testing.T) { + repo := &modelNotFoundAccountRepoStub{} + svc := &RateLimitService{accountRepo: repo} + account := openAICodexPlanGatedOAuthAccount() + account.Credentials["model_mapping"] = map[string]any{"gpt-5.6-sol": "gpt-5.6-sol-upstream"} + + handled := svc.HandleUpstreamError( + context.Background(), + account, + http.StatusBadRequest, + http.Header{}, + []byte(`{"detail":"The 'gpt-5.6-sol-upstream' model is not supported when using Codex with a ChatGPT account."}`), + "gpt-5.6-sol", + ) + + require.True(t, handled) + require.Len(t, repo.modelRateLimitCalls, 1) + require.Equal(t, "gpt-5.6-sol-upstream", repo.modelRateLimitCalls[0].scope) +} + +func TestRateLimitService_HandleUpstreamError_CodexPlanGatedModelIgnoresAPIKeyAccount(t *testing.T) { + repo := &modelNotFoundAccountRepoStub{} + svc := &RateLimitService{accountRepo: repo} + account := openAICodexPlanGatedOAuthAccount() + account.Type = AccountTypeAPIKey + + handled := svc.HandleUpstreamError( + context.Background(), + account, + http.StatusBadRequest, + http.Header{}, + []byte(`{"detail":"The 'gpt-5.6-sol' model is not supported when using Codex with a ChatGPT account."}`), + "gpt-5.6-sol", + ) + + require.False(t, handled) + require.Empty(t, repo.modelRateLimitCalls) +} + +func openAICodexPlanGatedOAuthAccount() *Account { + return &Account{ + ID: 202, + Platform: PlatformOpenAI, + Type: AccountTypeOAuth, + Status: StatusActive, + Schedulable: true, + Credentials: map[string]any{}, + } +}