diff --git a/backend/internal/service/openai_codex_models_service.go b/backend/internal/service/openai_codex_models_service.go index 1d9356441c..04f29c80a9 100644 --- a/backend/internal/service/openai_codex_models_service.go +++ b/backend/internal/service/openai_codex_models_service.go @@ -42,8 +42,10 @@ type CodexModelsManifest struct { } type codexModelsManifestUpstreamError struct { - err error - retryable bool + err error + retryable bool + statusCode int + body []byte } func (e *codexModelsManifestUpstreamError) Error() string { return e.err.Error() } @@ -314,9 +316,35 @@ func (s *OpenAIGatewayService) FetchCodexModelsManifest(ctx context.Context, acc if useAPIKeyUpstream { return s.fetchCachedAPIKeyCodexModelsManifest(ctx, request, ifNoneMatch) } + manifest, fetchErr := s.fetchCodexModelsManifestUpstream(ctx, request, ifNoneMatch) + if !credAccount.IsOpenAIAgentIdentity() || !isAgentIdentityTaskInvalidCodexModelsError(fetchErr) { + return manifest, fetchErr + } + expectedTaskID := strings.TrimSpace(credAccount.GetCredential("task_id")) + if recoverErr := s.recoverAgentIdentityTask(ctx, credAccount, expectedTaskID); recoverErr != nil { + return nil, infraerrors.Newf(http.StatusBadGateway, "OPENAI_CODEX_MODELS_AUTH_FAILED", "agent identity task recovery failed: %v", recoverErr) + } + authHeaders, authErr := s.buildOpenAIAuthenticationHeaders(ctx, credAccount, "") + if authErr != nil { + return nil, infraerrors.Newf(http.StatusBadGateway, "OPENAI_CODEX_MODELS_AUTH_FAILED", "build Codex models authentication after task recovery: %v", authErr) + } + request.headers.Del("Authorization") + request.headers.Del("ChatGPT-Account-ID") + for key, values := range authHeaders { + for _, value := range values { + request.headers.Add(key, value) + } + } + setOpenAIChatGPTAccountHeaders(request.headers, credAccount) return s.fetchCodexModelsManifestUpstream(ctx, request, ifNoneMatch) } +func isAgentIdentityTaskInvalidCodexModelsError(err error) bool { + var upstreamErr *codexModelsManifestUpstreamError + return errors.As(err, &upstreamErr) && + isAgentIdentityTaskInvalidHTTPResponse(upstreamErr.statusCode, upstreamErr.body) +} + func (s *OpenAIGatewayService) fetchCachedAPIKeyCodexModelsManifest(ctx context.Context, request codexModelsManifestRequest, ifNoneMatch string) (*CodexModelsManifest, error) { if err := ctx.Err(); err != nil { return nil, err @@ -415,7 +443,9 @@ func (s *OpenAIGatewayService) fetchCodexModelsManifestUpstream(ctx context.Cont message = resp.Status } return nil, &codexModelsManifestUpstreamError{ - err: infraerrors.Newf(http.StatusBadGateway, "OPENAI_CODEX_MODELS_UPSTREAM_FAILED", "codex models manifest upstream error %d: %s", resp.StatusCode, message), + err: infraerrors.Newf(http.StatusBadGateway, "OPENAI_CODEX_MODELS_UPSTREAM_FAILED", "codex models manifest upstream error %d: %s", resp.StatusCode, message), + statusCode: resp.StatusCode, + body: body, retryable: resp.StatusCode == http.StatusTooManyRequests || (resp.StatusCode >= http.StatusInternalServerError && resp.StatusCode < 600), } diff --git a/backend/internal/service/openai_codex_models_service_test.go b/backend/internal/service/openai_codex_models_service_test.go index 3f4ec96a2f..66ed2c2b5b 100644 --- a/backend/internal/service/openai_codex_models_service_test.go +++ b/backend/internal/service/openai_codex_models_service_test.go @@ -17,6 +17,7 @@ import ( "github.com/Wei-Shaw/sub2api/internal/config" infraerrors "github.com/Wei-Shaw/sub2api/internal/pkg/errors" "github.com/Wei-Shaw/sub2api/internal/pkg/tlsfingerprint" + "github.com/stretchr/testify/require" "golang.org/x/net/http2" ) @@ -242,6 +243,60 @@ func TestFetchCodexModelsManifestAgentIdentityUsesAssertionWithoutOAuthToken(t * } } +func TestFetchCodexModelsManifestAgentIdentityRecoversInvalidTaskOnce(t *testing.T) { + key, privateKey := newTestAgentIdentityKey(t) + account := &Account{ + ID: 4, + Platform: PlatformOpenAI, + Type: AccountTypeOAuth, + Credentials: map[string]any{ + "auth_mode": OpenAIAuthModeAgentIdentity, + "agent_runtime_id": key.runtimeID, + "agent_private_key": privateKey, + "task_id": "task-models-old", + "chatgpt_account_id": "acc-agent-recovery", + }, + } + repo := &stubQuotaAccountRepo{accounts: map[int64]*Account{account.ID: account}} + modelsCalls := 0 + registerCalls := 0 + var assertions []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("content-type", "application/json") + if strings.Contains(r.URL.Path, "/task/register") { + registerCalls++ + _, _ = w.Write([]byte(`{"task_id":"task-models-new"}`)) + return + } + modelsCalls++ + assertions = append(assertions, r.Header.Get("Authorization")) + if modelsCalls == 1 { + w.WriteHeader(http.StatusUnauthorized) + _, _ = w.Write([]byte(`{"error":{"code":"invalid_task_id"}}`)) + return + } + _, _ = w.Write([]byte(`{"models":[]}`)) + })) + defer server.Close() + + originalModelsURL := chatgptCodexModelsURL + chatgptCodexModelsURL = server.URL + t.Cleanup(func() { chatgptCodexModelsURL = originalModelsURL }) + originalAuthBase := openAIAgentIdentityAuthAPIBaseURL + openAIAgentIdentityAuthAPIBaseURL = server.URL + t.Cleanup(func() { openAIAgentIdentityAuthAPIBaseURL = originalAuthBase }) + + s := &OpenAIGatewayService{accountRepo: repo} + manifest, err := s.FetchCodexModelsManifest(context.Background(), account, "0.137.0", "") + require.NoError(t, err) + require.Equal(t, `{"models":[]}`, string(manifest.Body)) + require.Equal(t, 2, modelsCalls) + require.Equal(t, 1, registerCalls) + require.Len(t, assertions, 2) + require.Equal(t, "task-models-old", decodeAgentAssertionTask(t, assertions[0])) + require.Equal(t, "task-models-new", decodeAgentAssertionTask(t, assertions[1])) +} + func TestFetchCodexModelsManifestDefaultClientVersion(t *testing.T) { var gotClientVersion string server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/backend/internal/service/openai_quota_service.go b/backend/internal/service/openai_quota_service.go index a94d5fe3f8..ffda0f4f87 100644 --- a/backend/internal/service/openai_quota_service.go +++ b/backend/internal/service/openai_quota_service.go @@ -22,8 +22,6 @@ import ( // errors.Is still matches it by identity since ResetCredit returns this var. var ErrSparkShadowResetNotSupported = infraerrors.New(http.StatusConflict, "SPARK_SHADOW_RESET_NOT_SUPPORTED", "spark shadow account does not support credit reset; reset the parent account") -var ErrAgentIdentityResetNotSupported = infraerrors.New(http.StatusConflict, "AGENT_IDENTITY_RESET_NOT_SUPPORTED", "agent identity does not support rate-limit reset credit consumption") - // Endpoints used by the OpenAI/ChatGPT/Codex quota query and reset feature. const ( chatGPTUsageURL = "https://chatgpt.com/backend-api/wham/usage" @@ -260,9 +258,6 @@ func (s *OpenAIQuotaService) ResetCredit(ctx context.Context, accountID int64) ( if acc.IsShadow() { return nil, ErrSparkShadowResetNotSupported } - if acc.IsOpenAIAgentIdentity() { - return nil, ErrAgentIdentityResetNotSupported - } } accessToken, chatGPTAccountID, proxyURL, fedRAMP, err := s.prepareUpstreamCall(ctx, accountID) @@ -282,26 +277,38 @@ func (s *OpenAIQuotaService) ResetCredit(ctx context.Context, accountID int64) ( callCtx, cancel := context.WithTimeout(ctx, openaiQuotaUpstreamTimeout) defer cancel() + agentIdentity := s.isAgentIdentityAccount(ctx, accountID) + var payload OpenAIQuotaResetResult - headers, headerErr := s.buildCodexQuotaHeaders(callCtx, accountID, accessToken, chatGPTAccountID, fedRAMP) - if headerErr != nil { - return nil, infraerrors.Newf(http.StatusBadGateway, "OPENAI_QUOTA_AUTH_FAILED", "failed to build upstream authentication: %v", headerErr) - } - headers["content-type"] = "application/json" - resp, err := client.R(). - SetContext(callCtx). - SetHeaders(headers). - SetBody(map[string]string{"redeem_request_id": redeemRequestID}). - SetSuccessResult(&payload). - Post(chatGPTRateLimitResetURL) - if err != nil { - return nil, infraerrors.Newf(http.StatusBadGateway, "OPENAI_QUOTA_RESET_REQUEST_FAILED", "upstream request failed: %v", err) - } - if !resp.IsSuccessState() { - status := resp.StatusCode - body := truncate(s.redactQuotaErrorBody(callCtx, accountID, resp.String()), 240) - slog.Warn("openai_quota_reset_failed", "account_id", accountID, "status", status, "body", body) - return nil, infraerrors.Newf(mapUpstreamStatus(status), "OPENAI_QUOTA_RESET_UPSTREAM_ERROR", "upstream returned %d: %s", status, body) + for recovered := false; ; { + headers, headerErr := s.buildCodexQuotaHeaders(callCtx, accountID, accessToken, chatGPTAccountID, fedRAMP) + if headerErr != nil { + return nil, infraerrors.Newf(http.StatusBadGateway, "OPENAI_QUOTA_AUTH_FAILED", "failed to build upstream authentication: %v", headerErr) + } + headers["content-type"] = "application/json" + resp, err := client.R(). + SetContext(callCtx). + SetHeaders(headers). + SetBody(map[string]string{"redeem_request_id": redeemRequestID}). + SetSuccessResult(&payload). + Post(chatGPTRateLimitResetURL) + if err != nil { + return nil, infraerrors.Newf(http.StatusBadGateway, "OPENAI_QUOTA_RESET_REQUEST_FAILED", "upstream request failed: %v", err) + } + if !resp.IsSuccessState() { + if agentIdentity && !recovered && isAgentIdentityTaskInvalidHTTPResponse(resp.StatusCode, []byte(resp.String())) { + recovered = true + if err := s.recoverAgentIdentityTask(ctx, accountID); err != nil { + return nil, infraerrors.Newf(http.StatusBadGateway, "OPENAI_QUOTA_AUTH_FAILED", "agent identity task recovery failed: %v", err) + } + continue + } + status := resp.StatusCode + body := truncate(s.redactQuotaErrorBody(callCtx, accountID, resp.String()), 240) + slog.Warn("openai_quota_reset_failed", "account_id", accountID, "status", status, "body", body) + return nil, infraerrors.Newf(mapUpstreamStatus(status), "OPENAI_QUOTA_RESET_UPSTREAM_ERROR", "upstream returned %d: %s", status, body) + } + break } slog.Info("openai_quota_reset_success", diff --git a/backend/internal/service/openai_quota_spark_window_test.go b/backend/internal/service/openai_quota_spark_window_test.go index c521b149e9..acce031951 100644 --- a/backend/internal/service/openai_quota_spark_window_test.go +++ b/backend/internal/service/openai_quota_spark_window_test.go @@ -176,21 +176,65 @@ func TestResetCreditShadowRejected(t *testing.T) { "shadow ResetCredit 应映射为 409 Conflict 而非 500") } -func TestResetCreditAgentIdentityRejectedBeforeUpstream(t *testing.T) { +func TestResetCreditAgentIdentityUsesAssertionAndRecoversInvalidTaskOnce(t *testing.T) { + _, privateKey, err := ed25519.GenerateKey(rand.Reader) + require.NoError(t, err) + der, err := x509.MarshalPKCS8PrivateKey(privateKey) + require.NoError(t, err) account := &Account{ ID: 201, Platform: PlatformOpenAI, Type: AccountTypeOAuth, Credentials: map[string]any{ - "auth_mode": OpenAIAuthModeAgentIdentity, + "auth_mode": OpenAIAuthModeAgentIdentity, + "agent_runtime_id": "runtime-reset-recovery", + "agent_private_key": base64.StdEncoding.EncodeToString(der), + "task_id": "task-reset-old", + "chatgpt_account_id": "account-reset-recovery", }, } repo := &stubQuotaAccountRepo{accounts: map[int64]*Account{account.ID: account}} - svc := &OpenAIQuotaService{accountRepo: repo} + resetCalls := 0 + registerCalls := 0 + var assertions []string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("content-type", "application/json") + if strings.Contains(r.URL.Path, "/task/register") { + registerCalls++ + _, _ = w.Write([]byte(`{"task_id":"task-reset-new"}`)) + return + } + resetCalls++ + assertions = append(assertions, r.Header.Get("authorization")) + require.Equal(t, "account-reset-recovery", r.Header.Get("chatgpt-account-id")) + if resetCalls == 1 { + w.WriteHeader(http.StatusUnauthorized) + _, _ = w.Write([]byte(`{"error":{"code":"invalid_task_id"}}`)) + return + } + _, _ = w.Write([]byte(`{"code":"ok","windows_reset":2}`)) + })) + defer srv.Close() + oldBase := openAIAgentIdentityAuthAPIBaseURL + openAIAgentIdentityAuthAPIBaseURL = srv.URL + t.Cleanup(func() { openAIAgentIdentityAuthAPIBaseURL = oldBase }) - _, err := svc.ResetCredit(context.Background(), account.ID) - require.ErrorIs(t, err, ErrAgentIdentityResetNotSupported) - require.Equal(t, http.StatusConflict, infraerrors.Code(err)) + invalidator := &agentIdentityWSInvalidationRecorder{} + svc := NewOpenAIQuotaService(repo, nil, nil, newQuotaRedirectingFactory(srv)) + svc.agentIdentityWS = invalidator + + result, err := svc.ResetCredit(context.Background(), account.ID) + require.NoError(t, err) + require.Equal(t, "ok", result.Code) + require.Equal(t, 2, result.WindowsReset) + require.Equal(t, 2, resetCalls) + require.Equal(t, 1, registerCalls) + require.Len(t, assertions, 2) + require.True(t, strings.HasPrefix(assertions[0], "AgentAssertion ")) + require.True(t, strings.HasPrefix(assertions[1], "AgentAssertion ")) + require.NotEqual(t, assertions[0], assertions[1]) + require.Equal(t, "task-reset-new", account.GetCredential("task_id")) + require.Equal(t, []int64{account.ID}, invalidator.accountIDs) } // ── Part B: prepareUpstreamCall 影子 resolve ──────────────────────────────