fix(openai): 补齐身份失效恢复路径

This commit is contained in:
cat
2026-07-14 17:19:15 +08:00
parent 3fe7b4da78
commit 10aa88aab9
8 changed files with 378 additions and 48 deletions
@@ -590,8 +590,11 @@ func (s *AccountTestService) testOpenAIAccountConnection(c *gin.Context, account
payload := createOpenAITestPayload(testModelID, isOAuth)
payloadBytes, _ := json.Marshal(payload)
// Send test_start event
s.sendEvent(c, TestEvent{Type: "test_start", Model: testModelID})
// Send test_start event once. A task-invalid Agent Identity response may
// restart this probe after registering a replacement task.
if !agentIdentityTaskRecoveryWasTried(ctx) {
s.sendEvent(c, TestEvent{Type: "test_start", Model: testModelID})
}
req, err := http.NewRequestWithContext(ctx, "POST", apiURL, bytes.NewReader(payloadBytes))
if err != nil {
@@ -656,6 +659,14 @@ func (s *AccountTestService) testOpenAIAccountConnection(c *gin.Context, account
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
body = redactAgentIdentitySensitiveBodyForAccount(ctx, s.accountRepo, credentialAccount, body)
if !agentIdentityTaskRecoveryWasTried(ctx) && credentialAccount.IsOpenAIAgentIdentity() && isAgentIdentityTaskInvalidHTTPResponse(resp.StatusCode, body) {
expectedTaskID := credentialAccount.GetCredential("task_id")
if err := ensureAgentIdentityTaskForAccount(ctx, s.accountRepo, nil, &s.agentIdentityTaskMu, credentialAccount, expectedTaskID); err != nil {
return s.sendErrorAndEnd(c, fmt.Sprintf("Agent Identity task recovery failed: %s", err.Error()))
}
c.Request = c.Request.WithContext(markAgentIdentityTaskRecoveryTried(ctx))
return s.testOpenAIAccountConnection(c, account, modelID, prompt, mode)
}
if resp.StatusCode == http.StatusTooManyRequests {
s.reconcileOpenAI429State(ctx, account, resp.Header, body)
}
@@ -718,7 +729,9 @@ func (s *AccountTestService) testGrokAccountConnection(c *gin.Context, account *
return s.sendErrorAndEnd(c, "Failed to create Grok test payload")
}
s.sendEvent(c, TestEvent{Type: "test_start", Model: testModelID})
if !agentIdentityTaskRecoveryWasTried(ctx) {
s.sendEvent(c, TestEvent{Type: "test_start", Model: testModelID})
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, apiURL, bytes.NewReader(payloadBytes))
if err != nil {
@@ -869,7 +882,9 @@ func (s *AccountTestService) testOpenAICompactConnection(c *gin.Context, account
c.Writer.Flush()
payloadBytes, _ := json.Marshal(createOpenAICompactProbePayload(testModelID))
s.sendEvent(c, TestEvent{Type: "test_start", Model: testModelID})
if !agentIdentityTaskRecoveryWasTried(ctx) {
s.sendEvent(c, TestEvent{Type: "test_start", Model: testModelID})
}
req, err := http.NewRequestWithContext(ctx, "POST", apiURL, bytes.NewReader(payloadBytes))
if err != nil {
@@ -926,6 +941,14 @@ func (s *AccountTestService) testOpenAICompactConnection(c *gin.Context, account
body, _ := io.ReadAll(io.LimitReader(resp.Body, 2<<20))
body = redactAgentIdentitySensitiveBodyForAccount(ctx, s.accountRepo, credentialAccount, body)
if !agentIdentityTaskRecoveryWasTried(ctx) && credentialAccount.IsOpenAIAgentIdentity() && isAgentIdentityTaskInvalidHTTPResponse(resp.StatusCode, body) {
expectedTaskID := credentialAccount.GetCredential("task_id")
if err := ensureAgentIdentityTaskForAccount(ctx, s.accountRepo, nil, &s.agentIdentityTaskMu, credentialAccount, expectedTaskID); err != nil {
return s.sendErrorAndEnd(c, fmt.Sprintf("Agent Identity task recovery failed: %s", err.Error()))
}
c.Request = c.Request.WithContext(markAgentIdentityTaskRecoveryTried(ctx))
return s.testOpenAICompactConnection(c, account, testModelID)
}
if s.accountRepo != nil {
updates := buildOpenAICompactProbeExtraUpdates(resp, body, nil, time.Now())
@@ -317,13 +317,26 @@ func isAgentIdentityTaskInvalidHTTPResponse(statusCode int, body []byte) bool {
return false
}
lower := strings.ToLower(string(body))
compact := strings.NewReplacer(" ", "", "\t", "", "\r", "", "\n", "").Replace(lower)
for _, marker := range []string{
"invalid task",
"task_id",
"task id",
"task_not_found",
"task_expired",
"unknown task",
`"code":"invalid_task_id"`,
`"code":"task_not_found"`,
`"code":"task_expired"`,
`"error":"invalid_task_id"`,
} {
if strings.Contains(compact, marker) {
return true
}
}
for _, marker := range []string{
"invalid task_id",
"invalid task id",
"task_id is invalid",
"task id is invalid",
"task not found",
"task expired",
"unknown task_id",
"unknown task id",
} {
if strings.Contains(lower, marker) {
return true
@@ -332,6 +345,17 @@ func isAgentIdentityTaskInvalidHTTPResponse(statusCode int, body []byte) bool {
return false
}
type agentIdentityTaskRecoveryContextKey struct{}
func markAgentIdentityTaskRecoveryTried(ctx context.Context) context.Context {
return context.WithValue(ctx, agentIdentityTaskRecoveryContextKey{}, true)
}
func agentIdentityTaskRecoveryWasTried(ctx context.Context) bool {
tried, _ := ctx.Value(agentIdentityTaskRecoveryContextKey{}).(bool)
return tried
}
func isAgentIdentityTaskInvalidWSDialError(err *openAIWSDialError) bool {
return err != nil && isAgentIdentityTaskInvalidHTTPResponse(err.StatusCode, err.ResponseBody)
}
@@ -55,6 +55,52 @@ func TestAccountTestServiceOpenAICompactAgentIdentityUsesFreshAssertion(t *testi
require.NotContains(t, upstream.lastReq.Header.Get("Authorization"), privateKey)
}
func TestAccountTestServiceOpenAICompactAgentIdentityRecoversInvalidTaskOnce(t *testing.T) {
gin.SetMode(gin.TestMode)
key, privateKey := newTestAgentIdentityKey(t)
account := &Account{
ID: 22,
Name: "agent-identity-recovery",
Platform: PlatformOpenAI,
Type: AccountTypeOAuth,
Status: StatusActive,
Schedulable: true,
Concurrency: 1,
Credentials: map[string]any{
"auth_mode": OpenAIAuthModeAgentIdentity,
"agent_runtime_id": key.runtimeID,
"agent_private_key": privateKey,
"task_id": "task-compact-old",
"chatgpt_account_id": "account-agent-compact-recovery",
},
}
repo := &accountTestAgentIdentityRepo{account: account}
registerCalls := 0
registerServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
registerCalls++
_, _ = io.WriteString(w, `{"task_id":"task-compact-new"}`)
}))
defer registerServer.Close()
oldBase := openAIAgentIdentityAuthAPIBaseURL
openAIAgentIdentityAuthAPIBaseURL = registerServer.URL
t.Cleanup(func() { openAIAgentIdentityAuthAPIBaseURL = oldBase })
upstream := &httpUpstreamRecorder{responses: []*http.Response{
{StatusCode: http.StatusUnauthorized, Header: http.Header{"Content-Type": []string{"application/json"}}, Body: io.NopCloser(strings.NewReader(`{"error":{"code":"invalid_task_id"}}`))},
{StatusCode: http.StatusOK, Header: http.Header{"Content-Type": []string{"application/json"}}, Body: io.NopCloser(strings.NewReader(`{"id":"compact-agent","status":"completed"}`))},
}}
svc := &AccountTestService{accountRepo: repo, httpUpstream: upstream}
rec := httptest.NewRecorder()
c, _ := gin.CreateTestContext(rec)
c.Request = httptest.NewRequest(http.MethodPost, "/api/v1/admin/accounts/22/test", bytes.NewReader(nil))
require.NoError(t, svc.TestAccountConnection(c, account.ID, "gpt-5.4", "", AccountTestModeCompact))
require.Equal(t, 1, registerCalls)
require.Len(t, upstream.requests, 2)
require.Equal(t, "task-compact-new", account.GetCredential("task_id"))
require.Equal(t, 0, repo.setErrorCalls)
}
func TestOpenAIAgentIdentityPassthroughKeepsSessionAndPromptCacheHeaders(t *testing.T) {
gin.SetMode(gin.TestMode)
key, privateKey := newTestAgentIdentityKey(t)
@@ -225,6 +271,7 @@ func TestOpenAIAgentIdentityTaskInvalidRetriesExactlyOnce(t *testing.T) {
{StatusCode: http.StatusUnauthorized, Header: http.Header{"Content-Type": []string{"application/json"}}, Body: io.NopCloser(strings.NewReader(`{"error":{"code":"invalid_task_id"}}`))},
{StatusCode: http.StatusOK, Header: http.Header{"Content-Type": []string{"application/json"}}, Body: io.NopCloser(strings.NewReader(successBody))},
}}
require.True(t, isAgentIdentityTaskInvalidHTTPResponse(http.StatusUnauthorized, []byte(`{"error":{"code":"invalid_task_id"}}`)))
svc := &OpenAIGatewayService{cfg: &config.Config{}, accountRepo: repo, httpUpstream: upstream}
rec := httptest.NewRecorder()
c, _ := gin.CreateTestContext(rec)
@@ -256,7 +303,7 @@ func TestOpenAIAgentIdentityTaskInvalidRetriesExactlyOnce(t *testing.T) {
account.Credentials["task_id"] = "task-old-passthrough"
upstream.responses = []*http.Response{
{StatusCode: http.StatusUnauthorized, Header: http.Header{"Content-Type": []string{"application/json"}}, Body: io.NopCloser(strings.NewReader(`{"error":{"code":"invalid_task_id"}}`))},
{StatusCode: http.StatusOK, Header: http.Header{"Content-Type": []string{"application/json"}}, Body: io.NopCloser(strings.NewReader(successBody))},
{StatusCode: http.StatusOK, Header: http.Header{"Content-Type": []string{"text/event-stream"}}, Body: io.NopCloser(strings.NewReader("data: {\"type\":\"response.completed\",\"response\":{\"usage\":{\"input_tokens\":1,\"output_tokens\":1,\"total_tokens\":2}}}\n\ndata: [DONE]\n\n"))},
}
rec3 := httptest.NewRecorder()
c3, _ := gin.CreateTestContext(rec3)
@@ -267,6 +314,80 @@ func TestOpenAIAgentIdentityTaskInvalidRetriesExactlyOnce(t *testing.T) {
require.Len(t, upstream.requests, 6)
}
func TestOpenAIAgentIdentityCompatRoutesRecoverInvalidTaskOnce(t *testing.T) {
gin.SetMode(gin.TestMode)
tests := []struct {
name string
path string
body []byte
call func(*OpenAIGatewayService, context.Context, *gin.Context, *Account, []byte) (*OpenAIForwardResult, error)
}{
{
name: "chat completions",
path: "/v1/chat/completions",
body: []byte(`{"model":"gpt-5.4","stream":false,"messages":[{"role":"user","content":"hi"}]}`),
call: func(s *OpenAIGatewayService, ctx context.Context, c *gin.Context, account *Account, body []byte) (*OpenAIForwardResult, error) {
return s.ForwardAsChatCompletions(ctx, c, account, body, "", "gpt-5.4")
},
},
{
name: "anthropic messages",
path: "/v1/messages",
body: []byte(`{"model":"gpt-5.4","stream":false,"max_tokens":32,"messages":[{"role":"user","content":"hi"}]}`),
call: func(s *OpenAIGatewayService, ctx context.Context, c *gin.Context, account *Account, body []byte) (*OpenAIForwardResult, error) {
return s.ForwardAsAnthropic(ctx, c, account, body, "", "gpt-5.4")
},
},
}
for index, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
key, privateKey := newTestAgentIdentityKey(t)
account := &Account{
ID: int64(40 + index),
Name: "agent-identity-compat",
Platform: PlatformOpenAI,
Type: AccountTypeOAuth,
Status: StatusActive,
Schedulable: true,
Concurrency: 1,
Credentials: map[string]any{
"auth_mode": OpenAIAuthModeAgentIdentity,
"agent_runtime_id": key.runtimeID,
"agent_private_key": privateKey,
"task_id": "task-compat-old",
"chatgpt_account_id": "account-compat-recovery",
},
}
repo := &agentIdentityForwardRepo{account: account}
registerCalls := 0
registerServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
registerCalls++
_, _ = io.WriteString(w, `{"task_id":"task-compat-new"}`)
}))
defer registerServer.Close()
oldBase := openAIAgentIdentityAuthAPIBaseURL
openAIAgentIdentityAuthAPIBaseURL = registerServer.URL
t.Cleanup(func() { openAIAgentIdentityAuthAPIBaseURL = oldBase })
upstream := &httpUpstreamRecorder{responses: []*http.Response{
{StatusCode: http.StatusUnauthorized, Header: http.Header{"Content-Type": []string{"application/json"}}, Body: io.NopCloser(strings.NewReader(`{"error":{"code":"invalid_task_id"}}`))},
{StatusCode: http.StatusUnauthorized, Header: http.Header{"Content-Type": []string{"application/json"}}, Body: io.NopCloser(strings.NewReader(`{"error":{"code":"invalid_task_id"}}`))},
}}
svc := &OpenAIGatewayService{cfg: &config.Config{}, accountRepo: repo, httpUpstream: upstream}
rec := httptest.NewRecorder()
c, _ := gin.CreateTestContext(rec)
c.Request = httptest.NewRequest(http.MethodPost, tt.path, bytes.NewReader(tt.body))
_, err := tt.call(svc, context.Background(), c, account, tt.body)
require.Error(t, err)
require.Equal(t, 1, registerCalls)
require.Len(t, upstream.requests, 2)
require.Equal(t, "task-compat-new", account.GetCredential("task_id"))
})
}
}
func decodeAgentAssertionTask(t *testing.T, header string) string {
t.Helper()
encoded := strings.TrimPrefix(header, "AgentAssertion ")
@@ -284,6 +405,30 @@ type agentIdentityForwardRepo struct {
account *Account
}
type accountTestAgentIdentityRepo struct {
AccountRepository
account *Account
setErrorCalls int
}
func (r *accountTestAgentIdentityRepo) GetByID(_ context.Context, _ int64) (*Account, error) {
return r.account, nil
}
func (r *accountTestAgentIdentityRepo) UpdateCredentials(_ context.Context, _ int64, credentials map[string]any) error {
r.account.Credentials = credentials
return nil
}
func (r *accountTestAgentIdentityRepo) UpdateExtra(_ context.Context, _ int64, _ map[string]any) error {
return nil
}
func (r *accountTestAgentIdentityRepo) SetError(_ context.Context, _ int64, _ string) error {
r.setErrorCalls++
return nil
}
func (r *agentIdentityForwardRepo) GetByID(_ context.Context, _ int64) (*Account, error) {
return r.account, nil
}
@@ -267,6 +267,13 @@ func (s *OpenAIGatewayService) ForwardAsChatCompletions(
// 8. Handle error response with failover
if resp.StatusCode >= 400 {
respBody, upstreamMsg := s.readOpenAIUpstreamError(resp)
if !agentIdentityTaskRecoveryWasTried(ctx) && s.isAgentIdentityAccount(ctx, account) && isAgentIdentityTaskInvalidHTTPResponse(resp.StatusCode, respBody) {
expectedTaskID := account.GetCredential("task_id")
if err := s.recoverAgentIdentityTask(ctx, account, expectedTaskID); err != nil {
return nil, fmt.Errorf("agent identity task recovery failed: %w", err)
}
return s.ForwardAsChatCompletions(markAgentIdentityTaskRecoveryTried(ctx), c, account, body, promptCacheKey, defaultMappedModel)
}
if account.Type == AccountTypeAPIKey &&
openai_compat.ResolveResponsesSupport(account.Extra) == openai_compat.ResponsesSupportUnknown &&
!isResponsesEndpointSupportedByStatus(resp.StatusCode) {
@@ -316,6 +316,13 @@ func (s *OpenAIGatewayService) ForwardAsAnthropic(
// 8. Handle error response with failover
if resp.StatusCode >= 400 {
respBody, upstreamMsg := s.readOpenAIUpstreamError(resp)
if !agentIdentityTaskRecoveryWasTried(ctx) && s.isAgentIdentityAccount(ctx, account) && isAgentIdentityTaskInvalidHTTPResponse(resp.StatusCode, respBody) {
expectedTaskID := account.GetCredential("task_id")
if err := s.recoverAgentIdentityTask(ctx, account, expectedTaskID); err != nil {
return nil, fmt.Errorf("agent identity task recovery failed: %w", err)
}
return s.ForwardAsAnthropic(markAgentIdentityTaskRecoveryTried(ctx), c, account, body, promptCacheKey, defaultMappedModel)
}
if account.Platform == PlatformGrok {
s.updateGrokUsageSnapshot(ctx, account.ID, xai.ParseQuotaHeaders(resp.Header, resp.StatusCode))
s.handleGrokAccountUpstreamError(ctx, account, resp.StatusCode, resp.Header, respBody)
@@ -1560,6 +1560,14 @@ func (s *OpenAIGatewayService) forwardOpenAIImagesOAuth(
if resp.StatusCode >= 400 {
respBody := s.readUpstreamErrorBody(resp)
_ = resp.Body.Close()
respBody = s.redactAgentIdentitySensitiveBody(upstreamCtx, account, respBody)
if !agentIdentityTaskRecoveryWasTried(ctx) && s.isAgentIdentityAccount(ctx, account) && isAgentIdentityTaskInvalidHTTPResponse(resp.StatusCode, respBody) {
expectedTaskID := account.GetCredential("task_id")
if err := s.recoverAgentIdentityTask(ctx, account, expectedTaskID); err != nil {
return nil, fmt.Errorf("agent identity task recovery failed: %w", err)
}
return s.forwardOpenAIImagesOAuth(markAgentIdentityTaskRecoveryTried(ctx), c, account, parsed, channelMappedModel)
}
resp.Body = io.NopCloser(bytes.NewReader(respBody))
upstreamMsg := strings.TrimSpace(extractUpstreamErrorMessage(respBody))
upstreamMsg = sanitizeUpstreamErrorMessage(upstreamMsg)
@@ -155,25 +155,36 @@ func (s *OpenAIQuotaService) QueryUsage(ctx context.Context, accountID int64) (*
callCtx, cancel := context.WithTimeout(ctx, openaiQuotaUpstreamTimeout)
defer cancel()
agentIdentity := s.isAgentIdentityAccount(ctx, accountID)
quotaHeaders, headerErr := s.buildCodexQuotaHeaders(callCtx, accountID, accessToken, chatGPTAccountID, fedRAMP)
if headerErr != nil {
return nil, infraerrors.Newf(http.StatusBadGateway, "OPENAI_QUOTA_AUTH_FAILED", "failed to build upstream authentication: %v", headerErr)
}
var payload OpenAIQuotaUsage
resp, err := client.R().
SetContext(callCtx).
SetHeaders(quotaHeaders).
SetSuccessResult(&payload).
Get(chatGPTUsageURL)
if err != nil {
return nil, infraerrors.Newf(http.StatusBadGateway, "OPENAI_QUOTA_REQUEST_FAILED", "upstream request failed: %v", err)
}
if !resp.IsSuccessState() {
status := resp.StatusCode
body := truncate(s.redactQuotaErrorBody(ctx, accountID, resp.String()), 240)
slog.Warn("openai_quota_query_failed", "account_id", accountID, "status", status, "body", body)
return nil, infraerrors.Newf(mapUpstreamStatus(status), "OPENAI_QUOTA_UPSTREAM_ERROR", "upstream returned %d: %s", status, body)
for recovered := false; ; {
quotaHeaders, headerErr := s.buildCodexQuotaHeaders(callCtx, accountID, accessToken, chatGPTAccountID, fedRAMP)
if headerErr != nil {
return nil, infraerrors.Newf(http.StatusBadGateway, "OPENAI_QUOTA_AUTH_FAILED", "failed to build upstream authentication: %v", headerErr)
}
resp, err := client.R().
SetContext(callCtx).
SetHeaders(quotaHeaders).
SetSuccessResult(&payload).
Get(chatGPTUsageURL)
if err != nil {
return nil, infraerrors.Newf(http.StatusBadGateway, "OPENAI_QUOTA_REQUEST_FAILED", "upstream request failed: %v", err)
}
if !resp.IsSuccessState() {
if agentIdentity && !recovered && isAgentIdentityTaskInvalidHTTPResponse(resp.StatusCode, []byte(resp.String())) {
recovered = true
if err := s.recoverAgentIdentityTask(ctx, accountID); err != nil {
return nil, infraerrors.Newf(http.StatusBadGateway, "OPENAI_QUOTA_AUTH_FAILED", "agent identity task recovery failed: %v", err)
}
continue
}
status := resp.StatusCode
body := truncate(s.redactQuotaErrorBody(ctx, accountID, resp.String()), 240)
slog.Warn("openai_quota_query_failed", "account_id", accountID, "status", status, "body", body)
return nil, infraerrors.Newf(mapUpstreamStatus(status), "OPENAI_QUOTA_UPSTREAM_ERROR", "upstream returned %d: %s", status, body)
}
break
}
payload.FetchedAt = time.Now().Unix()
@@ -249,28 +260,38 @@ func (s *OpenAIQuotaService) ResetCredit(ctx context.Context, accountID int64) (
callCtx, cancel := context.WithTimeout(ctx, openaiQuotaUpstreamTimeout)
defer cancel()
headers, headerErr := s.buildCodexQuotaHeaders(callCtx, accountID, accessToken, chatGPTAccountID, fedRAMP)
if headerErr != nil {
return nil, infraerrors.Newf(http.StatusBadGateway, "OPENAI_QUOTA_AUTH_FAILED", "failed to build upstream authentication: %v", headerErr)
}
headers["content-type"] = "application/json"
agentIdentity := s.isAgentIdentityAccount(ctx, accountID)
var payload OpenAIQuotaResetResult
resp, err := client.R().
SetContext(callCtx).
SetHeaders(headers).
SetBody(map[string]string{"redeem_request_id": redeemRequestID}).
SetSuccessResult(&payload).
Post(chatGPTRateLimitResetURL)
if err != nil {
return nil, infraerrors.Newf(http.StatusBadGateway, "OPENAI_QUOTA_RESET_REQUEST_FAILED", "upstream request failed: %v", err)
}
if !resp.IsSuccessState() {
status := resp.StatusCode
body := truncate(s.redactQuotaErrorBody(callCtx, accountID, resp.String()), 240)
slog.Warn("openai_quota_reset_failed", "account_id", accountID, "status", status, "body", body)
return nil, infraerrors.Newf(mapUpstreamStatus(status), "OPENAI_QUOTA_RESET_UPSTREAM_ERROR", "upstream returned %d: %s", status, body)
for recovered := false; ; {
headers, headerErr := s.buildCodexQuotaHeaders(callCtx, accountID, accessToken, chatGPTAccountID, fedRAMP)
if headerErr != nil {
return nil, infraerrors.Newf(http.StatusBadGateway, "OPENAI_QUOTA_AUTH_FAILED", "failed to build upstream authentication: %v", headerErr)
}
headers["content-type"] = "application/json"
resp, err := client.R().
SetContext(callCtx).
SetHeaders(headers).
SetBody(map[string]string{"redeem_request_id": redeemRequestID}).
SetSuccessResult(&payload).
Post(chatGPTRateLimitResetURL)
if err != nil {
return nil, infraerrors.Newf(http.StatusBadGateway, "OPENAI_QUOTA_RESET_REQUEST_FAILED", "upstream request failed: %v", err)
}
if !resp.IsSuccessState() {
if agentIdentity && !recovered && isAgentIdentityTaskInvalidHTTPResponse(resp.StatusCode, []byte(resp.String())) {
recovered = true
if err := s.recoverAgentIdentityTask(ctx, accountID); err != nil {
return nil, infraerrors.Newf(http.StatusBadGateway, "OPENAI_QUOTA_AUTH_FAILED", "agent identity task recovery failed: %v", err)
}
continue
}
status := resp.StatusCode
body := truncate(s.redactQuotaErrorBody(callCtx, accountID, resp.String()), 240)
slog.Warn("openai_quota_reset_failed", "account_id", accountID, "status", status, "body", body)
return nil, infraerrors.Newf(mapUpstreamStatus(status), "OPENAI_QUOTA_RESET_UPSTREAM_ERROR", "upstream returned %d: %s", status, body)
}
break
}
slog.Info("openai_quota_reset_success",
@@ -356,6 +377,43 @@ func (s *OpenAIQuotaService) prepareUpstreamCall(ctx context.Context, accountID
return accessToken, chatGPTAccountID, proxyURL, fedRAMP, nil
}
func (s *OpenAIQuotaService) recoverAgentIdentityTask(ctx context.Context, accountID int64) error {
if s == nil || s.accountRepo == nil {
return fmt.Errorf("account repository is unavailable")
}
account, err := s.accountRepo.GetByID(ctx, accountID)
if err != nil || account == nil {
return fmt.Errorf("account is unavailable")
}
if account.IsShadow() {
account, err = resolveCredentialAccount(ctx, s.accountRepo, account)
if err != nil || account == nil {
return fmt.Errorf("credential account is unavailable")
}
}
if !account.IsOpenAIAgentIdentity() {
return nil
}
return ensureAgentIdentityTaskForAccount(ctx, s.accountRepo, nil, &s.agentIdentityTaskMu, account, account.GetCredential("task_id"))
}
func (s *OpenAIQuotaService) isAgentIdentityAccount(ctx context.Context, accountID int64) bool {
if s == nil || s.accountRepo == nil {
return false
}
account, err := s.accountRepo.GetByID(ctx, accountID)
if err != nil || account == nil {
return false
}
if account.IsShadow() {
account, err = resolveCredentialAccount(ctx, s.accountRepo, account)
if err != nil || account == nil {
return false
}
}
return account.IsOpenAIAgentIdentity()
}
func (s *OpenAIQuotaService) buildCodexQuotaHeaders(ctx context.Context, accountID int64, accessToken, chatGPTAccountID string, fedRAMP bool) (map[string]string, error) {
headers := buildCodexCommonHeaders(accessToken, chatGPTAccountID, fedRAMP)
if s == nil || s.accountRepo == nil {
@@ -38,6 +38,15 @@ func (r *stubQuotaAccountRepo) GetByID(_ context.Context, id int64) (*Account, e
return acc, nil
}
func (r *stubQuotaAccountRepo) UpdateCredentials(_ context.Context, id int64, credentials map[string]any) error {
acc, ok := r.accounts[id]
if !ok {
return fmt.Errorf("account %d not found", id)
}
acc.Credentials = credentials
return nil
}
// stubQuotaTokenCache 实现 OpenAITokenCache,返回预设静态 token。
type stubQuotaTokenCache struct {
tokens map[string]string
@@ -257,6 +266,55 @@ func TestQueryUsageAgentIdentityUsesAssertionWithoutOAuthToken(t *testing.T) {
require.Equal(t, "true", fedrampHeader)
}
func TestQueryUsageAgentIdentityRecoversInvalidTaskOnce(t *testing.T) {
_, privateKey, err := ed25519.GenerateKey(rand.Reader)
require.NoError(t, err)
der, err := x509.MarshalPKCS8PrivateKey(privateKey)
require.NoError(t, err)
account := &Account{
ID: 301,
Platform: PlatformOpenAI,
Type: AccountTypeOAuth,
Credentials: map[string]any{
"auth_mode": OpenAIAuthModeAgentIdentity,
"agent_runtime_id": "runtime-quota-recovery",
"agent_private_key": base64.StdEncoding.EncodeToString(der),
"task_id": "task-quota-old",
"chatgpt_account_id": "account-quota-recovery",
},
}
repo := &stubQuotaAccountRepo{accounts: map[int64]*Account{account.ID: account}}
usageCalls := 0
registerCalls := 0
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("content-type", "application/json")
if strings.Contains(r.URL.Path, "/task/register") {
registerCalls++
_, _ = w.Write([]byte(`{"task_id":"task-quota-new"}`))
return
}
usageCalls++
if usageCalls == 1 {
w.WriteHeader(http.StatusUnauthorized)
_, _ = w.Write([]byte(`{"error":{"code":"invalid_task_id"}}`))
return
}
_, _ = w.Write([]byte(`{"plan_type":"pro","rate_limit":{"allowed":true}}`))
}))
defer srv.Close()
oldBase := openAIAgentIdentityAuthAPIBaseURL
openAIAgentIdentityAuthAPIBaseURL = srv.URL
t.Cleanup(func() { openAIAgentIdentityAuthAPIBaseURL = oldBase })
svc := NewOpenAIQuotaService(repo, nil, nil, newQuotaRedirectingFactory(srv))
usage, err := svc.QueryUsage(context.Background(), account.ID)
require.NoError(t, err)
require.NotNil(t, usage)
require.Equal(t, 2, usageCalls)
require.Equal(t, 1, registerCalls)
require.Equal(t, "task-quota-new", account.GetCredential("task_id"))
}
func TestParseOpenAIRateLimitResetCreditDetails_CompatibleContainers(t *testing.T) {
tests := []struct {
name string