mirror of
https://github.com/Wei-Shaw/sub2api.git
synced 2026-09-24 16:05:44 +08:00
fix(grok): persist quota exhaustion as rate limit
This commit is contained in:
@@ -1286,6 +1286,38 @@ func (r *accountRepository) SetRateLimited(ctx context.Context, id int64, resetA
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetRateLimitedIfLater atomically extends an account-level rate limit. Grok
|
||||
// requests may finish concurrently, so an older response must not overwrite a
|
||||
// later reset boundary observed by another request or instance.
|
||||
func (r *accountRepository) SetRateLimitedIfLater(ctx context.Context, id int64, resetAt time.Time) error {
|
||||
now := time.Now()
|
||||
updated, err := r.client.Account.Update().
|
||||
Where(
|
||||
dbaccount.IDEQ(id),
|
||||
dbaccount.Or(
|
||||
dbaccount.RateLimitResetAtIsNil(),
|
||||
dbaccount.RateLimitResetAtLT(resetAt),
|
||||
),
|
||||
).
|
||||
SetRateLimitedAt(now).
|
||||
SetRateLimitResetAt(resetAt).
|
||||
Save(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if updated == 0 {
|
||||
// This instance may not have observed the later value written elsewhere.
|
||||
// Refresh its local scheduler snapshot even though no outbox event is needed.
|
||||
r.syncSchedulerAccountSnapshot(ctx, id)
|
||||
return nil
|
||||
}
|
||||
if err := enqueueSchedulerOutbox(ctx, r.sql, service.SchedulerOutboxEventAccountChanged, &id, nil, nil); err != nil {
|
||||
logger.LegacyPrintf("repository.account", "[SchedulerOutbox] enqueue extended rate limit failed: account=%d err=%v", id, err)
|
||||
}
|
||||
r.syncSchedulerAccountSnapshot(ctx, id)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *accountRepository) SetModelRateLimit(ctx context.Context, id int64, scope string, resetAt time.Time, reason ...string) error {
|
||||
if scope == "" {
|
||||
return nil
|
||||
|
||||
@@ -703,6 +703,25 @@ func (s *AccountRepoSuite) TestSetRateLimited() {
|
||||
s.Require().WithinDuration(resetAt, *got.RateLimitResetAt, time.Second)
|
||||
}
|
||||
|
||||
func (s *AccountRepoSuite) TestSetRateLimitedIfLaterDoesNotShortenReset() {
|
||||
account := mustCreateAccount(s.T(), s.client, &service.Account{Name: "acc-rl-monotonic"})
|
||||
later := time.Now().Add(30 * time.Minute).UTC().Truncate(time.Second)
|
||||
earlier := time.Now().Add(5 * time.Minute).UTC().Truncate(time.Second)
|
||||
cacheRecorder := &schedulerCacheRecorder{}
|
||||
s.repo.schedulerCache = cacheRecorder
|
||||
|
||||
s.Require().NoError(s.repo.SetRateLimitedIfLater(s.ctx, account.ID, later))
|
||||
s.Require().NoError(s.repo.SetRateLimitedIfLater(s.ctx, account.ID, earlier))
|
||||
|
||||
got, err := s.repo.GetByID(s.ctx, account.ID)
|
||||
s.Require().NoError(err)
|
||||
s.Require().NotNil(got.RateLimitResetAt)
|
||||
s.Require().WithinDuration(later, *got.RateLimitResetAt, time.Second)
|
||||
s.Require().Len(cacheRecorder.setAccounts, 2)
|
||||
s.Require().NotNil(cacheRecorder.setAccounts[1].RateLimitResetAt)
|
||||
s.Require().WithinDuration(later, *cacheRecorder.setAccounts[1].RateLimitResetAt, time.Second)
|
||||
}
|
||||
|
||||
func (s *AccountRepoSuite) TestClearRateLimit() {
|
||||
account := mustCreateAccount(s.T(), s.client, &service.Account{Name: "acc-clear"})
|
||||
until := time.Now().Add(1 * time.Hour)
|
||||
|
||||
@@ -723,10 +723,19 @@ func (s *AccountTestService) testGrokAccountConnection(c *gin.Context, account *
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if snapshot := xai.ParseQuotaHeaders(resp.Header, resp.StatusCode); snapshot != nil && s.accountRepo != nil {
|
||||
now := time.Now()
|
||||
snapshot := parseGrokQuotaSnapshot(resp.Header, resp.StatusCode, now)
|
||||
if snapshot != nil && s.accountRepo != nil {
|
||||
resetAt, limited := grokRateLimitResetAt(snapshot, now)
|
||||
if limited {
|
||||
normalizeGrokExhaustedWindowResets(snapshot, resetAt, now)
|
||||
}
|
||||
_ = s.accountRepo.UpdateExtra(ctx, account.ID, map[string]any{
|
||||
grokQuotaSnapshotExtraKey: snapshot,
|
||||
})
|
||||
if limited {
|
||||
persistGrokRateLimit(ctx, s.accountRepo, account, resetAt)
|
||||
}
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
@@ -15,6 +16,18 @@ import (
|
||||
"github.com/tidwall/gjson"
|
||||
)
|
||||
|
||||
type grokAccountTestRateLimitRepo struct {
|
||||
*mockAccountRepoForGemini
|
||||
rateLimitedCalls int
|
||||
resetAt time.Time
|
||||
}
|
||||
|
||||
func (r *grokAccountTestRateLimitRepo) SetRateLimited(_ context.Context, _ int64, resetAt time.Time) error {
|
||||
r.rateLimitedCalls++
|
||||
r.resetAt = resetAt
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestAccountTestService_TestAccountConnection_GrokUsesXAIResponses(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
|
||||
@@ -66,3 +79,73 @@ func TestAccountTestService_TestAccountConnection_GrokUsesXAIResponses(t *testin
|
||||
require.Contains(t, rec.Body.String(), `"model":"grok-4.3"`)
|
||||
require.Contains(t, rec.Body.String(), `"type":"test_complete"`)
|
||||
}
|
||||
|
||||
func TestAccountTestService_Grok429PersistsRateLimitReset(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
|
||||
account := &Account{
|
||||
ID: 14,
|
||||
Name: "grok-oauth-limited",
|
||||
Platform: PlatformGrok,
|
||||
Type: AccountTypeOAuth,
|
||||
Status: StatusActive,
|
||||
Schedulable: true,
|
||||
Concurrency: 1,
|
||||
Credentials: map[string]any{
|
||||
"access_token": "grok-access-token",
|
||||
"expires_at": time.Now().Add(time.Hour).UTC().Format(time.RFC3339),
|
||||
},
|
||||
}
|
||||
baseRepo := &mockAccountRepoForGemini{accountsByID: map[int64]*Account{account.ID: account}}
|
||||
repo := &grokAccountTestRateLimitRepo{mockAccountRepoForGemini: baseRepo}
|
||||
upstream := &httpUpstreamRecorder{resp: &http.Response{
|
||||
StatusCode: http.StatusTooManyRequests,
|
||||
Header: http.Header{"Retry-After": []string{"45"}},
|
||||
Body: io.NopCloser(strings.NewReader(`{"error":{"message":"rate limited"}}`)),
|
||||
}}
|
||||
svc := &AccountTestService{
|
||||
accountRepo: repo,
|
||||
grokTokenProvider: NewGrokTokenProvider(repo, nil),
|
||||
httpUpstream: upstream,
|
||||
}
|
||||
recorder := httptest.NewRecorder()
|
||||
c, _ := gin.CreateTestContext(recorder)
|
||||
c.Request = httptest.NewRequest(http.MethodPost, "/api/v1/admin/accounts/14/test", nil)
|
||||
|
||||
err := svc.TestAccountConnection(c, account.ID, "grok", "", AccountTestModeDefault)
|
||||
|
||||
require.Error(t, err)
|
||||
require.Equal(t, 1, repo.rateLimitedCalls)
|
||||
require.WithinDuration(t, time.Now().Add(45*time.Second), repo.resetAt, time.Second)
|
||||
}
|
||||
|
||||
func TestAccountTestService_Grok429WithoutQuotaHeadersUsesFallback(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
account := &Account{
|
||||
ID: 15, Name: "grok-oauth-limited-no-headers", Platform: PlatformGrok,
|
||||
Type: AccountTypeOAuth, Status: StatusActive, Schedulable: true, Concurrency: 1,
|
||||
Credentials: map[string]any{
|
||||
"access_token": "grok-access-token",
|
||||
"expires_at": time.Now().Add(time.Hour).UTC().Format(time.RFC3339),
|
||||
},
|
||||
}
|
||||
baseRepo := &mockAccountRepoForGemini{accountsByID: map[int64]*Account{account.ID: account}}
|
||||
repo := &grokAccountTestRateLimitRepo{mockAccountRepoForGemini: baseRepo}
|
||||
upstream := &httpUpstreamRecorder{resp: &http.Response{
|
||||
StatusCode: http.StatusTooManyRequests,
|
||||
Body: io.NopCloser(strings.NewReader(`{"error":{"message":"quota exhausted"}}`)),
|
||||
}}
|
||||
svc := &AccountTestService{
|
||||
accountRepo: repo, grokTokenProvider: NewGrokTokenProvider(repo, nil), httpUpstream: upstream,
|
||||
}
|
||||
recorder := httptest.NewRecorder()
|
||||
c, _ := gin.CreateTestContext(recorder)
|
||||
c.Request = httptest.NewRequest(http.MethodPost, "/api/v1/admin/accounts/15/test", nil)
|
||||
before := time.Now()
|
||||
|
||||
err := svc.TestAccountConnection(c, account.ID, "grok", "", AccountTestModeDefault)
|
||||
|
||||
require.Error(t, err)
|
||||
require.Equal(t, 1, repo.rateLimitedCalls)
|
||||
require.WithinDuration(t, before.Add(grokRateLimitFallbackCooldown), repo.resetAt, time.Second)
|
||||
}
|
||||
|
||||
@@ -357,11 +357,10 @@ func (s *OpenAIGatewayService) ForwardGrokMedia(
|
||||
requestIDHeader := firstNonEmpty(resp.Header.Get("x-request-id"), resp.Header.Get("xai-request-id"))
|
||||
requestModel := requestInfo.Model
|
||||
if resp.StatusCode >= 400 {
|
||||
s.updateGrokUsageSnapshot(ctx, account.ID, xai.ParseQuotaHeaders(resp.Header, resp.StatusCode))
|
||||
return s.handleGrokMediaErrorResponse(ctx, resp, c, account, requestIDHeader, requestModel)
|
||||
}
|
||||
|
||||
s.updateGrokUsageSnapshot(ctx, account.ID, xai.ParseQuotaHeaders(resp.Header, resp.StatusCode))
|
||||
s.updateGrokUsageSnapshot(ctx, account, xai.ParseQuotaHeaders(resp.Header, resp.StatusCode))
|
||||
respBody, err := ReadUpstreamResponseBody(resp.Body, s.cfg, c, openAITooLargeError)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -564,6 +563,9 @@ func (s *OpenAIGatewayService) handleGrokMediaErrorResponse(
|
||||
requestedModel string,
|
||||
) (*OpenAIForwardResult, error) {
|
||||
body := s.readUpstreamErrorBody(resp)
|
||||
// Reconcile readiness before configurable passthrough branches can return;
|
||||
// otherwise a Grok 429 can remain schedulable.
|
||||
s.handleGrokAccountUpstreamError(ctx, account, resp.StatusCode, resp.Header, body)
|
||||
upstreamMsg := sanitizeUpstreamErrorMessage(strings.TrimSpace(extractUpstreamErrorMessage(body)))
|
||||
if upstreamMsg == "" {
|
||||
upstreamMsg = fmt.Sprintf("xAI upstream returned status %d", resp.StatusCode)
|
||||
@@ -609,7 +611,6 @@ func (s *OpenAIGatewayService) handleGrokMediaErrorResponse(
|
||||
return nil, fmt.Errorf("upstream error: %d (not in custom error codes) message=%s", resp.StatusCode, upstreamMsg)
|
||||
}
|
||||
|
||||
s.handleGrokAccountUpstreamError(ctx, account, resp.StatusCode, resp.Header, body)
|
||||
kind := "http_error"
|
||||
if s.shouldFailoverUpstreamError(resp.StatusCode) {
|
||||
kind = "failover"
|
||||
|
||||
@@ -91,9 +91,16 @@ func (s *GrokQuotaService) ProbeUsage(ctx context.Context, accountID int64) (*Gr
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
snapshot := xai.ObserveQuotaHeaders(resp.Header, resp.StatusCode, "active_probe")
|
||||
resetAt, limited := grokRateLimitResetAt(snapshot, time.Now())
|
||||
if limited {
|
||||
normalizeGrokExhaustedWindowResets(snapshot, resetAt, time.Now())
|
||||
}
|
||||
_ = s.accountRepo.UpdateExtra(ctx, account.ID, map[string]any{
|
||||
grokQuotaSnapshotExtraKey: snapshot,
|
||||
})
|
||||
if limited {
|
||||
persistGrokRateLimit(ctx, s.accountRepo, account, resetAt)
|
||||
}
|
||||
|
||||
result := &GrokQuotaProbeResult{
|
||||
Source: "active_probe",
|
||||
|
||||
@@ -19,6 +19,10 @@ import (
|
||||
type grokQuotaAccountRepo struct {
|
||||
*mockAccountRepoForPlatform
|
||||
updates map[int64]map[string]any
|
||||
updateCalls int
|
||||
rateLimitedCalls int
|
||||
lastRateLimitedID int64
|
||||
lastRateLimitResetAt time.Time
|
||||
tempUnschedCalls int
|
||||
lastTempUnschedID int64
|
||||
lastTempUnschedUntil time.Time
|
||||
@@ -26,6 +30,7 @@ type grokQuotaAccountRepo struct {
|
||||
}
|
||||
|
||||
func (r *grokQuotaAccountRepo) UpdateExtra(_ context.Context, id int64, updates map[string]any) error {
|
||||
r.updateCalls++
|
||||
if r.updates == nil {
|
||||
r.updates = make(map[int64]map[string]any)
|
||||
}
|
||||
@@ -33,6 +38,17 @@ func (r *grokQuotaAccountRepo) UpdateExtra(_ context.Context, id int64, updates
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *grokQuotaAccountRepo) SetRateLimited(_ context.Context, id int64, resetAt time.Time) error {
|
||||
r.rateLimitedCalls++
|
||||
r.lastRateLimitedID = id
|
||||
r.lastRateLimitResetAt = resetAt
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *grokQuotaAccountRepo) SetRateLimitedIfLater(ctx context.Context, id int64, resetAt time.Time) error {
|
||||
return r.SetRateLimited(ctx, id, resetAt)
|
||||
}
|
||||
|
||||
func (r *grokQuotaAccountRepo) SetTempUnschedulable(_ context.Context, id int64, until time.Time, reason string) error {
|
||||
r.tempUnschedCalls++
|
||||
r.lastTempUnschedID = id
|
||||
@@ -286,6 +302,10 @@ func TestGrokQuotaServiceProbeUsageReturnsRateLimitedSnapshot(t *testing.T) {
|
||||
require.NotNil(t, result.Snapshot)
|
||||
require.NotNil(t, result.Snapshot.RetryAfterSeconds)
|
||||
require.Equal(t, 45, *result.Snapshot.RetryAfterSeconds)
|
||||
require.Equal(t, 1, repo.rateLimitedCalls)
|
||||
require.Equal(t, account.ID, repo.lastRateLimitedID)
|
||||
require.WithinDuration(t, time.Now().Add(45*time.Second), repo.lastRateLimitResetAt, time.Second)
|
||||
require.Zero(t, repo.tempUnschedCalls)
|
||||
}
|
||||
|
||||
func TestGrokQuotaServiceResetQuotaUnsupported(t *testing.T) {
|
||||
|
||||
@@ -88,6 +88,9 @@ func (s *OpenAIGatewayService) failoverOpenAIUpstreamHTTPError(
|
||||
upstreamMsg string,
|
||||
upstreamModel string,
|
||||
) *UpstreamFailoverError {
|
||||
if account != nil && account.Platform == PlatformGrok {
|
||||
s.handleGrokAccountUpstreamError(ctx, account, resp.StatusCode, resp.Header, respBody)
|
||||
}
|
||||
if !s.shouldFailoverOpenAIUpstreamResponse(resp.StatusCode, upstreamMsg, respBody) {
|
||||
return nil
|
||||
}
|
||||
@@ -109,7 +112,9 @@ func (s *OpenAIGatewayService) failoverOpenAIUpstreamHTTPError(
|
||||
Message: upstreamMsg,
|
||||
Detail: upstreamDetail,
|
||||
})
|
||||
s.handleOpenAIAccountUpstreamError(ctx, account, resp.StatusCode, resp.Header, respBody, upstreamModel)
|
||||
if account.Platform != PlatformGrok {
|
||||
s.handleOpenAIAccountUpstreamError(ctx, account, resp.StatusCode, resp.Header, respBody, upstreamModel)
|
||||
}
|
||||
return &UpstreamFailoverError{
|
||||
StatusCode: resp.StatusCode,
|
||||
ResponseBody: respBody,
|
||||
|
||||
@@ -174,7 +174,6 @@ func (s *OpenAIGatewayService) forwardAsRawChatCompletions(
|
||||
if resp.StatusCode >= 400 {
|
||||
respBody, upstreamMsg := s.readOpenAIUpstreamError(resp)
|
||||
if account.Platform == PlatformGrok {
|
||||
s.updateGrokUsageSnapshot(ctx, account.ID, xai.ParseQuotaHeaders(resp.Header, resp.StatusCode))
|
||||
appendOpsUpstreamError(c, OpsUpstreamErrorEvent{
|
||||
Platform: account.Platform,
|
||||
AccountID: account.ID,
|
||||
@@ -201,7 +200,7 @@ func (s *OpenAIGatewayService) forwardAsRawChatCompletions(
|
||||
}
|
||||
|
||||
if account.Platform == PlatformGrok {
|
||||
s.updateGrokUsageSnapshot(ctx, account.ID, xai.ParseQuotaHeaders(resp.Header, resp.StatusCode))
|
||||
s.updateGrokUsageSnapshot(ctx, account, xai.ParseQuotaHeaders(resp.Header, resp.StatusCode))
|
||||
}
|
||||
|
||||
// 8. Forward response
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -22,6 +23,7 @@ const (
|
||||
grokComposerImageBridgeMaxOutputTokens = 512
|
||||
grokUpstreamUserAgent = "sub2api-grok/1.0"
|
||||
grokCLIVersion = "0.2.93"
|
||||
grokRateLimitFallbackCooldown = 2 * time.Minute
|
||||
)
|
||||
|
||||
func (s *OpenAIGatewayService) forwardGrokResponses(
|
||||
@@ -79,7 +81,6 @@ func (s *OpenAIGatewayService) forwardGrokResponses(
|
||||
if resp.StatusCode >= 400 {
|
||||
respBody := s.readUpstreamErrorBody(resp)
|
||||
resp.Body = io.NopCloser(bytes.NewReader(respBody))
|
||||
s.updateGrokUsageSnapshot(ctx, account.ID, xai.ParseQuotaHeaders(resp.Header, resp.StatusCode))
|
||||
upstreamMsg := sanitizeUpstreamErrorMessage(extractUpstreamErrorMessage(respBody))
|
||||
if upstreamMsg == "" {
|
||||
upstreamMsg = fmt.Sprintf("xAI upstream returned status %d", resp.StatusCode)
|
||||
@@ -104,7 +105,7 @@ func (s *OpenAIGatewayService) forwardGrokResponses(
|
||||
return s.handleErrorResponse(ctx, resp, c, account, patchedBody, upstreamModel)
|
||||
}
|
||||
|
||||
s.updateGrokUsageSnapshot(ctx, account.ID, xai.ParseQuotaHeaders(resp.Header, resp.StatusCode))
|
||||
s.updateGrokUsageSnapshot(ctx, account, xai.ParseQuotaHeaders(resp.Header, resp.StatusCode))
|
||||
|
||||
var usage *OpenAIUsage
|
||||
var firstTokenMs *int
|
||||
@@ -485,7 +486,6 @@ func (s *OpenAIGatewayService) describeGrokComposerImage(
|
||||
|
||||
if resp.StatusCode >= 400 {
|
||||
respBody := s.readUpstreamErrorBody(resp)
|
||||
s.updateGrokUsageSnapshot(ctx, account.ID, xai.ParseQuotaHeaders(resp.Header, resp.StatusCode))
|
||||
upstreamMsg := sanitizeUpstreamErrorMessage(extractUpstreamErrorMessage(respBody))
|
||||
if upstreamMsg == "" {
|
||||
upstreamMsg = fmt.Sprintf("xAI image bridge upstream returned status %d", resp.StatusCode)
|
||||
@@ -510,7 +510,7 @@ func (s *OpenAIGatewayService) describeGrokComposerImage(
|
||||
return "", OpenAIUsage{}, fmt.Errorf("grok composer image bridge upstream error: %s", upstreamMsg)
|
||||
}
|
||||
|
||||
s.updateGrokUsageSnapshot(ctx, account.ID, xai.ParseQuotaHeaders(resp.Header, resp.StatusCode))
|
||||
s.updateGrokUsageSnapshot(ctx, account, xai.ParseQuotaHeaders(resp.Header, resp.StatusCode))
|
||||
respBody, err := ReadUpstreamResponseBody(resp.Body, s.cfg, c, nil)
|
||||
if err != nil {
|
||||
return "", OpenAIUsage{}, fmt.Errorf("read grok composer image bridge response: %w", err)
|
||||
@@ -664,33 +664,191 @@ func applyGrokCLIHeaders(headers http.Header) {
|
||||
headers.Set("X-Grok-Client-Version", grokCLIVersion)
|
||||
}
|
||||
|
||||
func (s *OpenAIGatewayService) updateGrokUsageSnapshot(ctx context.Context, accountID int64, snapshot *xai.QuotaSnapshot) {
|
||||
if s == nil || s.accountRepo == nil || accountID <= 0 || snapshot == nil {
|
||||
func (s *OpenAIGatewayService) updateGrokUsageSnapshot(ctx context.Context, account *Account, snapshot *xai.QuotaSnapshot) {
|
||||
if s == nil || account == nil || account.ID <= 0 || snapshot == nil {
|
||||
return
|
||||
}
|
||||
if s.codexSnapshotThrottle != nil && !s.codexSnapshotThrottle.Allow(accountID, time.Now()) {
|
||||
accountID := account.ID
|
||||
now := time.Now()
|
||||
resetAt, hasActiveLimit := grokRateLimitResetAt(snapshot, now)
|
||||
if hasActiveLimit {
|
||||
normalizeGrokExhaustedWindowResets(snapshot, resetAt, now)
|
||||
}
|
||||
critical := snapshot.StatusCode == http.StatusTooManyRequests || hasActiveLimit
|
||||
if s.codexSnapshotThrottle != nil {
|
||||
allowed := s.codexSnapshotThrottle.Allow(accountID, now)
|
||||
if !critical && !allowed {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
stateCtx := ctx
|
||||
if hasActiveLimit {
|
||||
var cancel context.CancelFunc
|
||||
stateCtx, cancel = openAIAccountStateContext(ctx)
|
||||
defer cancel()
|
||||
}
|
||||
if s.accountRepo != nil {
|
||||
_ = s.accountRepo.UpdateExtra(stateCtx, accountID, map[string]any{
|
||||
grokQuotaSnapshotExtraKey: snapshot,
|
||||
})
|
||||
}
|
||||
// Error responses are reconciled by handleGrokAccountUpstreamError, which
|
||||
// also installs the immediate in-memory scheduling block. Successful
|
||||
// responses can still consume the last available request/token, so persist
|
||||
// that exhausted window here as a real rate limit rather than relying only
|
||||
// on the passive snapshot scheduler check.
|
||||
if hasActiveLimit {
|
||||
s.rateLimitGrok(stateCtx, account, resetAt)
|
||||
}
|
||||
}
|
||||
|
||||
func parseGrokQuotaSnapshot(headers http.Header, statusCode int, now time.Time) *xai.QuotaSnapshot {
|
||||
snapshot := xai.ParseQuotaHeaders(headers, statusCode)
|
||||
if snapshot == nil && statusCode == http.StatusTooManyRequests {
|
||||
return &xai.QuotaSnapshot{
|
||||
StatusCode: statusCode,
|
||||
UpdatedAt: now.UTC().Format(time.RFC3339),
|
||||
}
|
||||
}
|
||||
return snapshot
|
||||
}
|
||||
|
||||
func normalizeGrokExhaustedWindowResets(snapshot *xai.QuotaSnapshot, resetAt, now time.Time) {
|
||||
if snapshot == nil || !resetAt.After(now) {
|
||||
return
|
||||
}
|
||||
_ = s.accountRepo.UpdateExtra(ctx, accountID, map[string]any{
|
||||
grokQuotaSnapshotExtraKey: snapshot,
|
||||
})
|
||||
for _, window := range []*xai.QuotaWindow{snapshot.Requests, snapshot.Tokens} {
|
||||
if window == nil || window.Remaining == nil || *window.Remaining > 0 {
|
||||
continue
|
||||
}
|
||||
candidate := time.Time{}
|
||||
if window.ResetUnix != nil && *window.ResetUnix > 0 {
|
||||
candidate = time.Unix(*window.ResetUnix, 0)
|
||||
} else if parsed, err := time.Parse(time.RFC3339, strings.TrimSpace(window.ResetAt)); err == nil {
|
||||
candidate = parsed
|
||||
}
|
||||
if !candidate.After(now) {
|
||||
candidate = resetAt
|
||||
}
|
||||
resetUnix := candidate.Unix()
|
||||
window.ResetUnix = &resetUnix
|
||||
window.ResetAt = candidate.UTC().Format(time.RFC3339)
|
||||
}
|
||||
}
|
||||
|
||||
func grokRateLimitResetAt(snapshot *xai.QuotaSnapshot, now time.Time) (time.Time, bool) {
|
||||
if snapshot == nil {
|
||||
return time.Time{}, false
|
||||
}
|
||||
|
||||
// Retry-After is xAI's explicit retry boundary. Use the observation time so
|
||||
// a persisted snapshot does not start a fresh cooldown every time it is read.
|
||||
retryAfterExpired := false
|
||||
var resetAt time.Time
|
||||
if snapshot.RetryAfterSeconds != nil && *snapshot.RetryAfterSeconds > 0 {
|
||||
observedAt := now
|
||||
if parsed, err := time.Parse(time.RFC3339, strings.TrimSpace(snapshot.UpdatedAt)); err == nil {
|
||||
observedAt = parsed
|
||||
}
|
||||
retryAfterResetAt := observedAt.Add(time.Duration(*snapshot.RetryAfterSeconds) * time.Second)
|
||||
if retryAfterResetAt.After(now) {
|
||||
resetAt = retryAfterResetAt
|
||||
} else {
|
||||
retryAfterExpired = true
|
||||
}
|
||||
}
|
||||
|
||||
exhausted := false
|
||||
for _, window := range []*xai.QuotaWindow{snapshot.Requests, snapshot.Tokens} {
|
||||
if window == nil || window.Remaining == nil || *window.Remaining > 0 {
|
||||
continue
|
||||
}
|
||||
exhausted = true
|
||||
candidate := time.Time{}
|
||||
if window.ResetUnix != nil && *window.ResetUnix > 0 {
|
||||
candidate = time.Unix(*window.ResetUnix, 0)
|
||||
} else if parsed, err := time.Parse(time.RFC3339, strings.TrimSpace(window.ResetAt)); err == nil {
|
||||
candidate = parsed
|
||||
}
|
||||
if candidate.After(now) && candidate.After(resetAt) {
|
||||
resetAt = candidate
|
||||
}
|
||||
}
|
||||
if !resetAt.IsZero() {
|
||||
return resetAt, true
|
||||
}
|
||||
// An observed Retry-After is an absolute boundary once combined with the
|
||||
// snapshot timestamp. Do not turn an expired persisted snapshot into a new
|
||||
// rolling fallback cooldown, but still allow a later explicit window reset.
|
||||
if retryAfterExpired {
|
||||
return time.Time{}, false
|
||||
}
|
||||
if exhausted || snapshot.StatusCode == http.StatusTooManyRequests {
|
||||
return now.Add(grokRateLimitFallbackCooldown), true
|
||||
}
|
||||
return time.Time{}, false
|
||||
}
|
||||
|
||||
func normalizeGrokRateLimitResetAt(account *Account, resetAt, now time.Time) time.Time {
|
||||
if !resetAt.After(now) {
|
||||
resetAt = now.Add(grokRateLimitFallbackCooldown)
|
||||
}
|
||||
if account != nil && account.RateLimitResetAt != nil && account.RateLimitResetAt.After(resetAt) {
|
||||
resetAt = *account.RateLimitResetAt
|
||||
}
|
||||
return resetAt
|
||||
}
|
||||
|
||||
type grokRateLimitExtendingRepository interface {
|
||||
SetRateLimitedIfLater(ctx context.Context, id int64, resetAt time.Time) error
|
||||
}
|
||||
|
||||
func persistGrokRateLimit(ctx context.Context, repo AccountRepository, account *Account, resetAt time.Time) {
|
||||
if repo == nil || account == nil || account.ID <= 0 {
|
||||
return
|
||||
}
|
||||
resetAt = normalizeGrokRateLimitResetAt(account, resetAt, time.Now())
|
||||
stateCtx, cancel := openAIAccountStateContext(ctx)
|
||||
defer cancel()
|
||||
var err error
|
||||
if extendingRepo, ok := repo.(grokRateLimitExtendingRepository); ok {
|
||||
err = extendingRepo.SetRateLimitedIfLater(stateCtx, account.ID, resetAt)
|
||||
} else {
|
||||
err = repo.SetRateLimited(stateCtx, account.ID, resetAt)
|
||||
}
|
||||
if err != nil {
|
||||
slog.Warn("persist_grok_rate_limit_failed", "account_id", account.ID, "reset_at", resetAt.UTC(), "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *OpenAIGatewayService) rateLimitGrok(ctx context.Context, account *Account, resetAt time.Time) {
|
||||
if s == nil || account == nil {
|
||||
return
|
||||
}
|
||||
resetAt = normalizeGrokRateLimitResetAt(account, resetAt, time.Now())
|
||||
|
||||
runtimeUntil := resetAt
|
||||
if account.TempUnschedulableUntil != nil && account.TempUnschedulableUntil.After(runtimeUntil) {
|
||||
runtimeUntil = *account.TempUnschedulableUntil
|
||||
}
|
||||
s.BlockAccountScheduling(account, runtimeUntil, "429")
|
||||
persistGrokRateLimit(ctx, s.accountRepo, account, resetAt)
|
||||
}
|
||||
|
||||
func (s *OpenAIGatewayService) handleGrokAccountUpstreamError(ctx context.Context, account *Account, statusCode int, headers http.Header, responseBody []byte) {
|
||||
if s == nil || account == nil {
|
||||
return
|
||||
}
|
||||
now := time.Now()
|
||||
s.updateGrokUsageSnapshot(ctx, account, parseGrokQuotaSnapshot(headers, statusCode, now))
|
||||
switch statusCode {
|
||||
case http.StatusUnauthorized:
|
||||
s.tempUnscheduleGrok(ctx, account, 10*time.Minute, "grok oauth token unauthorized")
|
||||
case http.StatusForbidden:
|
||||
s.tempUnscheduleGrok(ctx, account, 30*time.Minute, "grok entitlement or subscription tier denied")
|
||||
case http.StatusTooManyRequests:
|
||||
cooldown := 2 * time.Minute
|
||||
if snapshot := xai.ParseQuotaHeaders(headers, statusCode); snapshot != nil && snapshot.RetryAfterSeconds != nil && *snapshot.RetryAfterSeconds > 0 {
|
||||
cooldown = time.Duration(*snapshot.RetryAfterSeconds) * time.Second
|
||||
}
|
||||
s.tempUnscheduleGrok(ctx, account, cooldown, "grok rate limited")
|
||||
// updateGrokUsageSnapshot installs both runtime and durable rate-limit state.
|
||||
default:
|
||||
if statusCode >= 500 {
|
||||
s.tempUnscheduleGrok(ctx, account, 2*time.Minute, "grok upstream temporary error")
|
||||
|
||||
@@ -569,7 +569,7 @@ func TestBindGrokMediaVideoRequestAccountUsesRequestIDStickyHash(t *testing.T) {
|
||||
require.Equal(t, int64(63), accountID)
|
||||
}
|
||||
|
||||
func TestForwardGrokMediaErrorHonorsCustomErrorCodes(t *testing.T) {
|
||||
func TestForwardGrokMedia429ReconcilesRateLimitBeforeCustomErrorBypass(t *testing.T) {
|
||||
t.Setenv(xai.EnvAllowUnsafeURLOverrides, "true")
|
||||
gin.SetMode(gin.TestMode)
|
||||
|
||||
@@ -589,18 +589,20 @@ func TestForwardGrokMediaErrorHonorsCustomErrorCodes(t *testing.T) {
|
||||
"api_key": "api-key",
|
||||
"base_url": "https://xai.test/v1",
|
||||
"custom_error_codes_enabled": true,
|
||||
"custom_error_codes": []any{float64(http.StatusTooManyRequests)},
|
||||
"custom_error_codes": []any{float64(http.StatusBadRequest)},
|
||||
},
|
||||
}
|
||||
upstream := &httpUpstreamRecorder{resp: &http.Response{
|
||||
StatusCode: http.StatusBadRequest,
|
||||
StatusCode: http.StatusTooManyRequests,
|
||||
Header: http.Header{
|
||||
"Content-Type": []string{"application/json"},
|
||||
"Xai-Request-Id": []string{"xai-error-req"},
|
||||
"Retry-After": []string{"45"},
|
||||
},
|
||||
Body: io.NopCloser(strings.NewReader(`{"error":{"message":"do not expose this upstream detail"}}`)),
|
||||
}}
|
||||
svc := &OpenAIGatewayService{httpUpstream: upstream}
|
||||
repo := &grokQuotaAccountRepo{}
|
||||
svc := &OpenAIGatewayService{httpUpstream: upstream, accountRepo: repo}
|
||||
|
||||
result, err := svc.ForwardGrokMedia(context.Background(), c, account, GrokMediaEndpointImagesGenerations, "", body, "application/json")
|
||||
require.Error(t, err)
|
||||
@@ -608,6 +610,9 @@ func TestForwardGrokMediaErrorHonorsCustomErrorCodes(t *testing.T) {
|
||||
require.Equal(t, http.StatusInternalServerError, recorder.Code)
|
||||
require.Contains(t, recorder.Body.String(), "Upstream gateway error")
|
||||
require.NotContains(t, recorder.Body.String(), "do not expose")
|
||||
require.Equal(t, 1, repo.rateLimitedCalls)
|
||||
require.Zero(t, repo.tempUnschedCalls)
|
||||
require.True(t, svc.isOpenAIAccountRuntimeBlocked(account))
|
||||
}
|
||||
|
||||
func TestForwardAsChatCompletionsForGrokUsesXAIChatCompletionsAndSnapshots(t *testing.T) {
|
||||
@@ -1137,7 +1142,7 @@ func grokMessagesSSECompletedResponse(responseID string, cachedTokens int) *http
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleGrokAccountUpstreamErrorTempUnschedulesReadinessStates(t *testing.T) {
|
||||
func TestHandleGrokAccountUpstreamErrorTempUnschedulesNonRateLimitStates(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
status int
|
||||
@@ -1161,12 +1166,11 @@ func TestHandleGrokAccountUpstreamErrorTempUnschedulesReadinessStates(t *testing
|
||||
wantMaxCooldown: 30*time.Minute + time.Second,
|
||||
},
|
||||
{
|
||||
name: "rate limited retry after",
|
||||
status: http.StatusTooManyRequests,
|
||||
headers: http.Header{"Retry-After": []string{"45"}},
|
||||
wantReason: "grok rate limited",
|
||||
wantMinCooldown: 44 * time.Second,
|
||||
wantMaxCooldown: 46 * time.Second,
|
||||
name: "upstream temporary error",
|
||||
status: http.StatusInternalServerError,
|
||||
wantReason: "grok upstream temporary error",
|
||||
wantMinCooldown: 2*time.Minute - time.Second,
|
||||
wantMaxCooldown: 2*time.Minute + time.Second,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -1181,6 +1185,7 @@ func TestHandleGrokAccountUpstreamErrorTempUnschedulesReadinessStates(t *testing
|
||||
|
||||
require.True(t, svc.isOpenAIAccountRuntimeBlocked(account))
|
||||
require.Equal(t, 1, repo.tempUnschedCalls)
|
||||
require.Zero(t, repo.rateLimitedCalls)
|
||||
require.Equal(t, account.ID, repo.lastTempUnschedID)
|
||||
require.Equal(t, tt.wantReason, repo.lastTempUnschedReason)
|
||||
require.True(t, repo.lastTempUnschedUntil.After(before.Add(tt.wantMinCooldown)))
|
||||
@@ -1189,10 +1194,83 @@ func TestHandleGrokAccountUpstreamErrorTempUnschedulesReadinessStates(t *testing
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleGrokAccountUpstreamErrorDoesNotShortenExistingPause(t *testing.T) {
|
||||
func TestHandleGrokAccountUpstreamError429SetsRateLimitedFromRetryAfter(t *testing.T) {
|
||||
account := &Account{ID: 61, Platform: PlatformGrok, Type: AccountTypeOAuth}
|
||||
repo := &grokQuotaAccountRepo{}
|
||||
svc := &OpenAIGatewayService{accountRepo: repo}
|
||||
before := time.Now()
|
||||
|
||||
svc.handleGrokAccountUpstreamError(context.Background(), account, http.StatusTooManyRequests, http.Header{"Retry-After": []string{"45"}}, nil)
|
||||
|
||||
require.True(t, svc.isOpenAIAccountRuntimeBlocked(account))
|
||||
require.Equal(t, 1, repo.rateLimitedCalls)
|
||||
require.Equal(t, account.ID, repo.lastRateLimitedID)
|
||||
require.WithinDuration(t, before.Add(45*time.Second), repo.lastRateLimitResetAt, time.Second)
|
||||
require.Zero(t, repo.tempUnschedCalls)
|
||||
}
|
||||
|
||||
func TestHandleGrokAccountUpstreamError429UsesLatestExhaustedWindowReset(t *testing.T) {
|
||||
now := time.Now()
|
||||
requestReset := now.Add(10 * time.Minute).Truncate(time.Second)
|
||||
tokenReset := now.Add(20 * time.Minute).Truncate(time.Second)
|
||||
headers := http.Header{
|
||||
"X-Ratelimit-Limit-Requests": []string{"10"},
|
||||
"X-Ratelimit-Remaining-Requests": []string{"0"},
|
||||
"X-Ratelimit-Reset-Requests": []string{fmt.Sprintf("%d", requestReset.Unix())},
|
||||
"X-Ratelimit-Limit-Tokens": []string{"1000"},
|
||||
"X-Ratelimit-Remaining-Tokens": []string{"0"},
|
||||
"X-Ratelimit-Reset-Tokens": []string{fmt.Sprintf("%d", tokenReset.Unix())},
|
||||
}
|
||||
account := &Account{ID: 62, Platform: PlatformGrok, Type: AccountTypeOAuth}
|
||||
repo := &grokQuotaAccountRepo{}
|
||||
svc := &OpenAIGatewayService{accountRepo: repo}
|
||||
|
||||
svc.handleGrokAccountUpstreamError(context.Background(), account, http.StatusTooManyRequests, headers, nil)
|
||||
|
||||
require.Equal(t, 1, repo.rateLimitedCalls)
|
||||
require.WithinDuration(t, tokenReset, repo.lastRateLimitResetAt, time.Second)
|
||||
require.Zero(t, repo.tempUnschedCalls)
|
||||
}
|
||||
|
||||
func TestHandleGrokAccountUpstreamError429UsesFallbackReset(t *testing.T) {
|
||||
account := &Account{ID: 63, Platform: PlatformGrok, Type: AccountTypeOAuth}
|
||||
repo := &grokQuotaAccountRepo{}
|
||||
svc := &OpenAIGatewayService{accountRepo: repo}
|
||||
before := time.Now()
|
||||
|
||||
svc.handleGrokAccountUpstreamError(context.Background(), account, http.StatusTooManyRequests, nil, nil)
|
||||
|
||||
require.Equal(t, 1, repo.rateLimitedCalls)
|
||||
require.WithinDuration(t, before.Add(grokRateLimitFallbackCooldown), repo.lastRateLimitResetAt, time.Second)
|
||||
require.Zero(t, repo.tempUnschedCalls)
|
||||
}
|
||||
|
||||
func TestGrokRateLimitResetAtUsesFutureWindowAfterRetryAfterExpires(t *testing.T) {
|
||||
now := time.Now().UTC().Truncate(time.Second)
|
||||
observedAt := now.Add(-2 * time.Minute)
|
||||
windowReset := now.Add(15 * time.Minute)
|
||||
retryAfter := 30
|
||||
snapshot := &xai.QuotaSnapshot{
|
||||
StatusCode: http.StatusTooManyRequests,
|
||||
UpdatedAt: observedAt.Format(time.RFC3339),
|
||||
RetryAfterSeconds: &retryAfter,
|
||||
Requests: &xai.QuotaWindow{
|
||||
Limit: grokInt64PtrForTest(10),
|
||||
Remaining: grokInt64PtrForTest(0),
|
||||
ResetUnix: grokInt64PtrForTest(windowReset.Unix()),
|
||||
},
|
||||
}
|
||||
|
||||
resetAt, limited := grokRateLimitResetAt(snapshot, now)
|
||||
|
||||
require.True(t, limited)
|
||||
require.WithinDuration(t, windowReset, resetAt, time.Second)
|
||||
}
|
||||
|
||||
func TestHandleGrokAccountUpstreamError429DoesNotShortenExistingPause(t *testing.T) {
|
||||
existingUntil := time.Now().Add(15 * time.Minute)
|
||||
account := &Account{
|
||||
ID: 62,
|
||||
ID: 64,
|
||||
Platform: PlatformGrok,
|
||||
Type: AccountTypeOAuth,
|
||||
TempUnschedulableUntil: &existingUntil,
|
||||
@@ -1203,11 +1281,167 @@ func TestHandleGrokAccountUpstreamErrorDoesNotShortenExistingPause(t *testing.T)
|
||||
|
||||
svc.handleGrokAccountUpstreamError(context.Background(), account, http.StatusTooManyRequests, http.Header{"Retry-After": []string{"45"}}, nil)
|
||||
|
||||
require.Equal(t, 1, repo.tempUnschedCalls)
|
||||
require.WithinDuration(t, existingUntil, repo.lastTempUnschedUntil, time.Second)
|
||||
require.Equal(t, 1, repo.rateLimitedCalls)
|
||||
require.WithinDuration(t, time.Now().Add(45*time.Second), repo.lastRateLimitResetAt, time.Second)
|
||||
require.Zero(t, repo.tempUnschedCalls)
|
||||
value, ok := svc.openaiAccountRuntimeBlockUntil.Load(account.ID)
|
||||
require.True(t, ok)
|
||||
runtimeUntil, ok := value.(time.Time)
|
||||
require.True(t, ok)
|
||||
require.WithinDuration(t, existingUntil, runtimeUntil, time.Second)
|
||||
}
|
||||
|
||||
func TestUpdateGrokUsageSnapshotExhaustedSuccessBypassesThrottleAndSetsRateLimited(t *testing.T) {
|
||||
account := &Account{ID: 65, Platform: PlatformGrok, Type: AccountTypeOAuth}
|
||||
repo := &grokQuotaAccountRepo{}
|
||||
svc := &OpenAIGatewayService{
|
||||
accountRepo: repo,
|
||||
codexSnapshotThrottle: newAccountWriteThrottle(time.Hour),
|
||||
}
|
||||
now := time.Now()
|
||||
|
||||
// Consume the normal snapshot write allowance first.
|
||||
svc.updateGrokUsageSnapshot(context.Background(), account, &xai.QuotaSnapshot{
|
||||
StatusCode: http.StatusOK,
|
||||
Requests: &xai.QuotaWindow{
|
||||
Limit: grokInt64PtrForTest(10),
|
||||
Remaining: grokInt64PtrForTest(9),
|
||||
},
|
||||
UpdatedAt: now.UTC().Format(time.RFC3339),
|
||||
})
|
||||
resetAt := now.Add(30 * time.Minute).Truncate(time.Second)
|
||||
svc.updateGrokUsageSnapshot(context.Background(), account, &xai.QuotaSnapshot{
|
||||
StatusCode: http.StatusOK,
|
||||
Requests: &xai.QuotaWindow{
|
||||
Limit: grokInt64PtrForTest(10),
|
||||
Remaining: grokInt64PtrForTest(0),
|
||||
ResetUnix: grokInt64PtrForTest(resetAt.Unix()),
|
||||
ResetAt: resetAt.UTC().Format(time.RFC3339),
|
||||
},
|
||||
UpdatedAt: now.UTC().Format(time.RFC3339),
|
||||
})
|
||||
|
||||
require.Equal(t, 2, repo.updateCalls)
|
||||
require.Equal(t, 1, repo.rateLimitedCalls)
|
||||
require.Equal(t, account.ID, repo.lastRateLimitedID)
|
||||
require.WithinDuration(t, resetAt, repo.lastRateLimitResetAt, time.Second)
|
||||
require.True(t, svc.isOpenAIAccountRuntimeBlocked(account))
|
||||
}
|
||||
|
||||
func TestUpdateGrokUsageSnapshotAvailableSuccessDoesNotSetRateLimited(t *testing.T) {
|
||||
repo := &grokQuotaAccountRepo{}
|
||||
svc := &OpenAIGatewayService{accountRepo: repo}
|
||||
account := &Account{ID: 66, Platform: PlatformGrok, Type: AccountTypeOAuth}
|
||||
|
||||
svc.updateGrokUsageSnapshot(context.Background(), account, &xai.QuotaSnapshot{
|
||||
StatusCode: http.StatusOK,
|
||||
Requests: &xai.QuotaWindow{
|
||||
Limit: grokInt64PtrForTest(10),
|
||||
Remaining: grokInt64PtrForTest(1),
|
||||
},
|
||||
UpdatedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
})
|
||||
|
||||
require.Equal(t, 1, repo.updateCalls)
|
||||
require.Zero(t, repo.rateLimitedCalls)
|
||||
}
|
||||
|
||||
func TestUpdateGrokUsageSnapshotExhaustedSuccessWithoutResetUsesFallback(t *testing.T) {
|
||||
repo := &grokQuotaAccountRepo{}
|
||||
svc := &OpenAIGatewayService{accountRepo: repo}
|
||||
account := &Account{ID: 67, Platform: PlatformGrok, Type: AccountTypeOAuth}
|
||||
before := time.Now()
|
||||
|
||||
svc.updateGrokUsageSnapshot(context.Background(), account, &xai.QuotaSnapshot{
|
||||
StatusCode: http.StatusOK,
|
||||
Tokens: &xai.QuotaWindow{
|
||||
Limit: grokInt64PtrForTest(2_000_000),
|
||||
Remaining: grokInt64PtrForTest(0),
|
||||
},
|
||||
UpdatedAt: before.UTC().Format(time.RFC3339),
|
||||
})
|
||||
|
||||
require.Equal(t, 1, repo.rateLimitedCalls)
|
||||
require.WithinDuration(t, before.Add(grokRateLimitFallbackCooldown), repo.lastRateLimitResetAt, time.Second)
|
||||
stored, ok := repo.updates[account.ID][grokQuotaSnapshotExtraKey].(*xai.QuotaSnapshot)
|
||||
require.True(t, ok)
|
||||
require.NotNil(t, stored.Tokens.ResetUnix)
|
||||
paused, _ := shouldAutoPauseGrokQuotaWindow("tokens", stored.Tokens, before.Add(time.Second))
|
||||
require.True(t, paused)
|
||||
paused, _ = shouldAutoPauseGrokQuotaWindow("tokens", stored.Tokens, repo.lastRateLimitResetAt.Add(time.Second))
|
||||
require.False(t, paused)
|
||||
}
|
||||
|
||||
func TestOpenAIWSHTTPBridgeGrok429PersistsRateLimit(t *testing.T) {
|
||||
repo := &grokQuotaAccountRepo{}
|
||||
upstream := &httpUpstreamRecorder{resp: &http.Response{
|
||||
StatusCode: http.StatusTooManyRequests,
|
||||
Header: http.Header{"Retry-After": []string{"45"}},
|
||||
Body: io.NopCloser(strings.NewReader(`{"error":{"message":"rate limited"}}`)),
|
||||
}}
|
||||
svc := &OpenAIGatewayService{accountRepo: repo, httpUpstream: upstream}
|
||||
account := &Account{ID: 68, Platform: PlatformGrok, Type: AccountTypeOAuth, Concurrency: 1}
|
||||
before := time.Now()
|
||||
|
||||
result, err := svc.proxyOpenAIWSHTTPBridgeTurn(
|
||||
context.Background(), nil, account, "token",
|
||||
[]byte(`{"type":"response.create","model":"grok-4.3","input":"hi"}`),
|
||||
64, "grok-4.3", "", "", "", "cache-id", 1,
|
||||
func([]byte) error { return nil },
|
||||
)
|
||||
|
||||
require.Error(t, err)
|
||||
require.Nil(t, result)
|
||||
require.Equal(t, 1, repo.rateLimitedCalls)
|
||||
require.WithinDuration(t, before.Add(45*time.Second), repo.lastRateLimitResetAt, time.Second)
|
||||
require.Zero(t, repo.tempUnschedCalls)
|
||||
require.True(t, svc.isOpenAIAccountRuntimeBlocked(account))
|
||||
}
|
||||
|
||||
func TestOpenAIWSHTTPBridgeGrokExhaustedSuccessPersistsRateLimit(t *testing.T) {
|
||||
repo := &grokQuotaAccountRepo{}
|
||||
resetAt := time.Now().Add(20 * time.Minute).UTC().Truncate(time.Second)
|
||||
resp := grokMessagesSSECompletedResponse("resp_ws_limited", 0)
|
||||
resp.Header.Set("X-Ratelimit-Limit-Requests", "10")
|
||||
resp.Header.Set("X-Ratelimit-Remaining-Requests", "0")
|
||||
resp.Header.Set("X-Ratelimit-Reset-Requests", fmt.Sprintf("%d", resetAt.Unix()))
|
||||
upstream := &httpUpstreamRecorder{resp: resp}
|
||||
svc := &OpenAIGatewayService{accountRepo: repo, httpUpstream: upstream}
|
||||
account := &Account{ID: 69, Platform: PlatformGrok, Type: AccountTypeOAuth, Concurrency: 1}
|
||||
|
||||
result, err := svc.proxyOpenAIWSHTTPBridgeTurn(
|
||||
context.Background(), nil, account, "token",
|
||||
[]byte(`{"type":"response.create","model":"grok-4.3","input":"hi"}`),
|
||||
64, "grok-4.3", "", "", "", "cache-id", 1,
|
||||
func([]byte) error { return nil },
|
||||
)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, result)
|
||||
require.Equal(t, 1, repo.rateLimitedCalls)
|
||||
require.WithinDuration(t, resetAt, repo.lastRateLimitResetAt, time.Second)
|
||||
require.True(t, svc.isOpenAIAccountRuntimeBlocked(account))
|
||||
}
|
||||
|
||||
func TestFailoverOpenAIUpstreamHTTPErrorUsesOnlyGrokRateLimitPolicy(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
repo := &grokQuotaAccountRepo{}
|
||||
svc := &OpenAIGatewayService{accountRepo: repo}
|
||||
account := &Account{ID: 70, Platform: PlatformGrok, Type: AccountTypeOAuth}
|
||||
resp := &http.Response{
|
||||
StatusCode: http.StatusTooManyRequests,
|
||||
Header: http.Header{"Retry-After": []string{"45"}},
|
||||
}
|
||||
recorder := httptest.NewRecorder()
|
||||
c, _ := gin.CreateTestContext(recorder)
|
||||
c.Request = httptest.NewRequest(http.MethodPost, "/v1/messages", nil)
|
||||
|
||||
failoverErr := svc.failoverOpenAIUpstreamHTTPError(
|
||||
context.Background(), c, account, resp,
|
||||
[]byte(`{"error":{"message":"rate limited"}}`), "rate limited", "grok-4.3",
|
||||
)
|
||||
|
||||
require.NotNil(t, failoverErr)
|
||||
require.Equal(t, 1, repo.rateLimitedCalls)
|
||||
require.Zero(t, repo.tempUnschedCalls)
|
||||
}
|
||||
|
||||
@@ -321,11 +321,6 @@ func (s *OpenAIGatewayService) ForwardAsAnthropic(
|
||||
// 8. Handle error response with failover
|
||||
if resp.StatusCode >= 400 {
|
||||
respBody, upstreamMsg := s.readOpenAIUpstreamError(resp)
|
||||
if account.Platform == PlatformGrok {
|
||||
s.updateGrokUsageSnapshot(ctx, account.ID, xai.ParseQuotaHeaders(resp.Header, resp.StatusCode))
|
||||
s.handleGrokAccountUpstreamError(ctx, account, resp.StatusCode, resp.Header, respBody)
|
||||
}
|
||||
|
||||
if previousResponseID != "" && (isOpenAICompatPreviousResponseNotFound(resp.StatusCode, upstreamMsg, respBody) || isOpenAICompatPreviousResponseUnsupported(resp.StatusCode, upstreamMsg, respBody)) {
|
||||
if isOpenAICompatPreviousResponseUnsupported(resp.StatusCode, upstreamMsg, respBody) {
|
||||
s.disableOpenAICompatSessionContinuation(ctx, c, account, promptCacheKey)
|
||||
@@ -345,6 +340,9 @@ func (s *OpenAIGatewayService) ForwardAsAnthropic(
|
||||
// Non-failover error: return Anthropic-formatted error to client
|
||||
return s.handleAnthropicErrorResponse(resp, c, account, billingModel)
|
||||
}
|
||||
if account.Platform == PlatformGrok && account.Type == AccountTypeOAuth && !account.IsShadow() {
|
||||
s.updateGrokUsageSnapshot(ctx, account, xai.ParseQuotaHeaders(resp.Header, resp.StatusCode))
|
||||
}
|
||||
|
||||
if account.Type == AccountTypeOAuth && promptCacheKey != "" {
|
||||
if turnState := strings.TrimSpace(resp.Header.Get("x-codex-turn-state")); turnState != "" {
|
||||
@@ -392,10 +390,8 @@ func (s *OpenAIGatewayService) ForwardAsAnthropic(
|
||||
|
||||
// Extract and save Codex usage snapshot from response headers (for OAuth accounts).
|
||||
// 排除 spark 影子:其 codex_* 仅由 QueryUsage(/wham/usage bengalfox)更新(外审第7轮 P1)。
|
||||
if handleErr == nil && account.Type == AccountTypeOAuth && !account.IsShadow() {
|
||||
if account.Platform == PlatformGrok {
|
||||
s.updateGrokUsageSnapshot(ctx, account.ID, xai.ParseQuotaHeaders(resp.Header, resp.StatusCode))
|
||||
} else if snapshot := ParseCodexRateLimitHeaders(resp.Header); snapshot != nil {
|
||||
if handleErr == nil && account.Type == AccountTypeOAuth && !account.IsShadow() && account.Platform != PlatformGrok {
|
||||
if snapshot := ParseCodexRateLimitHeaders(resp.Header); snapshot != nil {
|
||||
s.updateCodexUsageSnapshot(ctx, account.ID, snapshot)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/Wei-Shaw/sub2api/internal/config"
|
||||
"github.com/Wei-Shaw/sub2api/internal/pkg/xai"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/tidwall/gjson"
|
||||
)
|
||||
@@ -221,6 +222,9 @@ func (s *OpenAIGatewayService) proxyOpenAIWSHTTPBridgeTurn(
|
||||
|
||||
if resp.StatusCode >= 400 {
|
||||
respBody, _ := io.ReadAll(io.LimitReader(resp.Body, openAIWSHTTPBridgeErrorBodyLimitBytes))
|
||||
if account.Platform == PlatformGrok {
|
||||
s.handleGrokAccountUpstreamError(ctx, account, resp.StatusCode, resp.Header, respBody)
|
||||
}
|
||||
upstreamMsg := sanitizeUpstreamErrorMessage(strings.TrimSpace(extractUpstreamErrorMessage(respBody)))
|
||||
if upstreamMsg == "" {
|
||||
upstreamMsg = http.StatusText(resp.StatusCode)
|
||||
@@ -228,6 +232,9 @@ func (s *OpenAIGatewayService) proxyOpenAIWSHTTPBridgeTurn(
|
||||
_ = writeClientMessage(buildOpenAIWSHTTPBridgeErrorEvent(resp.StatusCode, upstreamMsg))
|
||||
return nil, fmt.Errorf("upstream http bridge error: status=%d message=%s", resp.StatusCode, upstreamMsg)
|
||||
}
|
||||
if account.Platform == PlatformGrok {
|
||||
s.updateGrokUsageSnapshot(ctx, account, xai.ParseQuotaHeaders(resp.Header, resp.StatusCode))
|
||||
}
|
||||
|
||||
responseID := ""
|
||||
usage := OpenAIUsage{}
|
||||
|
||||
@@ -13,6 +13,14 @@ vi.mock('vue-i18n', async () => {
|
||||
}
|
||||
})
|
||||
|
||||
vi.mock('@/utils/format', async () => {
|
||||
const actual = await vi.importActual<typeof import('@/utils/format')>('@/utils/format')
|
||||
return {
|
||||
...actual,
|
||||
formatCountdown: () => '1h'
|
||||
}
|
||||
})
|
||||
|
||||
function makeAccount(overrides: Partial<Account>): Account {
|
||||
return {
|
||||
id: 1,
|
||||
@@ -43,6 +51,31 @@ function makeAccount(overrides: Partial<Account>): Account {
|
||||
}
|
||||
|
||||
describe('AccountStatusIndicator', () => {
|
||||
it('Grok 账号额度限流时显示自动恢复时间而非临时不可调度', () => {
|
||||
const wrapper = mount(AccountStatusIndicator, {
|
||||
props: {
|
||||
account: makeAccount({
|
||||
id: 5,
|
||||
name: 'grok-free-1',
|
||||
platform: 'grok',
|
||||
rate_limited_at: '2026-07-11T12:00:00Z',
|
||||
rate_limit_reset_at: '2099-07-11T13:00:00Z',
|
||||
temp_unschedulable_until: '2099-07-11T12:30:00Z',
|
||||
temp_unschedulable_reason: 'legacy grok rate limited'
|
||||
})
|
||||
},
|
||||
global: {
|
||||
stubs: {
|
||||
Icon: true
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
expect(wrapper.find('.badge-warning').text()).toBe('admin.accounts.status.rateLimited')
|
||||
expect(wrapper.text()).toContain('admin.accounts.status.rateLimitedAutoResume')
|
||||
expect(wrapper.text()).not.toContain('admin.accounts.status.tempUnschedulable')
|
||||
})
|
||||
|
||||
it('模型限流 + overages 启用 + 无 AICredits key → 显示 ⚡ (credits_active)', () => {
|
||||
const wrapper = mount(AccountStatusIndicator, {
|
||||
props: {
|
||||
|
||||
Reference in New Issue
Block a user