feat(antigravity): add usage status detection and 403 validation block

- Add needs_verify, is_banned, needs_reauth, error_code fields to UsageInfo
  for Antigravity accounts (omitempty ensures zero impact on other platforms)
- Add singleflight dedup for Antigravity usage requests with independent
  context to prevent caller cancel cascade
- Return degraded UsageInfo with error_code instead of HTTP 500 on errors
- Differentiate cache TTL: success/403 → 3min, recoverable errors → 1min
- Add 403 validation_block: Antigravity validation-type 403 triggers
  temp_unschedulable (10min auto-recovery) instead of permanent SetError
- Frontend: display needs_reauth and degraded error states in usage cell
- Bump version to 0.1.94.4
This commit is contained in:
erio
2026-03-10 14:01:03 +08:00
parent 9a4137add3
commit e2b9c40093
9 changed files with 245 additions and 46 deletions
+1 -1
View File
@@ -1 +1 @@
0.1.94.3
0.1.94.4
+125 -35
View File
@@ -100,6 +100,7 @@ type antigravityUsageCache struct {
const (
apiCacheTTL = 3 * time.Minute
apiErrorCacheTTL = 1 * time.Minute // 负缓存 TTL:429 等错误缓存 1 分钟
antigravityErrorTTL = 1 * time.Minute // Antigravity 错误缓存 TTL(可恢复错误)
apiQueryMaxJitter = 800 * time.Millisecond // 用量查询最大随机延迟
windowStatsCacheTTL = 1 * time.Minute
openAIProbeCacheTTL = 10 * time.Minute
@@ -108,11 +109,12 @@ const (
// UsageCache 封装账户使用量相关的缓存
type UsageCache struct {
apiCache sync.Map // accountID -> *apiUsageCache
windowStatsCache sync.Map // accountID -> *windowStatsCache
antigravityCache sync.Map // accountID -> *antigravityUsageCache
apiFlight singleflight.Group // 防止同一账号的并发请求击穿缓存
openAIProbeCache sync.Map // accountID -> time.Time
apiCache sync.Map // accountID -> *apiUsageCache
windowStatsCache sync.Map // accountID -> *windowStatsCache
antigravityCache sync.Map // accountID -> *antigravityUsageCache
apiFlight singleflight.Group // 防止同一账号的并发请求击穿缓存(Anthropic)
antigravityFlight singleflight.Group // 防止同一 Antigravity 账号的并发请求击穿缓存
openAIProbeCache sync.Map // accountID -> time.Time
}
// NewUsageCache 创建 UsageCache 实例
@@ -193,6 +195,14 @@ type UsageInfo struct {
ForbiddenType string `json:"forbidden_type,omitempty"` // "validation" / "violation" / "forbidden"
ValidationURL string `json:"validation_url,omitempty"` // 验证/申诉链接
// 状态标记(从 ForbiddenType / HTTP 错误码推导)
NeedsVerify bool `json:"needs_verify,omitempty"` // 需要人工验证(forbidden_type=validation)
IsBanned bool `json:"is_banned,omitempty"` // 账号被封(forbidden_type=violation)
NeedsReauth bool `json:"needs_reauth,omitempty"` // token 失效需重新授权(401)
// 错误码(机器可读):forbidden / unauthenticated / rate_limited / network_error
ErrorCode string `json:"error_code,omitempty"`
// 获取 usage 时的错误信息(降级返回,而非 500)
Error string `json:"error,omitempty"`
}
@@ -652,47 +662,127 @@ func (s *AccountUsageService) getAntigravityUsage(ctx context.Context, account *
return &UsageInfo{UpdatedAt: &now}, nil
}
// 1. 检查缓存(10 分钟)
// 1. 检查缓存
if cached, ok := s.cache.antigravityCache.Load(account.ID); ok {
if cache, ok := cached.(*antigravityUsageCache); ok && time.Since(cache.timestamp) < apiCacheTTL {
// 重新计算 RemainingSeconds
usage := cache.usageInfo
if usage.FiveHour != nil && usage.FiveHour.ResetsAt != nil {
usage.FiveHour.RemainingSeconds = int(time.Until(*usage.FiveHour.ResetsAt).Seconds())
if cache, ok := cached.(*antigravityUsageCache); ok {
ttl := antigravityCacheTTL(cache.usageInfo)
if time.Since(cache.timestamp) < ttl {
usage := cache.usageInfo
if usage.FiveHour != nil && usage.FiveHour.ResetsAt != nil {
usage.FiveHour.RemainingSeconds = int(time.Until(*usage.FiveHour.ResetsAt).Seconds())
}
return usage, nil
}
return usage, nil
}
}
// 2. 获取代理 URL
proxyURL := s.antigravityQuotaFetcher.GetProxyURL(ctx, account)
// 3. 调用 API 获取额度
result, err := s.antigravityQuotaFetcher.FetchQuota(ctx, account, proxyURL)
if err != nil {
// 降级返回带 error 字段的 UsageInfo,而非 500
now := time.Now()
errMsg := fmt.Sprintf("usage API error: %v", err)
slog.Warn("antigravity usage fetch failed, returning degraded response",
"account_id", account.ID, "error", err)
degraded := &UsageInfo{
UpdatedAt: &now,
Error: errMsg,
// 2. singleflight 防止并发击穿
flightKey := fmt.Sprintf("ag-usage:%d", account.ID)
result, flightErr, _ := s.cache.antigravityFlight.Do(flightKey, func() (any, error) {
// 再次检查缓存(等待期间可能已被填充)
if cached, ok := s.cache.antigravityCache.Load(account.ID); ok {
if cache, ok := cached.(*antigravityUsageCache); ok {
ttl := antigravityCacheTTL(cache.usageInfo)
if time.Since(cache.timestamp) < ttl {
usage := cache.usageInfo
// 重新计算 RemainingSeconds,避免返回过时的剩余秒数
recalcAntigravityRemainingSeconds(usage)
return usage, nil
}
}
}
// 使用独立 context,避免调用方 cancel 导致所有共享 flight 的请求失败
fetchCtx, fetchCancel := context.WithTimeout(context.Background(), 30*time.Second)
defer fetchCancel()
proxyURL := s.antigravityQuotaFetcher.GetProxyURL(fetchCtx, account)
fetchResult, err := s.antigravityQuotaFetcher.FetchQuota(fetchCtx, account, proxyURL)
if err != nil {
degraded := buildAntigravityDegradedUsage(err)
s.cache.antigravityCache.Store(account.ID, &antigravityUsageCache{
usageInfo: degraded,
timestamp: time.Now(),
})
return degraded, nil
}
s.cache.antigravityCache.Store(account.ID, &antigravityUsageCache{
usageInfo: degraded,
usageInfo: fetchResult.UsageInfo,
timestamp: time.Now(),
})
return degraded, nil
}
// 4. 缓存结果
s.cache.antigravityCache.Store(account.ID, &antigravityUsageCache{
usageInfo: result.UsageInfo,
timestamp: time.Now(),
return fetchResult.UsageInfo, nil
})
return result.UsageInfo, nil
if flightErr != nil {
return nil, flightErr
}
usage, ok := result.(*UsageInfo)
if !ok || usage == nil {
now := time.Now()
return &UsageInfo{UpdatedAt: &now}, nil
}
return usage, nil
}
// recalcAntigravityRemainingSeconds 重新计算 Antigravity UsageInfo 中各窗口的 RemainingSeconds
// 用于从缓存取出时更新倒计时,避免返回过时的剩余秒数
func recalcAntigravityRemainingSeconds(info *UsageInfo) {
if info == nil {
return
}
if info.FiveHour != nil && info.FiveHour.ResetsAt != nil {
remaining := int(time.Until(*info.FiveHour.ResetsAt).Seconds())
if remaining < 0 {
remaining = 0
}
info.FiveHour.RemainingSeconds = remaining
}
}
// antigravityCacheTTL 根据 UsageInfo 内容决定缓存 TTL
// 403 forbidden 状态稳定,缓存与成功相同(3 分钟);
// 其他错误(401/网络)可能快速恢复,缓存 1 分钟。
func antigravityCacheTTL(info *UsageInfo) time.Duration {
if info == nil {
return antigravityErrorTTL
}
if info.IsForbidden {
return apiCacheTTL // 封号/验证状态不会很快变
}
if info.ErrorCode != "" || info.Error != "" {
return antigravityErrorTTL
}
return apiCacheTTL
}
// buildAntigravityDegradedUsage 从 FetchQuota 错误构建降级 UsageInfo
func buildAntigravityDegradedUsage(err error) *UsageInfo {
now := time.Now()
errMsg := fmt.Sprintf("usage API error: %v", err)
slog.Warn("antigravity usage fetch failed, returning degraded response", "error", err)
info := &UsageInfo{
UpdatedAt: &now,
Error: errMsg,
}
// 从错误信息推断 error_code 和状态标记
// 错误格式来自 antigravity/client.go: "fetchAvailableModels 失败 (HTTP %d): ..."
errStr := err.Error()
switch {
case strings.Contains(errStr, "HTTP 401") ||
strings.Contains(errStr, "UNAUTHENTICATED") ||
strings.Contains(errStr, "invalid_grant"):
info.ErrorCode = errorCodeUnauthenticated
info.NeedsReauth = true
case strings.Contains(errStr, "HTTP 429"):
info.ErrorCode = errorCodeRateLimited
default:
info.ErrorCode = errorCodeNetworkError
}
return info
}
// addWindowStats 为 usage 数据添加窗口期统计
@@ -17,6 +17,12 @@ const (
forbiddenTypeValidation = "validation"
forbiddenTypeViolation = "violation"
forbiddenTypeForbidden = "forbidden"
// 机器可读的错误码
errorCodeForbidden = "forbidden"
errorCodeUnauthenticated = "unauthenticated"
errorCodeRateLimited = "rate_limited"
errorCodeNetworkError = "network_error"
)
// AntigravityQuotaFetcher 从 Antigravity API 获取额度
@@ -55,13 +61,17 @@ func (f *AntigravityQuotaFetcher) FetchQuota(ctx context.Context, account *Accou
var forbiddenErr *antigravity.ForbiddenError
if errors.As(err, &forbiddenErr) {
now := time.Now()
fbType := classifyForbiddenType(forbiddenErr.Body)
return &QuotaResult{
UsageInfo: &UsageInfo{
UpdatedAt: &now,
IsForbidden: true,
ForbiddenReason: forbiddenErr.Body,
ForbiddenType: classifyForbiddenType(forbiddenErr.Body),
ForbiddenType: fbType,
ValidationURL: extractValidationURL(forbiddenErr.Body),
NeedsVerify: fbType == forbiddenTypeValidation,
IsBanned: fbType == forbiddenTypeViolation,
ErrorCode: errorCodeForbidden,
},
}, nil
}
+67 -7
View File
@@ -200,11 +200,6 @@ func (s *RateLimitService) HandleUpstreamError(ctx context.Context, account *Acc
s.handleAuthError(ctx, account, msg)
shouldDisable = true
case 403:
// 禁止访问:停止调度,记录错误
msg := "Access forbidden (403): account may be suspended or lack permissions"
if upstreamMsg != "" {
msg = "Access forbidden (403): " + upstreamMsg
}
logger.LegacyPrintf(
"service.ratelimit",
"[HandleUpstreamErrorRaw] account_id=%d platform=%s type=%s status=403 request_id=%s cf_ray=%s upstream_msg=%s raw_body=%s",
@@ -216,8 +211,7 @@ func (s *RateLimitService) HandleUpstreamError(ctx context.Context, account *Acc
upstreamMsg,
truncateForLog(responseBody, 1024),
)
s.handleAuthError(ctx, account, msg)
shouldDisable = true
shouldDisable = s.handle403(ctx, account, upstreamMsg, responseBody)
case 429:
s.handle429(ctx, account, headers, responseBody)
shouldDisable = false
@@ -622,6 +616,72 @@ func (s *RateLimitService) handleAuthError(ctx context.Context, account *Account
slog.Warn("account_disabled_auth_error", "account_id", account.ID, "error", errorMsg)
}
// antigravityValidationBlockDuration Antigravity validation 类型 403 的临时封禁时长
// 与 Antigravity-Manager 一致(10 分钟)
const antigravityValidationBlockDuration = 10 * time.Minute
// handle403 处理 403 Forbidden 错误
// Antigravity 平台区分 validation(临时不可调度)和 violation(永久 SetError);
// 其他平台保持原有 SetError 行为。
func (s *RateLimitService) handle403(ctx context.Context, account *Account, upstreamMsg string, responseBody []byte) (shouldDisable bool) {
if account.Platform == PlatformAntigravity {
return s.handleAntigravity403(ctx, account, upstreamMsg, responseBody)
}
// 非 Antigravity 平台:保持原有行为
msg := "Access forbidden (403): account may be suspended or lack permissions"
if upstreamMsg != "" {
msg = "Access forbidden (403): " + upstreamMsg
}
s.handleAuthError(ctx, account, msg)
return true
}
// handleAntigravity403 处理 Antigravity 平台的 403 错误
// validation(需要验证)→ 临时不可调度(到期自动恢复)
// violation(违规封号)→ 永久 SetError(需人工处理)
// generic(通用禁止)→ 永久 SetError
func (s *RateLimitService) handleAntigravity403(ctx context.Context, account *Account, upstreamMsg string, responseBody []byte) (shouldDisable bool) {
fbType := classifyForbiddenType(string(responseBody))
switch fbType {
case forbiddenTypeValidation:
// VALIDATION_REQUIRED: 临时不可调度,到期自动恢复
until := time.Now().Add(antigravityValidationBlockDuration)
reason := "Validation required (403)"
if upstreamMsg != "" {
reason = "Validation required (403): " + upstreamMsg
}
if err := s.accountRepo.SetTempUnschedulable(ctx, account.ID, until, reason); err != nil {
slog.Warn("antigravity_validation_block_failed", "account_id", account.ID, "error", err)
} else {
slog.Info("antigravity_validation_block_set",
"account_id", account.ID,
"until", until,
"forbidden_type", fbType,
)
}
return true
case forbiddenTypeViolation:
// 违规封号: 永久禁用,需人工处理
msg := "Account violation (403): terms of service violation"
if upstreamMsg != "" {
msg = "Account violation (403): " + upstreamMsg
}
s.handleAuthError(ctx, account, msg)
return true
default:
// 通用 403: 保持原有行为
msg := "Access forbidden (403): account may be suspended or lack permissions"
if upstreamMsg != "" {
msg = "Access forbidden (403): " + upstreamMsg
}
s.handleAuthError(ctx, account, msg)
return true
}
}
// handleCustomErrorCode 处理自定义错误码,停止账号调度
func (s *RateLimitService) handleCustomErrorCode(ctx context.Context, account *Account, statusCode int, errorMsg string) {
msg := "Custom error code " + strconv.Itoa(statusCode) + ": " + errorMsg
@@ -335,7 +335,7 @@ func (s *SoraTaskService) forwardCreateToUpstream(
if err != nil {
return nil, fmt.Errorf("upstream request: %w", err)
}
defer resp.Body.Close()
defer func() { _ = resp.Body.Close() }()
respBody, err := io.ReadAll(io.LimitReader(resp.Body, 256*1024))
if err != nil {
@@ -224,6 +224,20 @@
</div>
</div>
<!-- Needs reauth (401) -->
<div v-else-if="needsReauth" class="space-y-1">
<span class="inline-block rounded px-1.5 py-0.5 text-[10px] font-medium bg-orange-100 text-orange-700 dark:bg-orange-900/40 dark:text-orange-300">
{{ t('admin.accounts.needsReauth') }}
</span>
</div>
<!-- Degraded error (non-403, non-401) -->
<div v-else-if="usageInfo?.error" class="space-y-1">
<span class="inline-block rounded px-1.5 py-0.5 text-[10px] font-medium bg-amber-100 text-amber-700 dark:bg-amber-900/40 dark:text-amber-300">
{{ usageErrorLabel }}
</span>
</div>
<!-- Loading state -->
<div v-else-if="loading" class="space-y-1.5">
<div class="flex items-center gap-1">
@@ -863,6 +877,16 @@ const isForbidden = computed(() => !!usageInfo.value?.is_forbidden)
const forbiddenType = computed(() => usageInfo.value?.forbidden_type || 'forbidden')
const validationURL = computed(() => usageInfo.value?.validation_url || '')
// 需要重新授权(401)
const needsReauth = computed(() => !!usageInfo.value?.needs_reauth)
// 降级错误标签(rate_limited / network_error)
const usageErrorLabel = computed(() => {
const code = usageInfo.value?.error_code
if (code === 'rate_limited') return t('admin.accounts.rateLimited')
return t('admin.accounts.usageError')
})
const forbiddenLabel = computed(() => {
switch (forbiddenType.value) {
case 'validation':
+4 -1
View File
@@ -2505,7 +2505,10 @@ export default {
forbiddenViolation: 'Violation Ban',
openVerification: 'Open Verification Link',
copyLink: 'Copy Link',
linkCopied: 'Link Copied'
linkCopied: 'Link Copied',
needsReauth: 'Re-auth Required',
rateLimited: 'Rate Limited',
usageError: 'Fetch Error'
},
// Scheduled Tests
+3
View File
@@ -1948,6 +1948,9 @@ export default {
openVerification: '打开验证链接',
copyLink: '复制链接',
linkCopied: '链接已复制',
needsReauth: '需要重新授权',
rateLimited: '限流中',
usageError: '获取失败',
form: {
nameLabel: '账号名称',
namePlaceholder: '请输入账号名称',
+9
View File
@@ -784,6 +784,15 @@ export interface AccountUsageInfo {
forbidden_reason?: string
forbidden_type?: string // "validation" | "violation" | "forbidden"
validation_url?: string // 验证/申诉链接
// 状态标记(后端自动推导)
needs_verify?: boolean // 需要人工验证(forbidden_type=validation)
is_banned?: boolean // 账号被封(forbidden_type=violation)
needs_reauth?: boolean // token 失效需重新授权(401)
// 机器可读错误码:forbidden / unauthenticated / rate_limited / network_error
error_code?: string
error?: string // usage 获取失败时的错误信息
}