fix(wiki): treat rate-limit-shaped 403 responses as transient

Wiki ingest makes several independent LLM calls per document and already
retries transient failures (408/429/5xx, timeouts, connection resets) with
exponential backoff, but some gateways report QPM/QPS throttling as HTTP
403 instead of 429 (e.g. DeepSeek MaaS: code 0x04030020, message
"调用频率(qpm)超限"). Those 403s were classified as permanent caller
errors and the wiki span failed immediately with EXTRACT_FAILED /
SUMMARY_FAILED, so a QPM burst aborted generation for the whole batch
(issue #2688).

isTransientLLMError now also matches "status 403" errors whose embedded
response body carries rate-limit wording (qpm/qps, rate limit, too many
requests, throttled, 调用频率/频率超限/请求过于频繁/繁忙, try again
later, slow down). Authorization-shaped 403s (invalid key, forbidden,
permission denied) and unrelated bodies stay permanent, so the change
cannot turn authz failures into retry loops.

Tests: 14-case table covering rate-limit 403 acceptance and
authorization-shaped 403 rejection, all existing isTransientLLMError
cases unchanged and passing.
This commit is contained in:
LuckVd
2026-09-07 14:41:48 +08:00
committed by lyingbug
parent 2544fedcd3
commit 25fcbb9401
2 changed files with 75 additions and 0 deletions
@@ -2698,6 +2698,29 @@ func (s *wikiIngestService) awaitWikiPromptWarmup(ctx context.Context, key strin
// - Substring matches on the error text for common transport failures
// ("timeout", "connection reset", "EOF") that providers surface
// without a structured status code.
//
// rateLimitErrorIndicators are substrings that mark an HTTP 403 response
// body as rate limiting rather than authorization failure. Providers embed
// the response body in their errors ("API request failed with status 403:
// {...}"), and some gateways throttle with 403 (e.g. code 0x04030020,
// "调用频率(qpm)超限") instead of the standard 429, so the status alone
// is not enough to classify the failure.
var rateLimitErrorIndicators = []string{
"qpm", // 网关 qpm 配额(0x04030020)
"qps", // 网关 qps 配额
"rate limit", // OpenAI-style "rate limit reached"
"rate_limit",
"too many requests", // RFC 6585 language
"throttl", // "throttled"
"调用频率", // 中文网关常见措辞
"频率超限",
"请求过于频繁",
"繁忙", // "服务繁忙,请稍后重试"
"try again later",
"retry later",
"slow down",
}
func isTransientLLMError(ctx context.Context, err error) bool {
if err == nil {
return false
@@ -2722,6 +2745,20 @@ func isTransientLLMError(ctx context.Context, err error) bool {
}
lower := strings.ToLower(msg)
// Some gateways report QPM/QPS throttling as HTTP 403 instead of 429
// (e.g. a MaaS gateway returning code 0x04030020, message
// "调用频率(qpm)超限"). A plain 403 is usually an authorization
// failure and must NOT be retried, so this stays gated on rate-limit
// indicators in the response body, which provider errors embed:
// "API request failed with status 403: {"code":0x04030020,...}".
if strings.Contains(msg, "status 403") {
for _, s := range rateLimitErrorIndicators {
if strings.Contains(lower, s) {
return true
}
}
}
for _, s := range []string{
"timeout",
"timed out",
@@ -97,3 +97,41 @@ func TestIsTransientLLMError_NilError(t *testing.T) {
t.Fatal("nil error should not be transient")
}
}
// TestIsTransientLLMError_RateLimit403 covers gateways that report QPM/QPS
// throttling as HTTP 403 with a rate-limit body instead of 429 (e.g. a MaaS
// gateway returning code 0x04030020, "调用频率(qpm)超限"). These carry the
// provider-embedded response body "API request failed with status 403: {...}",
// and only body-rated limit wording classifies them as transient — a plain
// authorization 403 must stay permanent.
func TestIsTransientLLMError_RateLimit403(t *testing.T) {
ctx := context.Background()
cases := []struct {
name string
msg string
want bool
}{
{"qpm gateway", `API request failed with status 403: {"code":"0x04030020","message":"调用频率(qpm)超限"}`, true},
{"qps gateway", `API request failed with status 403: {"message":"qps exceeded"}`, true},
{"rate limit", "API request failed with status 403: rate limit reached", true},
{"too many requests", "API request failed with status 403: too many requests", true},
{"throttled", "API request failed with status 403: request throttled", true},
{"chinese frequency", "API request failed with status 403: 请求过于频繁,请稍后重试", true},
{"busy", "API request failed with status 403: 服务繁忙", true},
{"try again later", "API request failed with status 403: please try again later", true},
{"slow down", "API request failed with status 403: slow down your requests", true},
// Authorization-shaped 403s stay permanent.
{"unauthorized", "API request failed with status 403: invalid api key", false},
{"forbidden", "API request failed with status 403: forbidden", false},
{"permission denied", "API request failed with status 403: permission denied", false},
{"quota exhausted", "API request failed with status 403: forbidden (quota exhausted)", false},
{"unknown body", `API request failed with status 403: {"error":"something else"}`, false},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
if got := isTransientLLMError(ctx, errors.New(tc.msg)); got != tc.want {
t.Errorf("isTransientLLMError(%q) = %v, want %v", tc.msg, got, tc.want)
}
})
}
}