mirror of
https://github.com/Tencent/WeKnora.git
synced 2026-10-28 03:29:30 +08:00
fix(wiki): treat rate-limit-shaped 403 responses as transient
Wiki ingest makes several independent LLM calls per document and already retries transient failures (408/429/5xx, timeouts, connection resets) with exponential backoff, but some gateways report QPM/QPS throttling as HTTP 403 instead of 429 (e.g. DeepSeek MaaS: code 0x04030020, message "调用频率(qpm)超限"). Those 403s were classified as permanent caller errors and the wiki span failed immediately with EXTRACT_FAILED / SUMMARY_FAILED, so a QPM burst aborted generation for the whole batch (issue #2688). isTransientLLMError now also matches "status 403" errors whose embedded response body carries rate-limit wording (qpm/qps, rate limit, too many requests, throttled, 调用频率/频率超限/请求过于频繁/繁忙, try again later, slow down). Authorization-shaped 403s (invalid key, forbidden, permission denied) and unrelated bodies stay permanent, so the change cannot turn authz failures into retry loops. Tests: 14-case table covering rate-limit 403 acceptance and authorization-shaped 403 rejection, all existing isTransientLLMError cases unchanged and passing.
This commit is contained in:
@@ -2698,6 +2698,29 @@ func (s *wikiIngestService) awaitWikiPromptWarmup(ctx context.Context, key strin
|
||||
// - Substring matches on the error text for common transport failures
|
||||
// ("timeout", "connection reset", "EOF") that providers surface
|
||||
// without a structured status code.
|
||||
//
|
||||
// rateLimitErrorIndicators are substrings that mark an HTTP 403 response
|
||||
// body as rate limiting rather than authorization failure. Providers embed
|
||||
// the response body in their errors ("API request failed with status 403:
|
||||
// {...}"), and some gateways throttle with 403 (e.g. code 0x04030020,
|
||||
// "调用频率(qpm)超限") instead of the standard 429, so the status alone
|
||||
// is not enough to classify the failure.
|
||||
var rateLimitErrorIndicators = []string{
|
||||
"qpm", // 网关 qpm 配额(0x04030020)
|
||||
"qps", // 网关 qps 配额
|
||||
"rate limit", // OpenAI-style "rate limit reached"
|
||||
"rate_limit",
|
||||
"too many requests", // RFC 6585 language
|
||||
"throttl", // "throttled"
|
||||
"调用频率", // 中文网关常见措辞
|
||||
"频率超限",
|
||||
"请求过于频繁",
|
||||
"繁忙", // "服务繁忙,请稍后重试"
|
||||
"try again later",
|
||||
"retry later",
|
||||
"slow down",
|
||||
}
|
||||
|
||||
func isTransientLLMError(ctx context.Context, err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
@@ -2722,6 +2745,20 @@ func isTransientLLMError(ctx context.Context, err error) bool {
|
||||
}
|
||||
|
||||
lower := strings.ToLower(msg)
|
||||
// Some gateways report QPM/QPS throttling as HTTP 403 instead of 429
|
||||
// (e.g. a MaaS gateway returning code 0x04030020, message
|
||||
// "调用频率(qpm)超限"). A plain 403 is usually an authorization
|
||||
// failure and must NOT be retried, so this stays gated on rate-limit
|
||||
// indicators in the response body, which provider errors embed:
|
||||
// "API request failed with status 403: {"code":0x04030020,...}".
|
||||
if strings.Contains(msg, "status 403") {
|
||||
for _, s := range rateLimitErrorIndicators {
|
||||
if strings.Contains(lower, s) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, s := range []string{
|
||||
"timeout",
|
||||
"timed out",
|
||||
|
||||
@@ -97,3 +97,41 @@ func TestIsTransientLLMError_NilError(t *testing.T) {
|
||||
t.Fatal("nil error should not be transient")
|
||||
}
|
||||
}
|
||||
|
||||
// TestIsTransientLLMError_RateLimit403 covers gateways that report QPM/QPS
|
||||
// throttling as HTTP 403 with a rate-limit body instead of 429 (e.g. a MaaS
|
||||
// gateway returning code 0x04030020, "调用频率(qpm)超限"). These carry the
|
||||
// provider-embedded response body "API request failed with status 403: {...}",
|
||||
// and only body-rated limit wording classifies them as transient — a plain
|
||||
// authorization 403 must stay permanent.
|
||||
func TestIsTransientLLMError_RateLimit403(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cases := []struct {
|
||||
name string
|
||||
msg string
|
||||
want bool
|
||||
}{
|
||||
{"qpm gateway", `API request failed with status 403: {"code":"0x04030020","message":"调用频率(qpm)超限"}`, true},
|
||||
{"qps gateway", `API request failed with status 403: {"message":"qps exceeded"}`, true},
|
||||
{"rate limit", "API request failed with status 403: rate limit reached", true},
|
||||
{"too many requests", "API request failed with status 403: too many requests", true},
|
||||
{"throttled", "API request failed with status 403: request throttled", true},
|
||||
{"chinese frequency", "API request failed with status 403: 请求过于频繁,请稍后重试", true},
|
||||
{"busy", "API request failed with status 403: 服务繁忙", true},
|
||||
{"try again later", "API request failed with status 403: please try again later", true},
|
||||
{"slow down", "API request failed with status 403: slow down your requests", true},
|
||||
// Authorization-shaped 403s stay permanent.
|
||||
{"unauthorized", "API request failed with status 403: invalid api key", false},
|
||||
{"forbidden", "API request failed with status 403: forbidden", false},
|
||||
{"permission denied", "API request failed with status 403: permission denied", false},
|
||||
{"quota exhausted", "API request failed with status 403: forbidden (quota exhausted)", false},
|
||||
{"unknown body", `API request failed with status 403: {"error":"something else"}`, false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := isTransientLLMError(ctx, errors.New(tc.msg)); got != tc.want {
|
||||
t.Errorf("isTransientLLMError(%q) = %v, want %v", tc.msg, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user