fix(coderd/x/chatd): retry quota 429s (#26200)

Fixes CODAGT-495

Provider 429 responses that mention quota or billing were classified as
non-retryable usage limits before the rate-limit rule could run, which
suppressed retries for Gemini and Azure OpenAI rate limits.

- Treats broad quota and billing prose as rate-limit retryable when the provider returns HTTP 429.
- Preserves `insufficient_quota` as a terminal usage-limit signal for OpenAI billing exhaustion.
- Includes structured provider details in usage-limit matching so response-body error codes are classified consistently.

Generated by Coder Agents on behalf of @johnstcn.

coderd/x/chatd/chaterror/classify.go

@@ -204,7 +204,12 @@ func Classify(err error) ClassifiedError {
 	providerDisabledMatch := containsAny(lower, providerDisabledPatterns...)
 	deadline := errors.Is(err, context.DeadlineExceeded) || strings.Contains(lower, "context deadline exceeded")
 	overloadedMatch := statusCode == 529 || containsAny(lower, overloadedPatterns...)
-	usageLimitMatch := containsAny(lower, usageLimitPatterns...)
+	// Usage limits do not have a dedicated status code, so provider
+	// response bodies can be the only reliable signal. Other classes
+	// already have status-code signals or transport wrapper text.
+	usageLimitText := lower + "\n" + strings.ToLower(structured.detail)
+	usageLimitMatch := containsAny(usageLimitText, usageLimitAnyStatusPatterns...) ||
+		(statusCode != 429 && containsAny(usageLimitText, usageLimitPatterns...))
 	authStrong := statusCode == 401 || containsAny(lower, authStrongPatterns...)
 	configMatch := containsAny(lower, configPatterns...)
 	authWeak := statusCode == 403 || containsAny(lower, authWeakPatterns...)
@@ -226,8 +231,8 @@ func Classify(err error) ClassifiedError {
 	// transient-looking errors like "503 invalid model" fail fast.
 	// Overloaded stays ahead because 529/overloaded is a dedicated
 	// provider saturation signal, not a common transport wrapper.
-	// Usage-limit fires before auth so that quota/billing text wins
-	// over whatever HTTP status code the provider happened to use.
+	// Usage-limit fires before auth so non-429 quota/billing text,
+	// plus insufficient_quota at any status, wins over auth signals.
 	// Strong auth still stays above config because bad credentials are
 	// the root cause when both signals appear.
 	// Provider-disabled must precede timeout because disabled providers

coderd/x/chatd/chaterror/classify_test.go

@@ -165,6 +165,17 @@ func TestClassify(t *testing.T) {
 				StatusCode: 429,
 			},
 		},
+		{
+			name: "UsageLimitPatternDoesNotBeatConfigWith429",
+			err:  xerrors.New("status 429: invalid model quota"),
+			want: chaterror.ClassifiedError{
+				Message:    "The AI provider rejected the model configuration. Check the selected model and provider settings.",
+				Kind:       codersdk.ChatErrorKindConfig,
+				Provider:   "",
+				Retryable:  false,
+				StatusCode: 429,
+			},
+		},
 		{
 			name: "ServiceUnavailableClassifiesAsRetryableTimeout",
 			err:  xerrors.New("service unavailable"),
@@ -878,13 +889,6 @@ func TestClassify_UsageLimitBeatsAuth(t *testing.T) {
 			wantKind:  codersdk.ChatErrorKindUsageLimit,
 			wantRetry: false,
 		},
-		{
-			name:       "QuotaWith429Status",
-			err:        "status 429: insufficient_quota",
-			wantKind:   codersdk.ChatErrorKindUsageLimit,
-			wantRetry:  false,
-			wantStatus: 429,
-		},
 		{
 			name:      "PureAuthStillWorks",
 			err:       "unauthorized",
@@ -927,6 +931,97 @@ func TestClassify_UsageLimitBeatsAuth(t *testing.T) {
 	}
 }
 
+func TestClassify_UsageLimitMatchesStructuredDetail(t *testing.T) {
+	t.Parallel()
+
+	classified := chaterror.Classify(testProviderError(
+		"upstream failed",
+		500,
+		nil,
+		testProviderResponseDump(`{"error":{"message":"check your billing plan"}}`),
+	))
+
+	require.Equal(t, codersdk.ChatErrorKindUsageLimit, classified.Kind)
+	require.False(t, classified.Retryable)
+	require.Equal(t, 500, classified.StatusCode)
+	require.Equal(t, "check your billing plan", classified.Detail)
+}
+
+func TestClassify_InsufficientQuotaBeats429RateLimit(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name string
+		err  error
+	}{
+		{
+			name: "StatusText",
+			err:  xerrors.New("status 429: insufficient_quota"),
+		},
+		{
+			name: "StructuredProviderError",
+			err: testProviderError(
+				"upstream failed",
+				429,
+				nil,
+				testProviderResponseDump(`{"error":{"message":"insufficient_quota"}}`),
+			),
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			classified := chaterror.Classify(tt.err)
+			require.Equal(t, codersdk.ChatErrorKindUsageLimit, classified.Kind)
+			require.False(t, classified.Retryable)
+			require.Equal(t, 429, classified.StatusCode)
+		})
+	}
+}
+
+func TestClassify_UsageLimitPatternsDoNotBeat429(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name         string
+		err          error
+		wantProvider string
+	}{
+		{
+			name:         "GoogleGeminiQuotaText",
+			err:          xerrors.New("gemini status 429: Resource has been exhausted (e.g. check quota)."),
+			wantProvider: "google",
+		},
+		{
+			name:         "AzureOpenAIQuotaRemaining",
+			err:          xerrors.New("azure openai exceeded token rate limit; quota remaining: 0; status 429"),
+			wantProvider: "azure",
+		},
+		{
+			name: "BillingPlanRateLimit",
+			err:  xerrors.New("status 429: rate limited: upgrade your billing plan for higher rate limits"),
+		},
+		{
+			name: "StructuredProviderQuotaText",
+			err:  testProviderError("Resource has been exhausted (e.g. check quota).", 429, nil),
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			classified := chaterror.Classify(tt.err)
+			require.Equal(t, codersdk.ChatErrorKindRateLimit, classified.Kind)
+			require.True(t, classified.Retryable)
+			require.Equal(t, 429, classified.StatusCode)
+			require.Equal(t, tt.wantProvider, classified.Provider)
+		})
+	}
+}
+
 // TestClassify_StatusCodeBeatsHTTP2Transport ensures explicit status
 // codes still win over the new HTTP/2 patterns.
 func TestClassify_StatusCodeBeatsHTTP2Transport(t *testing.T) {

coderd/x/chatd/chaterror/signals.go

@@ -69,10 +69,12 @@ var (
 	usageLimitPatterns = []string{
 		"quota",
 		"billing",
-		"insufficient_quota",
 		"payment required",
 	}
-	configPatterns = []string{
+	// Hard usage exhaustion codes that fire at any HTTP status,
+	// including 429.
+	usageLimitAnyStatusPatterns = []string{"insufficient_quota"}
+	configPatterns              = []string{
 		"invalid model",
 		"model not found",
 		"model_not_found",