mirror of
https://github.com/coder/coder.git
synced 2026-09-24 15:04:27 +08:00
feat(chatd): add LLM stream retry with exponential backoff (#22418)
## Summary Adds automatic retry with exponential backoff for transient LLM errors during chat streaming and title generation. Inspired by [coder/mux](https://github.com/coder/mux)'s retry mechanism. ## Key Behaviors - **Infinite retries** with exponential backoff: 1s → 2s → 4s → ... → 60s cap - **Deterministic delays** (no jitter) - **Error classification**: retryable (429, 5xx, overloaded, rate limit, network errors) vs non-retryable (auth, quota, context exceeded, model not found, canceled) - **Retry status published to SSE stream** so frontend can show "Retrying in Xs..." UI - **Title generation** retries silently (best-effort, nil onRetry callback) ## New Package: `coderd/chatd/chatretry/` | File | Purpose | |------|---------| | `classify.go` | `IsRetryable(err)` and `StatusCodeRetryable(code)` | | `backoff.go` | `Delay(attempt)` — exponential doubling with 60s cap | | `retry.go` | `Retry(ctx, fn, onRetry)` — infinite loop with context-aware timer | ## Test Helpers: `coderd/chatd/chattest/errors.go` Anthropic and OpenAI error response builders for use in chattest providers: - `AnthropicErrorResponse()`, `AnthropicOverloadedResponse()`, `AnthropicRateLimitResponse()` - `OpenAIErrorResponse()`, `OpenAIRateLimitResponse()`, `OpenAIServerErrorResponse()` ## SDK Changes: `codersdk/chats.go` - New `ChatStreamEventType: "retry"` - New `ChatStreamRetry` struct with `Attempt`, `DelayMs`, `Error`, `RetryingAt` fields - TypeScript types auto-generated ## Changed Files - `coderd/chatd/chatloop/chatloop.go` — wraps `agent.Stream()` in `chatretry.Retry()` - `coderd/chatd/chatd.go` — publishes retry events to SSE stream with logging - `coderd/chatd/title.go` — wraps `model.Generate()` in silent retry - `coderd/chatd/chattest/anthropic.go` / `openai.go` — error injection support ## Tests 42 tests covering classification (33), backoff (9), and retry scenarios (8).
This commit is contained in:
@@ -8,6 +8,7 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"charm.land/fantasy"
|
||||
fantasyanthropic "charm.land/fantasy/providers/anthropic"
|
||||
@@ -15,6 +16,7 @@ import (
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
"github.com/coder/coder/v2/coderd/chatd/chatprompt"
|
||||
"github.com/coder/coder/v2/coderd/chatd/chatretry"
|
||||
"github.com/coder/coder/v2/codersdk"
|
||||
)
|
||||
|
||||
@@ -54,6 +56,12 @@ type RunOptions struct {
|
||||
)
|
||||
Compaction *CompactionOptions
|
||||
|
||||
// OnRetry is called before each retry attempt when the LLM
|
||||
// stream fails with a retryable error. It provides the attempt
|
||||
// number, error, and backoff delay so callers can publish status
|
||||
// events to connected clients.
|
||||
OnRetry chatretry.OnRetryFn
|
||||
|
||||
OnInterruptedPersistError func(error)
|
||||
}
|
||||
|
||||
@@ -443,15 +451,39 @@ func Run(ctx context.Context, opts RunOptions) (*fantasy.AgentResult, error) {
|
||||
})
|
||||
}
|
||||
|
||||
result, err := agent.Stream(ctx, streamCall)
|
||||
if err != nil {
|
||||
if errors.Is(err, context.Canceled) &&
|
||||
errors.Is(context.Cause(ctx), ErrInterrupted) {
|
||||
if persistErr := persistInterruptedStep(); persistErr != nil {
|
||||
if opts.OnInterruptedPersistError != nil {
|
||||
opts.OnInterruptedPersistError(persistErr)
|
||||
var result *fantasy.AgentResult
|
||||
err := chatretry.Retry(ctx, func(retryCtx context.Context) error {
|
||||
var streamErr error
|
||||
result, streamErr = agent.Stream(retryCtx, streamCall)
|
||||
if streamErr != nil {
|
||||
// Interrupts are not retryable — propagate them
|
||||
// immediately so processChat can set the correct
|
||||
// status.
|
||||
if errors.Is(streamErr, context.Canceled) &&
|
||||
errors.Is(context.Cause(retryCtx), ErrInterrupted) {
|
||||
if persistErr := persistInterruptedStep(); persistErr != nil {
|
||||
if opts.OnInterruptedPersistError != nil {
|
||||
opts.OnInterruptedPersistError(persistErr)
|
||||
}
|
||||
}
|
||||
// Return ErrInterrupted directly so the retry
|
||||
// loop sees a non-retryable error and stops.
|
||||
return ErrInterrupted
|
||||
}
|
||||
return streamErr
|
||||
}
|
||||
return nil
|
||||
}, func(attempt int, retryErr error, delay time.Duration) {
|
||||
// Reset accumulated draft state from the failed attempt
|
||||
// so the next attempt starts clean.
|
||||
resetStepState()
|
||||
|
||||
if opts.OnRetry != nil {
|
||||
opts.OnRetry(attempt, retryErr, delay)
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrInterrupted) {
|
||||
return nil, ErrInterrupted
|
||||
}
|
||||
return nil, xerrors.Errorf("stream response: %w", err)
|
||||
|
||||
Reference in New Issue
Block a user