mirror of
https://github.com/coder/coder.git
synced 2026-09-24 15:04:27 +08:00
feat(codersdk): add circuit breaker configuration support for aibridge (#21546)
## Summary Add circuit breaker support for AI Bridge to protect against cascading failures from upstream AI provider rate limits (HTTP 429, 503, and Anthropic's 529 overloaded responses). ## Changes - Add 5 new CLI options for circuit breaker configuration: - `--aibridge-circuit-breaker-enabled` (default: false) - `--aibridge-circuit-breaker-failure-threshold` (default: 5) - `--aibridge-circuit-breaker-interval` (default: 10s) - `--aibridge-circuit-breaker-timeout` (default: 30s) - `--aibridge-circuit-breaker-max-requests` (default: 3) - Update aibridge dependency to include circuit breaker support - Add tests for pool creation with circuit breaker providers ## Notes - Circuit breaker is **disabled by default** for backward compatibility - When enabled, applies to both OpenAI and Anthropic providers - Uses sony/gobreaker internally via the aibridge library ## Testing ``` make test RUN=TestPoolWithCircuitBreakerProviders ```
This commit is contained in:
@@ -3494,6 +3494,72 @@ Write out the current server config as YAML to stdout.`,
|
||||
Group: &deploymentGroupAIBridge,
|
||||
YAML: "structuredLogging",
|
||||
},
|
||||
{
|
||||
Name: "AI Bridge Circuit Breaker Enabled",
|
||||
Description: "Enable the circuit breaker to protect against cascading failures from upstream AI provider rate limits (429, 503, 529 overloaded).",
|
||||
Flag: "aibridge-circuit-breaker-enabled",
|
||||
Env: "CODER_AIBRIDGE_CIRCUIT_BREAKER_ENABLED",
|
||||
Value: &c.AI.BridgeConfig.CircuitBreakerEnabled,
|
||||
Default: "false",
|
||||
Group: &deploymentGroupAIBridge,
|
||||
YAML: "circuitBreakerEnabled",
|
||||
},
|
||||
{
|
||||
Name: "AI Bridge Circuit Breaker Failure Threshold",
|
||||
Description: "Number of consecutive failures that triggers the circuit breaker to open.",
|
||||
Flag: "aibridge-circuit-breaker-failure-threshold",
|
||||
Env: "CODER_AIBRIDGE_CIRCUIT_BREAKER_FAILURE_THRESHOLD",
|
||||
Value: serpent.Validate(&c.AI.BridgeConfig.CircuitBreakerFailureThreshold, func(value *serpent.Int64) error {
|
||||
if value.Value() <= 0 || value.Value() > 100 {
|
||||
return xerrors.New("must be between 1 and 100")
|
||||
}
|
||||
return nil
|
||||
}),
|
||||
Default: "5",
|
||||
Hidden: true,
|
||||
Group: &deploymentGroupAIBridge,
|
||||
YAML: "circuitBreakerFailureThreshold",
|
||||
},
|
||||
{
|
||||
Name: "AI Bridge Circuit Breaker Interval",
|
||||
Description: "Cyclic period of the closed state for clearing internal failure counts.",
|
||||
Flag: "aibridge-circuit-breaker-interval",
|
||||
Env: "CODER_AIBRIDGE_CIRCUIT_BREAKER_INTERVAL",
|
||||
Value: &c.AI.BridgeConfig.CircuitBreakerInterval,
|
||||
Default: "10s",
|
||||
Hidden: true,
|
||||
Group: &deploymentGroupAIBridge,
|
||||
YAML: "circuitBreakerInterval",
|
||||
Annotations: serpent.Annotations{}.Mark(annotationFormatDuration, "true"),
|
||||
},
|
||||
{
|
||||
Name: "AI Bridge Circuit Breaker Timeout",
|
||||
Description: "How long the circuit breaker stays open before transitioning to half-open state.",
|
||||
Flag: "aibridge-circuit-breaker-timeout",
|
||||
Env: "CODER_AIBRIDGE_CIRCUIT_BREAKER_TIMEOUT",
|
||||
Value: &c.AI.BridgeConfig.CircuitBreakerTimeout,
|
||||
Default: "30s",
|
||||
Hidden: true,
|
||||
Group: &deploymentGroupAIBridge,
|
||||
YAML: "circuitBreakerTimeout",
|
||||
Annotations: serpent.Annotations{}.Mark(annotationFormatDuration, "true"),
|
||||
},
|
||||
{
|
||||
Name: "AI Bridge Circuit Breaker Max Requests",
|
||||
Description: "Maximum number of requests allowed in half-open state before deciding to close or re-open the circuit.",
|
||||
Flag: "aibridge-circuit-breaker-max-requests",
|
||||
Env: "CODER_AIBRIDGE_CIRCUIT_BREAKER_MAX_REQUESTS",
|
||||
Value: serpent.Validate(&c.AI.BridgeConfig.CircuitBreakerMaxRequests, func(value *serpent.Int64) error {
|
||||
if value.Value() <= 0 || value.Value() > 100 {
|
||||
return xerrors.New("must be between 1 and 100")
|
||||
}
|
||||
return nil
|
||||
}),
|
||||
Default: "3",
|
||||
Hidden: true,
|
||||
Group: &deploymentGroupAIBridge,
|
||||
YAML: "circuitBreakerMaxRequests",
|
||||
},
|
||||
|
||||
// AI Bridge Proxy Options
|
||||
{
|
||||
@@ -3641,6 +3707,13 @@ type AIBridgeConfig struct {
|
||||
MaxConcurrency serpent.Int64 `json:"max_concurrency" typescript:",notnull"`
|
||||
RateLimit serpent.Int64 `json:"rate_limit" typescript:",notnull"`
|
||||
StructuredLogging serpent.Bool `json:"structured_logging" typescript:",notnull"`
|
||||
// Circuit breaker protects against cascading failures from upstream AI
|
||||
// provider rate limits (429, 503, 529 overloaded).
|
||||
CircuitBreakerEnabled serpent.Bool `json:"circuit_breaker_enabled" typescript:",notnull"`
|
||||
CircuitBreakerFailureThreshold serpent.Int64 `json:"circuit_breaker_failure_threshold" typescript:",notnull"`
|
||||
CircuitBreakerInterval serpent.Duration `json:"circuit_breaker_interval" typescript:",notnull"`
|
||||
CircuitBreakerTimeout serpent.Duration `json:"circuit_breaker_timeout" typescript:",notnull"`
|
||||
CircuitBreakerMaxRequests serpent.Int64 `json:"circuit_breaker_max_requests" typescript:",notnull"`
|
||||
}
|
||||
|
||||
type AIBridgeOpenAIConfig struct {
|
||||
|
||||
Reference in New Issue
Block a user