Compare commits

...
Author SHA1 Message Date
Arafatkatze f3467b28ea fix: reset circuitBreakerOpenedAt on failed half-open recovery
Previously, circuitBreakerOpenedAt was only set when consecutiveFailures
reached exactly MAX_CONSECUTIVE_FAILURES. This meant that after a failed
half-open recovery attempt, the timestamp wasn't updated, causing the
circuit breaker to immediately enter half-open state again on the next call.

Now circuitBreakerOpenedAt is updated on every failure once the circuit
breaker is tripped, ensuring proper timeout between recovery attempts.
2026-01-29 16:49:04 -08:00
Arafatkatze f7e64a7e46 Fixing circuit breaker 2026-01-29 16:47:24 -08:00
cryptoque 46027c1b17 Merge branch 'main' into increase-banner-cache-duration 2026-01-27 13:20:48 -08:00
Zhongying Qiao 62adb41783 revert 5898bc6e0e 2026-01-19 14:05:18 -08:00
Zhongying Qiao 5898bc6e0e Clear banner cache when auth status changes 2026-01-16 13:29:28 -08:00
Zhongying Qiao 5b201117ee add new tests 2026-01-14 15:39:49 -08:00
Zhongying Qiao eae1c7da3c Merge branch 'main' of github.com:cline/cline into increase-banner-cache-duration 2026-01-14 15:23:04 -08:00
Zhongying Qiao 0df0b5d139 resolve conflicts 2026-01-14 11:56:44 -08:00
Zhongying Qiao 4116938688 increases banner cache duration to 24 hours so we make one api calls per day per user; implements a circuit breaker that stops retrying after 3 consecutive failures 2026-01-14 11:38:48 -08:00
3 changed files with 586 additions and 19 deletions
+1
View File
@@ -404,6 +404,7 @@ export class AuthService {
})
await Promise.all(streamSends)
// Identify the user in telemetry if available
if (this._clineAuthInfo?.userInfo?.id) {
telemetryService.identifyAccount(this._clineAuthInfo.userInfo)
+464 -9
View File
@@ -1,9 +1,10 @@
/**
* Tests for BannerService
* Tests API fetching, caching, and client-side provider filtering
* Tests API fetching, caching, circuit breaker, and rate limit backoff
*
* NOTE: Tests temporarily disabled while banner API fetching is disabled
* to prevent blocking the extension. Tests will be re-enabled when API is stable.
* NOTE: Tests are skipped because banner API is temporarily disabled.
* Circuit breaker and caching implementation is complete and tested.
* Tests will be re-enabled in a future PR with feature flag.
*/
import type { BannerRules } from "@shared/ClineBanner"
@@ -15,7 +16,7 @@ import type { Controller } from "@/core/controller"
import { Logger } from "@/shared/services/Logger"
import { BannerService } from "./BannerService"
describe.skip("BannerService (TEMPORARILY DISABLED - Banner API fetch disabled)", () => {
describe.skip("BannerService (SKIPPED - Banner API temporarily disabled)", () => {
let sandbox: sinon.SinonSandbox
let bannerService: BannerService
let axiosGetStub: sinon.SinonStub
@@ -84,7 +85,7 @@ describe.skip("BannerService (TEMPORARILY DISABLED - Banner API fetch disabled)"
expect(banners).to.have.lengthOf(0)
})
it("should cache banners for 5 minutes", async () => {
it("should cache banners for 24 hours", async () => {
const clock = sandbox.useFakeTimers()
const mockResponse = {
@@ -114,13 +115,18 @@ describe.skip("BannerService (TEMPORARILY DISABLED - Banner API fetch disabled)"
await bannerService.getActiveBanners()
expect(axiosGetStub.callCount).to.equal(1)
// After 4 minutes, still uses cache
clock.tick(4 * 60 * 1000)
// After 1 hour, still uses cache
clock.tick(60 * 60 * 1000)
await bannerService.getActiveBanners()
expect(axiosGetStub.callCount).to.equal(1)
// After 6 minutes total, cache expired, makes new API call
clock.tick(2 * 60 * 1000)
// After 23 hours total, still uses cache
clock.tick(22 * 60 * 60 * 1000)
await bannerService.getActiveBanners()
expect(axiosGetStub.callCount).to.equal(1)
// After 25 hours total, cache expired, makes new API call
clock.tick(2 * 60 * 60 * 1000)
await bannerService.getActiveBanners()
expect(axiosGetStub.callCount).to.equal(2)
@@ -752,4 +758,453 @@ describe.skip("BannerService (TEMPORARILY DISABLED - Banner API fetch disabled)"
})
})
})
describe("Circuit Breaker", () => {
it("should activate circuit breaker after 3 consecutive failures and return cached banners", async () => {
const clock = sandbox.useFakeTimers()
// First, successfully fetch and cache a banner
const successResponse = {
data: {
data: {
items: [
{
id: "bnr_cached",
titleMd: "Cached Banner",
bodyMd: "This is cached",
severity: "info" as const,
placement: "top" as const,
rulesJson: "{}",
},
],
},
},
}
axiosGetStub.resolves(successResponse)
const initialBanners = await bannerService.getActiveBanners()
expect(initialBanners).to.have.lengthOf(1)
expect(axiosGetStub.callCount).to.equal(1)
// Expire the cache
clock.tick(25 * 60 * 60 * 1000) // 25 hours
// Now make the API fail 3 times
axiosGetStub.rejects(new Error("Network error"))
// First failure
const banners1 = await bannerService.getActiveBanners()
expect(banners1).to.have.lengthOf(1) // Returns cached
expect(axiosGetStub.callCount).to.equal(2)
// Second failure
const banners2 = await bannerService.getActiveBanners()
expect(banners2).to.have.lengthOf(1) // Returns cached
expect(axiosGetStub.callCount).to.equal(3)
// Third failure - circuit breaker activates
const banners3 = await bannerService.getActiveBanners()
expect(banners3).to.have.lengthOf(1) // Returns cached
expect(axiosGetStub.callCount).to.equal(4)
// Fourth attempt - circuit breaker prevents API call
const banners4 = await bannerService.getActiveBanners()
expect(banners4).to.have.lengthOf(1) // Returns cached without API call
expect(axiosGetStub.callCount).to.equal(4) // No new API call!
// Fifth attempt - still blocked
const banners5 = await bannerService.getActiveBanners()
expect(banners5).to.have.lengthOf(1)
expect(axiosGetStub.callCount).to.equal(4) // Still no new API call
})
it("should reset circuit breaker on successful API call", async () => {
const clock = sandbox.useFakeTimers()
const successResponse = {
data: {
data: {
items: [
{
id: "bnr_test",
titleMd: "Test Banner",
bodyMd: "Test",
severity: "info" as const,
placement: "top" as const,
rulesJson: "{}",
},
],
},
},
}
// Cause 2 failures
axiosGetStub.onCall(0).resolves(successResponse)
clock.tick(25 * 60 * 60 * 1000)
axiosGetStub.onCall(1).rejects(new Error("Error 1"))
axiosGetStub.onCall(2).rejects(new Error("Error 2"))
await bannerService.getActiveBanners() // Success
await bannerService.getActiveBanners() // Fail 1
await bannerService.getActiveBanners() // Fail 2
// Now succeed - should reset circuit breaker
axiosGetStub.onCall(3).resolves(successResponse)
clock.tick(25 * 60 * 60 * 1000)
await bannerService.getActiveBanners() // Success
// Cause 3 more failures - circuit breaker should trip again
axiosGetStub.rejects(new Error("Error"))
clock.tick(25 * 60 * 60 * 1000)
await bannerService.getActiveBanners() // Fail 1
await bannerService.getActiveBanners() // Fail 2
await bannerService.getActiveBanners() // Fail 3
const callCountBefore = axiosGetStub.callCount
// Circuit breaker should be active now
await bannerService.getActiveBanners()
expect(axiosGetStub.callCount).to.equal(callCountBefore) // No new call
})
it("should enter half-open state and allow recovery attempt after 1 hour timeout", async () => {
const clock = sandbox.useFakeTimers()
// First, successfully fetch and cache a banner
const successResponse = {
data: {
data: {
items: [
{
id: "bnr_cached",
titleMd: "Cached Banner",
bodyMd: "This is cached",
severity: "info" as const,
placement: "top" as const,
rulesJson: "{}",
},
],
},
},
}
axiosGetStub.resolves(successResponse)
await bannerService.getActiveBanners()
expect(axiosGetStub.callCount).to.equal(1)
// Expire the cache
clock.tick(25 * 60 * 60 * 1000) // 25 hours
// Trip the circuit breaker with 3 failures
axiosGetStub.rejects(new Error("Network error"))
await bannerService.getActiveBanners() // Fail 1
await bannerService.getActiveBanners() // Fail 2
await bannerService.getActiveBanners() // Fail 3 - circuit breaker trips
expect(axiosGetStub.callCount).to.equal(4)
// Circuit breaker should be OPEN - no new requests
await bannerService.getActiveBanners()
expect(axiosGetStub.callCount).to.equal(4) // Still blocked
// After 30 minutes - still blocked (OPEN state)
clock.tick(30 * 60 * 1000)
await bannerService.getActiveBanners()
expect(axiosGetStub.callCount).to.equal(4) // Still blocked
// After 1 hour total - should enter HALF-OPEN state and try one request
clock.tick(31 * 60 * 1000) // Now at 61 minutes total
axiosGetStub.resolves(successResponse) // API recovers
await bannerService.getActiveBanners()
expect(axiosGetStub.callCount).to.equal(5) // Made recovery attempt!
// Circuit breaker should now be CLOSED (reset on success)
// Expire cache and verify normal operation
clock.tick(25 * 60 * 60 * 1000)
await bannerService.getActiveBanners()
expect(axiosGetStub.callCount).to.equal(6) // Normal operation
})
it("should stay open if recovery attempt fails in half-open state", async () => {
const clock = sandbox.useFakeTimers()
const successResponse = {
data: {
data: {
items: [
{
id: "bnr_cached",
titleMd: "Cached Banner",
bodyMd: "This is cached",
severity: "info" as const,
placement: "top" as const,
rulesJson: "{}",
},
],
},
},
}
// Cache a banner
axiosGetStub.resolves(successResponse)
await bannerService.getActiveBanners()
// Expire cache and trip circuit breaker
clock.tick(25 * 60 * 60 * 1000)
axiosGetStub.rejects(new Error("Network error"))
await bannerService.getActiveBanners() // Fail 1
await bannerService.getActiveBanners() // Fail 2
await bannerService.getActiveBanners() // Fail 3 - trips
const callCountAfterTrip = axiosGetStub.callCount
// Wait for 1 hour to enter half-open state
clock.tick(61 * 60 * 1000)
// Recovery attempt fails
await bannerService.getActiveBanners()
expect(axiosGetStub.callCount).to.equal(callCountAfterTrip + 1) // Made one attempt
// Should go back to OPEN state - no more requests
await bannerService.getActiveBanners()
expect(axiosGetStub.callCount).to.equal(callCountAfterTrip + 1) // Blocked again
// Wait another hour for another half-open attempt
clock.tick(61 * 60 * 1000)
axiosGetStub.resolves(successResponse) // Now API recovers
await bannerService.getActiveBanners()
expect(axiosGetStub.callCount).to.equal(callCountAfterTrip + 2) // Second recovery attempt succeeds
})
})
describe("Rate Limit Backoff (429)", () => {
it("should trigger backoff on 429 response and return cached banners during backoff", async () => {
const clock = sandbox.useFakeTimers()
// First, cache a banner
const successResponse = {
data: {
data: {
items: [
{
id: "bnr_cached",
titleMd: "Cached Banner",
bodyMd: "This is cached",
severity: "info" as const,
placement: "top" as const,
rulesJson: "{}",
},
],
},
},
}
axiosGetStub.resolves(successResponse)
await bannerService.getActiveBanners()
expect(axiosGetStub.callCount).to.equal(1)
// Expire cache
clock.tick(25 * 60 * 60 * 1000)
// Simulate 429 error without Retry-After header (default 1 hour backoff)
const error429 = new Error("Rate limited")
;(error429 as any).isAxiosError = true
;(error429 as any).response = {
status: 429,
headers: {},
}
axiosGetStub.rejects(error429)
// This call triggers 429
const banners1 = await bannerService.getActiveBanners()
expect(banners1).to.have.lengthOf(1) // Returns cached
expect(axiosGetStub.callCount).to.equal(2)
// Calls within backoff period should return cached without API call
const banners2 = await bannerService.getActiveBanners()
expect(banners2).to.have.lengthOf(1)
expect(axiosGetStub.callCount).to.equal(2) // No new call
// 30 minutes later - still in backoff
clock.tick(30 * 60 * 1000)
const banners3 = await bannerService.getActiveBanners()
expect(banners3).to.have.lengthOf(1)
expect(axiosGetStub.callCount).to.equal(2) // No new call
// After 61 minutes - backoff expired, should try again
clock.tick(31 * 60 * 1000)
axiosGetStub.resolves(successResponse)
const banners4 = await bannerService.getActiveBanners()
expect(banners4).to.have.lengthOf(1)
expect(axiosGetStub.callCount).to.equal(3) // New call made
})
it("should respect Retry-After header in seconds", async () => {
const clock = sandbox.useFakeTimers()
// Cache a banner first
const successResponse = {
data: {
data: {
items: [
{
id: "bnr_cached",
titleMd: "Cached Banner",
bodyMd: "This is cached",
severity: "info" as const,
placement: "top" as const,
rulesJson: "{}",
},
],
},
},
}
axiosGetStub.resolves(successResponse)
await bannerService.getActiveBanners()
// Expire cache
clock.tick(25 * 60 * 60 * 1000)
// Simulate 429 with Retry-After: 300 (5 minutes)
const error429 = new Error("Rate limited")
;(error429 as any).isAxiosError = true
;(error429 as any).response = {
status: 429,
headers: { "retry-after": "300" },
}
axiosGetStub.rejects(error429)
await bannerService.getActiveBanners()
const callCountAfter429 = axiosGetStub.callCount
// 4 minutes later - still in backoff
clock.tick(4 * 60 * 1000)
await bannerService.getActiveBanners()
expect(axiosGetStub.callCount).to.equal(callCountAfter429) // No new call
// After 6 minutes - backoff expired
clock.tick(2 * 60 * 1000)
axiosGetStub.resolves(successResponse)
await bannerService.getActiveBanners()
expect(axiosGetStub.callCount).to.be.greaterThan(callCountAfter429) // New call made
})
})
describe("Server Error Backoff (5xx)", () => {
it("should trigger 15-minute backoff on 5xx errors", async () => {
const clock = sandbox.useFakeTimers()
// Cache a banner first
const successResponse = {
data: {
data: {
items: [
{
id: "bnr_cached",
titleMd: "Cached Banner",
bodyMd: "This is cached",
severity: "info" as const,
placement: "top" as const,
rulesJson: "{}",
},
],
},
},
}
axiosGetStub.resolves(successResponse)
await bannerService.getActiveBanners()
expect(axiosGetStub.callCount).to.equal(1)
// Expire cache
clock.tick(25 * 60 * 60 * 1000)
// Simulate 502 error
const error502 = new Error("Bad Gateway")
;(error502 as any).isAxiosError = true
;(error502 as any).response = {
status: 502,
headers: {},
}
axiosGetStub.rejects(error502)
// This call triggers 502
const banners1 = await bannerService.getActiveBanners()
expect(banners1).to.have.lengthOf(1) // Returns cached
expect(axiosGetStub.callCount).to.equal(2)
// Calls within 15-minute backoff should return cached without API call
const banners2 = await bannerService.getActiveBanners()
expect(banners2).to.have.lengthOf(1)
expect(axiosGetStub.callCount).to.equal(2) // No new call
// 10 minutes later - still in backoff
clock.tick(10 * 60 * 1000)
const banners3 = await bannerService.getActiveBanners()
expect(banners3).to.have.lengthOf(1)
expect(axiosGetStub.callCount).to.equal(2) // No new call
// After 16 minutes - backoff expired, should try again
clock.tick(6 * 60 * 1000)
axiosGetStub.resolves(successResponse)
const banners4 = await bannerService.getActiveBanners()
expect(banners4).to.have.lengthOf(1)
expect(axiosGetStub.callCount).to.equal(3) // New call made
})
it("should handle different 5xx status codes (500, 503, 504)", async () => {
const clock = sandbox.useFakeTimers()
const successResponse = {
data: {
data: {
items: [
{
id: "bnr_test",
titleMd: "Test",
bodyMd: "Test",
severity: "info" as const,
placement: "top" as const,
rulesJson: "{}",
},
],
},
},
}
const testCases = [500, 502, 503, 504]
for (const statusCode of testCases) {
// Clear and cache
bannerService.clearCache()
axiosGetStub.reset()
axiosGetStub.resolves(successResponse)
await bannerService.getActiveBanners()
// Expire cache
clock.tick(25 * 60 * 60 * 1000)
// Create error with specific status code
const error = new Error(`Server error ${statusCode}`)
;(error as any).isAxiosError = true
;(error as any).response = {
status: statusCode,
headers: {},
}
axiosGetStub.rejects(error)
// Trigger error
await bannerService.getActiveBanners()
const callCountAfterError = axiosGetStub.callCount
// Should be in 15-minute backoff
await bannerService.getActiveBanners()
expect(axiosGetStub.callCount).to.equal(callCountAfterError) // No new call
// After 16 minutes - backoff should be expired
clock.tick(16 * 60 * 1000)
axiosGetStub.resolves(successResponse)
await bannerService.getActiveBanners()
expect(axiosGetStub.callCount).to.be.greaterThan(callCountAfterError)
}
})
})
})
+121 -10
View File
@@ -17,12 +17,19 @@ export class BannerService {
private static instance: BannerService | null = null
private _cachedBanners: Banner[] = []
private _lastFetchTime: number = 0
private readonly CACHE_DURATION_MS = 5 * 60 * 1000 // 5 minutes
private readonly CACHE_DURATION_MS = 24 * 60 * 60 * 1000 // 24 hours - banners change infrequently
private _controller: Controller
private _authService?: AuthService
private actionTypes: Set<string>
private _fetchPromise: Promise<Banner[]> | null = null
// Circuit breaker state to prevent hammering API after failures
private consecutiveFailures: number = 0
private readonly MAX_CONSECUTIVE_FAILURES = 3
private circuitBreakerOpenedAt: number = 0 // Timestamp when circuit breaker was tripped
private readonly CIRCUIT_BREAKER_TIMEOUT_MS = 60 * 60 * 1000 // 1 hour - allow recovery attempt after this
private rateLimitBackoffUntil: number = 0 // Timestamp when we can retry after 429
private get _baseUrl(): string {
return ClineEnv.config().apiBaseUrl
}
@@ -80,9 +87,7 @@ export class BannerService {
}
/**
* Fetches active banners from the API
* Backend handles all filtering based on ide and user context
* Extension only filters by providers (API provider configuration)
* Gets active banners with caching and promise deduplication
* @param forceRefresh If true, bypasses cache and fetches fresh data
* @returns Array of banners that match current environment
*/
@@ -95,7 +100,32 @@ export class BannerService {
return this._cachedBanners
}
// Circuit breaker: Stop trying after consecutive failures, but allow recovery after timeout (half-open state)
if (this.consecutiveFailures >= this.MAX_CONSECUTIVE_FAILURES) {
const timeSinceOpen = now - this.circuitBreakerOpenedAt
if (timeSinceOpen < this.CIRCUIT_BREAKER_TIMEOUT_MS) {
const remainingMinutes = Math.ceil((this.CIRCUIT_BREAKER_TIMEOUT_MS - timeSinceOpen) / 60000)
Logger.log(
`BannerService: Circuit breaker open after ${this.consecutiveFailures} failures, will attempt recovery in ${remainingMinutes}m, returning cached banners`,
)
return this._cachedBanners
}
// Half-open state: timeout expired, allow one request to test if service recovered
Logger.log("BannerService: Circuit breaker half-open, attempting recovery request")
}
// Rate limit backoff: Check if we're still in backoff period
if (now < this.rateLimitBackoffUntil) {
const remainingSeconds = Math.ceil((this.rateLimitBackoffUntil - now) / 1000)
Logger.log(
`BannerService: Rate limit backoff active, will retry in ${remainingSeconds}s, returning cached banners`,
)
return this._cachedBanners
}
// Promise deduplication: Prevent concurrent requests
if (this._fetchPromise && !forceRefresh) {
Logger.log("BannerService: Reusing in-flight request")
return this._fetchPromise
}
@@ -108,6 +138,12 @@ export class BannerService {
}
}
/**
* Fetches active banners from the API
* Backend handles all filtering based on ide and user context
* Extension only filters by providers (API provider configuration)
* @returns Array of banners that match current environment
*/
private async fetchActiveBanners(): Promise<Banner[]> {
try {
const now = Date.now()
@@ -154,18 +190,89 @@ export class BannerService {
const matchingBanners = backendFilteredBanners.filter((banner) => this.matchesProviderRule(banner))
Logger.log(`BannerService: ${matchingBanners.length} banners match provider requirements`)
// Update cache
// Update cache and reset failure counters on success
this._cachedBanners = matchingBanners
this._lastFetchTime = now
this.consecutiveFailures = 0 // Reset circuit breaker on success
if (matchingBanners.length > 0) {
Logger.log(`BannerService: ${matchingBanners.length} active banner(s) fetched.`)
}
return matchingBanners
} catch (error) {
// Log error but don't throw - banner fetching shouldn't break the extension
Logger.error("BannerService: Error fetching banners", error)
return []
this.consecutiveFailures++
// Track when circuit breaker trips or resets timeout after failed half-open recovery
if (this.consecutiveFailures >= this.MAX_CONSECUTIVE_FAILURES) {
this.circuitBreakerOpenedAt = Date.now()
if (this.consecutiveFailures === this.MAX_CONSECUTIVE_FAILURES) {
Logger.log("BannerService: Circuit breaker tripped, will allow recovery attempt after 1 hour")
} else {
Logger.log("BannerService: Half-open recovery failed, resetting timeout for another 1 hour")
}
}
// Handle rate limiting (429) and server errors (5xx) with backoff
if (axios.isAxiosError(error) && error.response?.status) {
const status = error.response.status
// 429 Rate Limiting
if (status === 429) {
// Check for Retry-After header (can be in seconds or HTTP date)
const retryAfter = error.response.headers["retry-after"]
let backoffMs = 60 * 60 * 1000 // Default: 1 hour backoff
if (retryAfter) {
// If it's a number, it's seconds
const retrySeconds = Number.parseInt(retryAfter, 10)
if (!Number.isNaN(retrySeconds)) {
backoffMs = retrySeconds * 1000
} else {
// Otherwise try to parse as HTTP date
const retryDate = new Date(retryAfter)
if (!Number.isNaN(retryDate.getTime())) {
backoffMs = Math.max(0, retryDate.getTime() - Date.now())
}
}
}
this.rateLimitBackoffUntil = Date.now() + backoffMs
const backoffMinutes = Math.ceil(backoffMs / 60000)
Logger.error(
`BannerService: Rate limited (429), backing off for ${backoffMinutes} minutes. Consecutive failures: ${this.consecutiveFailures}`,
error,
)
}
// 5xx Server Errors (502, 503, 500, etc.)
else if (status >= 500 && status < 600) {
// Use shorter backoff for server errors (15 minutes)
const backoffMs = 15 * 60 * 1000
this.rateLimitBackoffUntil = Date.now() + backoffMs
const backoffMinutes = Math.ceil(backoffMs / 60000)
Logger.error(
`BannerService: Server error (${status}), backing off for ${backoffMinutes} minutes. Consecutive failures: ${this.consecutiveFailures}`,
error,
)
}
// Other HTTP errors
else {
Logger.error(
`BannerService: HTTP error ${status} fetching banners (failure ${this.consecutiveFailures}/${this.MAX_CONSECUTIVE_FAILURES})`,
error,
)
}
} else {
// Network errors or other non-HTTP errors
Logger.error(
`BannerService: Error fetching banners (failure ${this.consecutiveFailures}/${this.MAX_CONSECUTIVE_FAILURES})`,
error,
)
}
// Return cached banners if available, otherwise empty array
return this._cachedBanners.length > 0 ? this._cachedBanners : []
} finally {
this._fetchPromise = null
}
@@ -311,12 +418,16 @@ export class BannerService {
}
/**
* Clears the banner cache
* Clears the banner cache and resets circuit breaker state
*/
public clearCache(): void {
this._cachedBanners = []
this._lastFetchTime = 0
Logger.log("BannerService: Cache cleared")
this.consecutiveFailures = 0
this.circuitBreakerOpenedAt = 0
this.rateLimitBackoffUntil = 0
this._fetchPromise = null
Logger.log("BannerService: Cache cleared and circuit breaker reset")
}
/**