fix(cli): recover remote session connection that silently dies and never reconnects (#12393)

* fix(cli): bound token and connection attempts with a single fenced retry owner

* fix(cli): bounded heartbeat gather with freshness-fenced attach

* chore: changeset for CLI live reconnect fix

* fix(cli): harden heartbeat gather slot release and connection-close signalling

* fix(cli): ignore late token continuation for a settled connection generation

open() only bailed on `closed` after awaiting getToken(); if the connect-
attempt deadline settled the generation while the token was still pending,
the late continuation would construct and assign a WebSocket for an expired
generation, clobbering the newer generation's live socket. Bail on
`g.settled` too so a settled generation never builds a socket.

Also make the AC2a token-rejection test genuinely exercise rejection (flush
the rejection before the deadline can fire) and add AC3g covering the late-
token continuation.

* fix(cli): fence attach announcements on heartbeat session-id containment

An attach announcement resolved on any fresh heartbeat, even one whose
gathered session list omitted the announced id (getSessions() drops a
session whose per-id lookup fails via Effect.orElseSucceed while the gather
still succeeds). That falsely reported a session as attached when the relay
never received it. Thread an optional requireSessionId through
Connection.heartbeat: an id-gated waiter now resolves only when a fresh
heartbeat whose payload contains that id is actually sent, and is otherwise
requeued for the next fresh cycle (rejecting on connection shutdown).
announce(id) forwards the id; presence heartbeats remain id-agnostic.

Adds AC4a (degraded heartbeat preserves last known-good sessions), AC6d
(id-containment fence), and AC6e (in-flight-cycle waiter rejects on
permanent close).

* ci: retrigger checks after GitHub Actions incident recovery
This commit is contained in:
Igor Šćekić
2026-07-22 13:46:19 +02:00
committed by GitHub
parent bcff5cb360
commit 9262f2b49a
6 changed files with 1856 additions and 140 deletions
+5
View File
@@ -0,0 +1,5 @@
---
"@kilocode/cli": patch
---
Remote CLI sessions no longer appear frozen on mobile when the connection to the session relay stalls; they now recover on their own instead of staying read-only until the CLI is restarted. Token acquisition and connection attempts are bounded by deadlines with a single fenced retry owner, and heartbeat session gathers are bounded so one stuck gather can no longer silently kill every future heartbeat.
@@ -30,8 +30,13 @@ export namespace AttachedState {
export type Options = {
/** Fires the relay heartbeat. May be fire-and-forget or awaited. Must
* reject (not resolve) when no relay connection is available so that
* `announce` cannot silently mark a session as attached. */
heartbeat: () => Promise<void>
* `announce` cannot silently mark a session as attached.
*
* `opts.requireSessionId` is forwarded by `announce(id)` so the relay
* only resolves the attach promise when a fresh heartbeat whose
* payload contains that id was actually sent. Presence fire-and-
* forget heartbeats call without an id and resolve on any fresh send. */
heartbeat: (opts?: { requireSessionId?: string }) => Promise<void>
log?: { warn: (msg: string, meta?: unknown) => void }
}
@@ -141,7 +146,10 @@ export namespace AttachedState {
const owned = (async () => {
pending.add(id)
try {
await options.heartbeat()
// kilocode_change - forward the announced id so the relay only
// resolves the attach once a fresh heartbeat whose payload
// contains this id was actually sent (id-containment fence).
await options.heartbeat({ requireSessionId: id })
} catch (err) {
// Roll back only the entry this call added. If presence adopted
// the id while the heartbeat was in flight, presence is the
@@ -221,8 +221,10 @@ export namespace KiloSessions {
// create_session's catch block turns that into the sanitized failure
// response and the user retries manually.
const attachedState = AttachedState.create({
heartbeat: () =>
remote ? remote.conn.heartbeat() : Promise.reject(new Error("attachRemoteSession: no remote connection")),
heartbeat: (opts) =>
remote
? remote.conn.heartbeat(opts)
: Promise.reject(new Error("attachRemoteSession: no remote connection")),
log: attachedLog,
})
// kilocode_change end
+388 -100
View File
@@ -4,6 +4,13 @@ import { InstallationVersion } from "@opencode-ai/core/installation/version"
export namespace RemoteWS {
export type SessionInfo = RemoteProtocol.SessionInfo
export type Timers = {
setTimeout: (fn: () => void, ms?: number) => unknown
clearTimeout: (t: unknown) => void
setInterval: (fn: () => void, ms?: number) => unknown
clearInterval: (t: unknown) => void
}
export type Options = {
url: string
getToken: () => Promise<string | undefined>
@@ -23,79 +30,272 @@ export namespace RemoteWS {
onClose?: (code: number, reason: string) => void
/** Inactivity timeout in ms — force-close if no inbound message within this window */
timeout?: number
/** Injectable timer primitives for deterministic testing. Defaults to globals. */
timers?: Timers
/** Injectable clock for deterministic testing. Defaults to Date.now. */
now?: () => number
/** Token-acquisition deadline in ms. Defaults to 15_000. */
tokenTimeout?: number
/** Connection-attempt deadline (token acquisition through onopen) in ms. Defaults to 30_000. */
connectTimeout?: number
/** Session-gather deadline in ms. Defaults to 15_000. */
gatherTimeout?: number
/** Max unresolved gather operations before cycles send degraded heartbeats. Defaults to 4. */
maxOutstandingGathers?: number
}
export type Connection = {
readonly connectionId: string
send(msg: RemoteProtocol.Outbound): void
heartbeat(): Promise<void>
/**
* Resolves when a heartbeat built from a FRESH gather has actually been
* sent over a live socket. Degraded sends and non-live buffered sends
* leave the returned promise pending; `close()` rejects it.
*
* When `opts.requireSessionId` is provided, the promise only resolves
* when the sent fresh payload's session list contains that id. This
* fences attach-announce waiters so a fresh heartbeat that legitimately
* omits the announced id (e.g. the gather's `Effect.orElseSucceed`
* filtered it out) does not falsely report the session as attached.
*/
heartbeat(opts?: { requireSessionId?: string }): Promise<void>
close(): void
readonly connected: boolean
}
type Timer = ReturnType<typeof setTimeout>
const defaultTimers: Timers = {
setTimeout: (fn, ms) => setTimeout(fn, ms),
clearTimeout: (t) => clearTimeout(t as ReturnType<typeof setTimeout>),
setInterval: (fn, ms) => setInterval(fn, ms),
clearInterval: (t) => clearInterval(t as ReturnType<typeof setInterval>),
}
type Gen = { id: number; settled: boolean; opened: boolean }
export function connect(options: Options): Connection {
const interval = options.heartbeat ?? 10_000
const connectionId = crypto.randomUUID()
const withContext = options.withContext ?? ((fn) => fn())
const timers = options.timers ?? defaultTimers
const now = options.now ?? Date.now
const tokenTimeout = options.tokenTimeout ?? 15_000
const connectTimeout = options.connectTimeout ?? 30_000
let ws: WebSocket | undefined
let backoff = 1000
let timer: Timer | undefined
let beat: Timer | undefined
let timer: unknown
let beat: unknown
let closed = false
const buffer: string[] = []
let beating: Promise<void> | undefined
let queued = false
// --- Bounded heartbeat gather with freshness-fenced attach (Path D fix) ---
// A single never-settling getSessions() must not permanently kill heartbeats.
// Each cycle bounds the gather with a deadline; on failure it sends a
// "degraded" heartbeat carrying the last known-good session list so
// server-side liveness is preserved even when metadata is stale. Callers
// awaiting heartbeat() (session attach announcements) resolve ONLY when a
// heartbeat built from a FRESH gather is actually sent over a live socket;
// degraded sends leave them pending, a transient reconnect keeps them
// pending (they resolve on the next fresh send over the new socket), and
// Connection.close() rejects them.
//
// Degraded-mode limitation (deliberate, bounded by recovery): while gathers
// keep failing, session membership/status/title/gitUrl/gitBranch can be
// stale indefinitely, and sessions created or closed during degradation are
// reflected only after the first fresh gather succeeds.
//
// Attach-announce fencing (AC6): a waiter registered with
// `requireSessionId` stays pending until a fresh heartbeat whose
// payload contains that id is sent over a live socket. A fresh
// gather whose session list omits the required id (e.g. the
// upstream `get(id)` was filtered by `Effect.orElseSucceed`) does
// NOT resolve the waiter — it is requeued and re-evaluated on the
// next fresh cycle. The periodic interval keeps calling
// `requestCycle`, so recovery is automatic once a fresh gather
// includes the id. Permanent close (Connection.close) rejects the
// waiter.
const gatherTimeout = options.gatherTimeout ?? 15_000
const maxOutstandingGathers = options.maxOutstandingGathers ?? 4
let lastGood: SessionInfo[] | undefined
let outstanding = 0
let degradedCount = 0
type Waiter = { resolve: () => void; reject: (err: unknown) => void; requireSessionId?: string }
let waiters: Waiter[] = []
function heartbeat(): Promise<void> {
queued = true
if (beating) return beating
function makeWaiter(): { promise: Promise<void>; waiter: Waiter } {
let resolve!: () => void
let reject!: (err: unknown) => void
const promise = new Promise<void>((res, rej) => {
resolve = res
reject = rej
})
return { promise, waiter: { resolve, reject } }
}
const current = Promise.resolve(
withContext(async () => {
while (queued) {
if (closed) return
queued = false
const sessions = await options.getSessions()
if (closed) return
send({ type: "heartbeat", protocolVersion: InstallationVersion, ...sessions })
}
}),
).finally(() => {
beating = undefined
if (!queued || closed) return
void heartbeat().catch((err) => {
options.log.error("remote-ws heartbeat failed", { error: String(err) })
function rejectWaiters(list: Waiter[], err: unknown) {
for (const w of list) w.reject(err)
}
// One bounded gather. Never throws. Returns the fresh session list, or
// undefined to signal a degraded cycle (caller sends last known-good).
async function gatherOnce(): Promise<SessionInfo[] | undefined> {
if (outstanding >= maxOutstandingGathers) {
degradedCount++
options.log.warn("remote-ws heartbeat gather cap reached, degraded heartbeat", {
outstanding,
degraded: degradedCount,
})
return undefined
}
outstanding++
let released = false
const release = () => {
if (released) return
released = true
outstanding--
}
// Free the slot on ANY settle — success, rejection, or a late settle after
// this cycle abandoned it on timeout. A late result is never read below,
// so an abandoned gather's eventual value is discarded, never emitted.
const normalized = Promise.resolve()
.then(() => options.getSessions())
.then(
(r) => {
release()
return { ok: true as const, sessions: r.sessions }
},
(err) => {
release()
return { ok: false as const, error: err }
},
)
const outcome = await new Promise<
{ kind: "ok"; sessions: SessionInfo[] } | { kind: "err"; error: unknown } | { kind: "timeout" }
>((resolve) => {
let done = false
const t = timers.setTimeout(() => {
if (done) return
done = true
resolve({ kind: "timeout" })
}, gatherTimeout)
void normalized.then((res) => {
if (done) return
done = true
timers.clearTimeout(t)
resolve(res.ok ? { kind: "ok", sessions: res.sessions } : { kind: "err", error: res.error })
})
})
beating = current
return current
if (outcome.kind === "ok") return outcome.sessions
degradedCount++
if (outcome.kind === "err") {
options.log.warn("remote-ws heartbeat gather rejected, degraded heartbeat", {
error: String(outcome.error),
degraded: degradedCount,
})
} else {
options.log.warn("remote-ws heartbeat gather timeout, degraded heartbeat", {
outstanding,
degraded: degradedCount,
})
}
return undefined
}
function heartbeat(opts?: { requireSessionId?: string }): Promise<void> {
if (closed) return Promise.reject(new Error("remote-ws connection closed"))
const { promise, waiter } = makeWaiter()
waiter.requireSessionId = opts?.requireSessionId
waiters.push(waiter)
requestCycle()
return promise
}
// Interval-driven ticks call requestCycle directly so the periodic heartbeat
// never registers a waiter (no waiter accumulation during degradation).
function requestCycle() {
queued = true
runLoop()
}
function runLoop() {
if (beating || closed) return
beating = Promise.resolve(
withContext(async () => {
while (queued && !closed) {
queued = false
const cycleWaiters = waiters
waiters = []
const fresh = await gatherOnce()
if (closed) {
rejectWaiters(cycleWaiters, new Error("remote-ws connection closed"))
return
}
if (fresh !== undefined) {
lastGood = fresh
const sentLive = ws?.readyState === WebSocket.OPEN
send({ type: "heartbeat", protocolVersion: InstallationVersion, sessions: fresh })
if (sentLive) {
// A waiter requiring a specific id is satisfied only when
// the sent payload contains that id. Unsatisfied waiters
// are requeued so the periodic interval keeps evaluating
// them; they resolve on a future fresh send whose payload
// includes their required id (or reject on close).
const satisfied: Waiter[] = []
const unsatisfied: Waiter[] = []
for (const w of cycleWaiters) {
if (
w.requireSessionId === undefined ||
fresh.some((s) => s.id === w.requireSessionId)
) {
satisfied.push(w)
} else {
unsatisfied.push(w)
}
}
for (const w of satisfied) w.resolve()
if (unsatisfied.length > 0) {
waiters = unsatisfied.concat(waiters)
}
} else {
// Buffered because the socket is not open; resolve on the next
// fresh send over the (re)connected socket.
waiters = cycleWaiters.concat(waiters)
}
} else {
// Degraded: preserve liveness with the last known-good list (empty
// on cold start) and keep waiters pending for a future fresh send.
send({ type: "heartbeat", protocolVersion: InstallationVersion, sessions: lastGood ?? [] })
waiters = cycleWaiters.concat(waiters)
}
}
}),
)
.catch((err) => options.log.error("remote-ws heartbeat loop failed", { error: String(err) }))
.finally(() => {
beating = undefined
if (queued && !closed) runLoop()
})
}
function startHeartbeat() {
stopHeartbeat()
beat = setInterval(() => {
void heartbeat().catch((err) => {
options.log.error("remote-ws heartbeat failed", { error: String(err) })
})
}, interval)
beat = timers.setInterval(() => requestCycle(), interval)
}
function stopHeartbeat() {
if (beat) clearInterval(beat)
if (beat) timers.clearInterval(beat)
beat = undefined
}
let activity = Date.now()
let watchdog: Timer | undefined
let activity = now()
let watchdog: unknown
const timeout = options.timeout ?? 30_000
function startWatchdog() {
stopWatchdog()
watchdog = setInterval(
watchdog = timers.setInterval(
() => {
if (Date.now() - activity > timeout) {
if (now() - activity > timeout) {
options.log.warn("remote-ws activity timeout, forcing reconnect")
stopWatchdog()
ws?.close(4000, "activity timeout")
@@ -106,88 +306,172 @@ export namespace RemoteWS {
}
function stopWatchdog() {
if (watchdog) clearInterval(watchdog)
if (watchdog) timers.clearInterval(watchdog)
watchdog = undefined
}
// Connect-attempt deadline (covers token acquisition through onopen).
let connectDeadline: unknown
let currentGen = 0
function startConnectDeadline(g: Gen) {
stopConnectDeadline()
connectDeadline = timers.setTimeout(() => {
connectDeadline = undefined
if (closed || g.settled) return
options.log.warn("remote-ws connect attempt deadline, will retry", { gen: g.id })
if (ws) ws.close(4001, "connect timeout")
scheduleRetry(g)
}, connectTimeout)
}
function stopConnectDeadline() {
if (connectDeadline) timers.clearTimeout(connectDeadline)
connectDeadline = undefined
}
// Single fenced retry owner: exactly one of {token-failure, connect-deadline,
// onclose, sync-throw} may schedule a retry for a given generation.
function scheduleRetry(g: Gen) {
if (closed || g.settled) return
g.settled = true
schedule()
}
function withTimeout<T>(promise: Promise<T>, ms: number, label: string): Promise<T> {
return new Promise<T>((resolve, reject) => {
let done = false
const t = timers.setTimeout(() => {
if (done) return
done = true
reject(new Error(label))
}, ms)
promise.then(
(v) => {
if (done) return
done = true
timers.clearTimeout(t)
resolve(v)
},
(err) => {
if (done) return
done = true
timers.clearTimeout(t)
reject(err)
},
)
})
}
async function open() {
if (closed) return
const token = await options.getToken()
if (closed) return
if (!token) {
options.log.warn("remote-ws no token, will retry")
schedule()
return
}
const endpoint = `${options.url}/api/user/cli?token=${encodeURIComponent(token)}&connectionId=${connectionId}`
options.log.info("remote-ws connecting", { connectionId, endpoint: endpoint.replace(/token=[^&]+/, "token=***") })
const socket = new WebSocket(endpoint)
ws = socket
socket.onopen = () => {
if (ws !== socket || closed) {
socket.close()
return
}
options.log.info("remote-ws connected", { buffered: buffer.length })
void withContext(() => options.onOpen?.())
backoff = 1000
for (const msg of buffer) socket.send(msg)
buffer.length = 0
activity = Date.now()
startHeartbeat()
startWatchdog()
}
socket.onmessage = (event) => {
if (ws !== socket || closed) return
activity = Date.now()
const raw = String(event.data)
let json: unknown
const g: Gen = { id: ++currentGen, settled: false, opened: false }
startConnectDeadline(g)
try {
let token: string | undefined
try {
json = JSON.parse(raw)
} catch {
options.log.warn("remote-ws invalid JSON", { bytes: raw.length })
token = await withTimeout(options.getToken(), tokenTimeout, "remote-ws token timeout")
} catch (err) {
if (closed) return
options.log.warn("remote-ws getToken failed, will retry", { gen: g.id, error: String(err) })
scheduleRetry(g)
return
}
const preview = RemoteProtocol.Preview.safeParse(json)
options.log.info("remote-ws received", { bytes: raw.length, ...preview.data })
const parsed = RemoteProtocol.Inbound.safeParse(json)
if (!parsed.success) {
options.log.warn("remote-ws message parse failed", { error: parsed.error })
if (closed || g.settled) return
if (!token) {
options.log.warn("remote-ws no token, will retry", { gen: g.id })
scheduleRetry(g)
return
}
options.onMessage?.(parsed.data)
}
const endpoint = `${options.url}/api/user/cli?token=${encodeURIComponent(token)}&connectionId=${connectionId}`
options.log.info("remote-ws connecting", { connectionId, gen: g.id, endpoint: endpoint.replace(/token=[^&]+/, "token=***") })
let socket: WebSocket
try {
socket = new WebSocket(endpoint)
} catch (err) {
if (closed) return
options.log.warn("remote-ws constructor threw, will retry", { gen: g.id, error: String(err) })
scheduleRetry(g)
return
}
ws = socket
socket.onclose = (event) => {
if (ws !== socket) return
options.log.info("remote-ws closed", { code: event.code, reason: event.reason })
ws = undefined
stopHeartbeat()
stopWatchdog()
socket.onopen = () => {
if (g.settled || ws !== socket || closed) {
socket.close()
return
}
g.opened = true
stopConnectDeadline()
options.log.info("remote-ws connected", { gen: g.id, buffered: buffer.length })
void withContext(() => options.onOpen?.())
backoff = 1000
for (const msg of buffer) socket.send(msg)
buffer.length = 0
activity = now()
startHeartbeat()
startWatchdog()
if (waiters.length > 0) requestCycle()
}
socket.onmessage = (event) => {
if (g.settled || ws !== socket || closed) return
activity = now()
const raw = String(event.data)
let json: unknown
try {
json = JSON.parse(raw)
} catch {
options.log.warn("remote-ws invalid JSON", { bytes: raw.length })
return
}
const preview = RemoteProtocol.Preview.safeParse(json)
options.log.info("remote-ws received", { bytes: raw.length, ...preview.data })
const parsed = RemoteProtocol.Inbound.safeParse(json)
if (!parsed.success) {
options.log.warn("remote-ws message parse failed", { error: parsed.error })
return
}
options.onMessage?.(parsed.data)
}
socket.onclose = (event) => {
if (ws !== socket) return
stopConnectDeadline()
options.log.info("remote-ws closed", { code: event.code, reason: event.reason, gen: g.id })
ws = undefined
stopHeartbeat()
stopWatchdog()
if (closed) return
if (event.code === 4401 || event.code === 4403 || event.code === 4409) {
options.log.warn("remote-ws closed permanently", {
code: event.code,
reason: event.reason,
})
const pending = waiters
waiters = []
rejectWaiters(pending, new Error("remote-ws connection permanently closed"))
void withContext(() => options.onClose?.(event.code, event.reason))
return
}
if (g.opened) void withContext(() => options.onDisconnect?.())
scheduleRetry(g)
}
socket.onerror = (event) => {
if (g.settled || ws !== socket || closed) return
options.log.error("remote-ws error", { error: event })
}
} catch (err) {
if (closed) return
if (event.code === 4401 || event.code === 4403 || event.code === 4409) {
options.log.warn("remote-ws closed permanently", {
code: event.code,
reason: event.reason,
})
void withContext(() => options.onClose?.(event.code, event.reason))
return
}
void withContext(() => options.onDisconnect?.())
schedule()
}
socket.onerror = (event) => {
if (ws !== socket || closed) return
options.log.error("remote-ws error", { error: event })
options.log.warn("remote-ws open threw, will retry", { gen: g.id, error: String(err) })
scheduleRetry(g)
}
}
function schedule() {
if (closed) return
timer = setTimeout(() => open(), backoff)
timer = timers.setTimeout(() => open(), backoff)
backoff = Math.min(backoff * 2, 60000)
}
@@ -206,8 +490,12 @@ export namespace RemoteWS {
queued = false
stopHeartbeat()
stopWatchdog()
if (timer) clearTimeout(timer)
stopConnectDeadline()
if (timer) timers.clearTimeout(timer)
if (ws) ws.close()
const pending = waiters
waiters = []
rejectWaiters(pending, new Error("remote-ws connection closed"))
}
void open()
@@ -744,4 +744,29 @@ describe("AttachedState", () => {
await replacement
expect([...state.union()].sort()).toEqual(["ses_x"])
})
// AC6d: announce(id) must forward { requireSessionId: id } to the
// heartbeat callback so the relay only resolves the attach once a fresh
// heartbeat whose payload contains that id was actually sent. Presence
// fire-and-forget heartbeats (from setPresence) continue to call
// without an id and resolve on any fresh send.
test("announce(id) forwards { requireSessionId: id } to the heartbeat callback", async () => {
const calls: Array<{ requireSessionId?: string }> = []
const state = AttachedState.create({
heartbeat: (opts) => {
calls.push(opts ? { ...opts } : {})
return Promise.resolve()
},
log: nolog,
})
// setPresence fires a fire-and-forget heartbeat with NO id.
state.setPresence(["ses_a"])
await Promise.resolve()
// announce(id) forwards the id to the awaited heartbeat.
await state.announce("ses_b")
expect(calls).toEqual([{}, { requireSessionId: "ses_b" }])
})
})
File diff suppressed because it is too large Load Diff