mirror of
https://github.com/Kilo-Org/kilocode.git
synced 2026-08-30 17:14:40 +08:00
fix(cli): recover remote session connection that silently dies and never reconnects (#12393)
* fix(cli): bound token and connection attempts with a single fenced retry owner * fix(cli): bounded heartbeat gather with freshness-fenced attach * chore: changeset for CLI live reconnect fix * fix(cli): harden heartbeat gather slot release and connection-close signalling * fix(cli): ignore late token continuation for a settled connection generation open() only bailed on `closed` after awaiting getToken(); if the connect- attempt deadline settled the generation while the token was still pending, the late continuation would construct and assign a WebSocket for an expired generation, clobbering the newer generation's live socket. Bail on `g.settled` too so a settled generation never builds a socket. Also make the AC2a token-rejection test genuinely exercise rejection (flush the rejection before the deadline can fire) and add AC3g covering the late- token continuation. * fix(cli): fence attach announcements on heartbeat session-id containment An attach announcement resolved on any fresh heartbeat, even one whose gathered session list omitted the announced id (getSessions() drops a session whose per-id lookup fails via Effect.orElseSucceed while the gather still succeeds). That falsely reported a session as attached when the relay never received it. Thread an optional requireSessionId through Connection.heartbeat: an id-gated waiter now resolves only when a fresh heartbeat whose payload contains that id is actually sent, and is otherwise requeued for the next fresh cycle (rejecting on connection shutdown). announce(id) forwards the id; presence heartbeats remain id-agnostic. Adds AC4a (degraded heartbeat preserves last known-good sessions), AC6d (id-containment fence), and AC6e (in-flight-cycle waiter rejects on permanent close). * ci: retrigger checks after GitHub Actions incident recovery
This commit is contained in:
@@ -0,0 +1,5 @@
|
||||
---
|
||||
"@kilocode/cli": patch
|
||||
---
|
||||
|
||||
Remote CLI sessions no longer appear frozen on mobile when the connection to the session relay stalls; they now recover on their own instead of staying read-only until the CLI is restarted. Token acquisition and connection attempts are bounded by deadlines with a single fenced retry owner, and heartbeat session gathers are bounded so one stuck gather can no longer silently kill every future heartbeat.
|
||||
@@ -30,8 +30,13 @@ export namespace AttachedState {
|
||||
export type Options = {
|
||||
/** Fires the relay heartbeat. May be fire-and-forget or awaited. Must
|
||||
* reject (not resolve) when no relay connection is available so that
|
||||
* `announce` cannot silently mark a session as attached. */
|
||||
heartbeat: () => Promise<void>
|
||||
* `announce` cannot silently mark a session as attached.
|
||||
*
|
||||
* `opts.requireSessionId` is forwarded by `announce(id)` so the relay
|
||||
* only resolves the attach promise when a fresh heartbeat whose
|
||||
* payload contains that id was actually sent. Presence fire-and-
|
||||
* forget heartbeats call without an id and resolve on any fresh send. */
|
||||
heartbeat: (opts?: { requireSessionId?: string }) => Promise<void>
|
||||
log?: { warn: (msg: string, meta?: unknown) => void }
|
||||
}
|
||||
|
||||
@@ -141,7 +146,10 @@ export namespace AttachedState {
|
||||
const owned = (async () => {
|
||||
pending.add(id)
|
||||
try {
|
||||
await options.heartbeat()
|
||||
// kilocode_change - forward the announced id so the relay only
|
||||
// resolves the attach once a fresh heartbeat whose payload
|
||||
// contains this id was actually sent (id-containment fence).
|
||||
await options.heartbeat({ requireSessionId: id })
|
||||
} catch (err) {
|
||||
// Roll back only the entry this call added. If presence adopted
|
||||
// the id while the heartbeat was in flight, presence is the
|
||||
|
||||
@@ -221,8 +221,10 @@ export namespace KiloSessions {
|
||||
// create_session's catch block turns that into the sanitized failure
|
||||
// response and the user retries manually.
|
||||
const attachedState = AttachedState.create({
|
||||
heartbeat: () =>
|
||||
remote ? remote.conn.heartbeat() : Promise.reject(new Error("attachRemoteSession: no remote connection")),
|
||||
heartbeat: (opts) =>
|
||||
remote
|
||||
? remote.conn.heartbeat(opts)
|
||||
: Promise.reject(new Error("attachRemoteSession: no remote connection")),
|
||||
log: attachedLog,
|
||||
})
|
||||
// kilocode_change end
|
||||
|
||||
@@ -4,6 +4,13 @@ import { InstallationVersion } from "@opencode-ai/core/installation/version"
|
||||
export namespace RemoteWS {
|
||||
export type SessionInfo = RemoteProtocol.SessionInfo
|
||||
|
||||
export type Timers = {
|
||||
setTimeout: (fn: () => void, ms?: number) => unknown
|
||||
clearTimeout: (t: unknown) => void
|
||||
setInterval: (fn: () => void, ms?: number) => unknown
|
||||
clearInterval: (t: unknown) => void
|
||||
}
|
||||
|
||||
export type Options = {
|
||||
url: string
|
||||
getToken: () => Promise<string | undefined>
|
||||
@@ -23,79 +30,272 @@ export namespace RemoteWS {
|
||||
onClose?: (code: number, reason: string) => void
|
||||
/** Inactivity timeout in ms — force-close if no inbound message within this window */
|
||||
timeout?: number
|
||||
/** Injectable timer primitives for deterministic testing. Defaults to globals. */
|
||||
timers?: Timers
|
||||
/** Injectable clock for deterministic testing. Defaults to Date.now. */
|
||||
now?: () => number
|
||||
/** Token-acquisition deadline in ms. Defaults to 15_000. */
|
||||
tokenTimeout?: number
|
||||
/** Connection-attempt deadline (token acquisition through onopen) in ms. Defaults to 30_000. */
|
||||
connectTimeout?: number
|
||||
/** Session-gather deadline in ms. Defaults to 15_000. */
|
||||
gatherTimeout?: number
|
||||
/** Max unresolved gather operations before cycles send degraded heartbeats. Defaults to 4. */
|
||||
maxOutstandingGathers?: number
|
||||
}
|
||||
|
||||
export type Connection = {
|
||||
readonly connectionId: string
|
||||
send(msg: RemoteProtocol.Outbound): void
|
||||
heartbeat(): Promise<void>
|
||||
/**
|
||||
* Resolves when a heartbeat built from a FRESH gather has actually been
|
||||
* sent over a live socket. Degraded sends and non-live buffered sends
|
||||
* leave the returned promise pending; `close()` rejects it.
|
||||
*
|
||||
* When `opts.requireSessionId` is provided, the promise only resolves
|
||||
* when the sent fresh payload's session list contains that id. This
|
||||
* fences attach-announce waiters so a fresh heartbeat that legitimately
|
||||
* omits the announced id (e.g. the gather's `Effect.orElseSucceed`
|
||||
* filtered it out) does not falsely report the session as attached.
|
||||
*/
|
||||
heartbeat(opts?: { requireSessionId?: string }): Promise<void>
|
||||
close(): void
|
||||
readonly connected: boolean
|
||||
}
|
||||
|
||||
type Timer = ReturnType<typeof setTimeout>
|
||||
const defaultTimers: Timers = {
|
||||
setTimeout: (fn, ms) => setTimeout(fn, ms),
|
||||
clearTimeout: (t) => clearTimeout(t as ReturnType<typeof setTimeout>),
|
||||
setInterval: (fn, ms) => setInterval(fn, ms),
|
||||
clearInterval: (t) => clearInterval(t as ReturnType<typeof setInterval>),
|
||||
}
|
||||
|
||||
type Gen = { id: number; settled: boolean; opened: boolean }
|
||||
|
||||
export function connect(options: Options): Connection {
|
||||
const interval = options.heartbeat ?? 10_000
|
||||
const connectionId = crypto.randomUUID()
|
||||
const withContext = options.withContext ?? ((fn) => fn())
|
||||
const timers = options.timers ?? defaultTimers
|
||||
const now = options.now ?? Date.now
|
||||
const tokenTimeout = options.tokenTimeout ?? 15_000
|
||||
const connectTimeout = options.connectTimeout ?? 30_000
|
||||
let ws: WebSocket | undefined
|
||||
let backoff = 1000
|
||||
let timer: Timer | undefined
|
||||
let beat: Timer | undefined
|
||||
let timer: unknown
|
||||
let beat: unknown
|
||||
let closed = false
|
||||
const buffer: string[] = []
|
||||
let beating: Promise<void> | undefined
|
||||
let queued = false
|
||||
// --- Bounded heartbeat gather with freshness-fenced attach (Path D fix) ---
|
||||
// A single never-settling getSessions() must not permanently kill heartbeats.
|
||||
// Each cycle bounds the gather with a deadline; on failure it sends a
|
||||
// "degraded" heartbeat carrying the last known-good session list so
|
||||
// server-side liveness is preserved even when metadata is stale. Callers
|
||||
// awaiting heartbeat() (session attach announcements) resolve ONLY when a
|
||||
// heartbeat built from a FRESH gather is actually sent over a live socket;
|
||||
// degraded sends leave them pending, a transient reconnect keeps them
|
||||
// pending (they resolve on the next fresh send over the new socket), and
|
||||
// Connection.close() rejects them.
|
||||
//
|
||||
// Degraded-mode limitation (deliberate, bounded by recovery): while gathers
|
||||
// keep failing, session membership/status/title/gitUrl/gitBranch can be
|
||||
// stale indefinitely, and sessions created or closed during degradation are
|
||||
// reflected only after the first fresh gather succeeds.
|
||||
//
|
||||
// Attach-announce fencing (AC6): a waiter registered with
|
||||
// `requireSessionId` stays pending until a fresh heartbeat whose
|
||||
// payload contains that id is sent over a live socket. A fresh
|
||||
// gather whose session list omits the required id (e.g. the
|
||||
// upstream `get(id)` was filtered by `Effect.orElseSucceed`) does
|
||||
// NOT resolve the waiter — it is requeued and re-evaluated on the
|
||||
// next fresh cycle. The periodic interval keeps calling
|
||||
// `requestCycle`, so recovery is automatic once a fresh gather
|
||||
// includes the id. Permanent close (Connection.close) rejects the
|
||||
// waiter.
|
||||
const gatherTimeout = options.gatherTimeout ?? 15_000
|
||||
const maxOutstandingGathers = options.maxOutstandingGathers ?? 4
|
||||
let lastGood: SessionInfo[] | undefined
|
||||
let outstanding = 0
|
||||
let degradedCount = 0
|
||||
type Waiter = { resolve: () => void; reject: (err: unknown) => void; requireSessionId?: string }
|
||||
let waiters: Waiter[] = []
|
||||
|
||||
function heartbeat(): Promise<void> {
|
||||
queued = true
|
||||
if (beating) return beating
|
||||
function makeWaiter(): { promise: Promise<void>; waiter: Waiter } {
|
||||
let resolve!: () => void
|
||||
let reject!: (err: unknown) => void
|
||||
const promise = new Promise<void>((res, rej) => {
|
||||
resolve = res
|
||||
reject = rej
|
||||
})
|
||||
return { promise, waiter: { resolve, reject } }
|
||||
}
|
||||
|
||||
const current = Promise.resolve(
|
||||
withContext(async () => {
|
||||
while (queued) {
|
||||
if (closed) return
|
||||
queued = false
|
||||
const sessions = await options.getSessions()
|
||||
if (closed) return
|
||||
send({ type: "heartbeat", protocolVersion: InstallationVersion, ...sessions })
|
||||
}
|
||||
}),
|
||||
).finally(() => {
|
||||
beating = undefined
|
||||
if (!queued || closed) return
|
||||
void heartbeat().catch((err) => {
|
||||
options.log.error("remote-ws heartbeat failed", { error: String(err) })
|
||||
function rejectWaiters(list: Waiter[], err: unknown) {
|
||||
for (const w of list) w.reject(err)
|
||||
}
|
||||
|
||||
// One bounded gather. Never throws. Returns the fresh session list, or
|
||||
// undefined to signal a degraded cycle (caller sends last known-good).
|
||||
async function gatherOnce(): Promise<SessionInfo[] | undefined> {
|
||||
if (outstanding >= maxOutstandingGathers) {
|
||||
degradedCount++
|
||||
options.log.warn("remote-ws heartbeat gather cap reached, degraded heartbeat", {
|
||||
outstanding,
|
||||
degraded: degradedCount,
|
||||
})
|
||||
return undefined
|
||||
}
|
||||
outstanding++
|
||||
let released = false
|
||||
const release = () => {
|
||||
if (released) return
|
||||
released = true
|
||||
outstanding--
|
||||
}
|
||||
// Free the slot on ANY settle — success, rejection, or a late settle after
|
||||
// this cycle abandoned it on timeout. A late result is never read below,
|
||||
// so an abandoned gather's eventual value is discarded, never emitted.
|
||||
const normalized = Promise.resolve()
|
||||
.then(() => options.getSessions())
|
||||
.then(
|
||||
(r) => {
|
||||
release()
|
||||
return { ok: true as const, sessions: r.sessions }
|
||||
},
|
||||
(err) => {
|
||||
release()
|
||||
return { ok: false as const, error: err }
|
||||
},
|
||||
)
|
||||
const outcome = await new Promise<
|
||||
{ kind: "ok"; sessions: SessionInfo[] } | { kind: "err"; error: unknown } | { kind: "timeout" }
|
||||
>((resolve) => {
|
||||
let done = false
|
||||
const t = timers.setTimeout(() => {
|
||||
if (done) return
|
||||
done = true
|
||||
resolve({ kind: "timeout" })
|
||||
}, gatherTimeout)
|
||||
void normalized.then((res) => {
|
||||
if (done) return
|
||||
done = true
|
||||
timers.clearTimeout(t)
|
||||
resolve(res.ok ? { kind: "ok", sessions: res.sessions } : { kind: "err", error: res.error })
|
||||
})
|
||||
})
|
||||
beating = current
|
||||
return current
|
||||
if (outcome.kind === "ok") return outcome.sessions
|
||||
degradedCount++
|
||||
if (outcome.kind === "err") {
|
||||
options.log.warn("remote-ws heartbeat gather rejected, degraded heartbeat", {
|
||||
error: String(outcome.error),
|
||||
degraded: degradedCount,
|
||||
})
|
||||
} else {
|
||||
options.log.warn("remote-ws heartbeat gather timeout, degraded heartbeat", {
|
||||
outstanding,
|
||||
degraded: degradedCount,
|
||||
})
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
function heartbeat(opts?: { requireSessionId?: string }): Promise<void> {
|
||||
if (closed) return Promise.reject(new Error("remote-ws connection closed"))
|
||||
const { promise, waiter } = makeWaiter()
|
||||
waiter.requireSessionId = opts?.requireSessionId
|
||||
waiters.push(waiter)
|
||||
requestCycle()
|
||||
return promise
|
||||
}
|
||||
|
||||
// Interval-driven ticks call requestCycle directly so the periodic heartbeat
|
||||
// never registers a waiter (no waiter accumulation during degradation).
|
||||
function requestCycle() {
|
||||
queued = true
|
||||
runLoop()
|
||||
}
|
||||
|
||||
function runLoop() {
|
||||
if (beating || closed) return
|
||||
beating = Promise.resolve(
|
||||
withContext(async () => {
|
||||
while (queued && !closed) {
|
||||
queued = false
|
||||
const cycleWaiters = waiters
|
||||
waiters = []
|
||||
const fresh = await gatherOnce()
|
||||
if (closed) {
|
||||
rejectWaiters(cycleWaiters, new Error("remote-ws connection closed"))
|
||||
return
|
||||
}
|
||||
if (fresh !== undefined) {
|
||||
lastGood = fresh
|
||||
const sentLive = ws?.readyState === WebSocket.OPEN
|
||||
send({ type: "heartbeat", protocolVersion: InstallationVersion, sessions: fresh })
|
||||
if (sentLive) {
|
||||
// A waiter requiring a specific id is satisfied only when
|
||||
// the sent payload contains that id. Unsatisfied waiters
|
||||
// are requeued so the periodic interval keeps evaluating
|
||||
// them; they resolve on a future fresh send whose payload
|
||||
// includes their required id (or reject on close).
|
||||
const satisfied: Waiter[] = []
|
||||
const unsatisfied: Waiter[] = []
|
||||
for (const w of cycleWaiters) {
|
||||
if (
|
||||
w.requireSessionId === undefined ||
|
||||
fresh.some((s) => s.id === w.requireSessionId)
|
||||
) {
|
||||
satisfied.push(w)
|
||||
} else {
|
||||
unsatisfied.push(w)
|
||||
}
|
||||
}
|
||||
for (const w of satisfied) w.resolve()
|
||||
if (unsatisfied.length > 0) {
|
||||
waiters = unsatisfied.concat(waiters)
|
||||
}
|
||||
} else {
|
||||
// Buffered because the socket is not open; resolve on the next
|
||||
// fresh send over the (re)connected socket.
|
||||
waiters = cycleWaiters.concat(waiters)
|
||||
}
|
||||
} else {
|
||||
// Degraded: preserve liveness with the last known-good list (empty
|
||||
// on cold start) and keep waiters pending for a future fresh send.
|
||||
send({ type: "heartbeat", protocolVersion: InstallationVersion, sessions: lastGood ?? [] })
|
||||
waiters = cycleWaiters.concat(waiters)
|
||||
}
|
||||
}
|
||||
}),
|
||||
)
|
||||
.catch((err) => options.log.error("remote-ws heartbeat loop failed", { error: String(err) }))
|
||||
.finally(() => {
|
||||
beating = undefined
|
||||
if (queued && !closed) runLoop()
|
||||
})
|
||||
}
|
||||
|
||||
function startHeartbeat() {
|
||||
stopHeartbeat()
|
||||
beat = setInterval(() => {
|
||||
void heartbeat().catch((err) => {
|
||||
options.log.error("remote-ws heartbeat failed", { error: String(err) })
|
||||
})
|
||||
}, interval)
|
||||
beat = timers.setInterval(() => requestCycle(), interval)
|
||||
}
|
||||
|
||||
function stopHeartbeat() {
|
||||
if (beat) clearInterval(beat)
|
||||
if (beat) timers.clearInterval(beat)
|
||||
beat = undefined
|
||||
}
|
||||
|
||||
let activity = Date.now()
|
||||
let watchdog: Timer | undefined
|
||||
let activity = now()
|
||||
let watchdog: unknown
|
||||
const timeout = options.timeout ?? 30_000
|
||||
|
||||
function startWatchdog() {
|
||||
stopWatchdog()
|
||||
watchdog = setInterval(
|
||||
watchdog = timers.setInterval(
|
||||
() => {
|
||||
if (Date.now() - activity > timeout) {
|
||||
if (now() - activity > timeout) {
|
||||
options.log.warn("remote-ws activity timeout, forcing reconnect")
|
||||
stopWatchdog()
|
||||
ws?.close(4000, "activity timeout")
|
||||
@@ -106,88 +306,172 @@ export namespace RemoteWS {
|
||||
}
|
||||
|
||||
function stopWatchdog() {
|
||||
if (watchdog) clearInterval(watchdog)
|
||||
if (watchdog) timers.clearInterval(watchdog)
|
||||
watchdog = undefined
|
||||
}
|
||||
|
||||
// Connect-attempt deadline (covers token acquisition through onopen).
|
||||
let connectDeadline: unknown
|
||||
let currentGen = 0
|
||||
|
||||
function startConnectDeadline(g: Gen) {
|
||||
stopConnectDeadline()
|
||||
connectDeadline = timers.setTimeout(() => {
|
||||
connectDeadline = undefined
|
||||
if (closed || g.settled) return
|
||||
options.log.warn("remote-ws connect attempt deadline, will retry", { gen: g.id })
|
||||
if (ws) ws.close(4001, "connect timeout")
|
||||
scheduleRetry(g)
|
||||
}, connectTimeout)
|
||||
}
|
||||
|
||||
function stopConnectDeadline() {
|
||||
if (connectDeadline) timers.clearTimeout(connectDeadline)
|
||||
connectDeadline = undefined
|
||||
}
|
||||
|
||||
// Single fenced retry owner: exactly one of {token-failure, connect-deadline,
|
||||
// onclose, sync-throw} may schedule a retry for a given generation.
|
||||
function scheduleRetry(g: Gen) {
|
||||
if (closed || g.settled) return
|
||||
g.settled = true
|
||||
schedule()
|
||||
}
|
||||
|
||||
function withTimeout<T>(promise: Promise<T>, ms: number, label: string): Promise<T> {
|
||||
return new Promise<T>((resolve, reject) => {
|
||||
let done = false
|
||||
const t = timers.setTimeout(() => {
|
||||
if (done) return
|
||||
done = true
|
||||
reject(new Error(label))
|
||||
}, ms)
|
||||
promise.then(
|
||||
(v) => {
|
||||
if (done) return
|
||||
done = true
|
||||
timers.clearTimeout(t)
|
||||
resolve(v)
|
||||
},
|
||||
(err) => {
|
||||
if (done) return
|
||||
done = true
|
||||
timers.clearTimeout(t)
|
||||
reject(err)
|
||||
},
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
async function open() {
|
||||
if (closed) return
|
||||
const token = await options.getToken()
|
||||
if (closed) return
|
||||
if (!token) {
|
||||
options.log.warn("remote-ws no token, will retry")
|
||||
schedule()
|
||||
return
|
||||
}
|
||||
const endpoint = `${options.url}/api/user/cli?token=${encodeURIComponent(token)}&connectionId=${connectionId}`
|
||||
options.log.info("remote-ws connecting", { connectionId, endpoint: endpoint.replace(/token=[^&]+/, "token=***") })
|
||||
const socket = new WebSocket(endpoint)
|
||||
ws = socket
|
||||
|
||||
socket.onopen = () => {
|
||||
if (ws !== socket || closed) {
|
||||
socket.close()
|
||||
return
|
||||
}
|
||||
options.log.info("remote-ws connected", { buffered: buffer.length })
|
||||
void withContext(() => options.onOpen?.())
|
||||
backoff = 1000
|
||||
for (const msg of buffer) socket.send(msg)
|
||||
buffer.length = 0
|
||||
activity = Date.now()
|
||||
startHeartbeat()
|
||||
startWatchdog()
|
||||
}
|
||||
|
||||
socket.onmessage = (event) => {
|
||||
if (ws !== socket || closed) return
|
||||
activity = Date.now()
|
||||
const raw = String(event.data)
|
||||
let json: unknown
|
||||
const g: Gen = { id: ++currentGen, settled: false, opened: false }
|
||||
startConnectDeadline(g)
|
||||
try {
|
||||
let token: string | undefined
|
||||
try {
|
||||
json = JSON.parse(raw)
|
||||
} catch {
|
||||
options.log.warn("remote-ws invalid JSON", { bytes: raw.length })
|
||||
token = await withTimeout(options.getToken(), tokenTimeout, "remote-ws token timeout")
|
||||
} catch (err) {
|
||||
if (closed) return
|
||||
options.log.warn("remote-ws getToken failed, will retry", { gen: g.id, error: String(err) })
|
||||
scheduleRetry(g)
|
||||
return
|
||||
}
|
||||
const preview = RemoteProtocol.Preview.safeParse(json)
|
||||
options.log.info("remote-ws received", { bytes: raw.length, ...preview.data })
|
||||
const parsed = RemoteProtocol.Inbound.safeParse(json)
|
||||
if (!parsed.success) {
|
||||
options.log.warn("remote-ws message parse failed", { error: parsed.error })
|
||||
if (closed || g.settled) return
|
||||
if (!token) {
|
||||
options.log.warn("remote-ws no token, will retry", { gen: g.id })
|
||||
scheduleRetry(g)
|
||||
return
|
||||
}
|
||||
options.onMessage?.(parsed.data)
|
||||
}
|
||||
const endpoint = `${options.url}/api/user/cli?token=${encodeURIComponent(token)}&connectionId=${connectionId}`
|
||||
options.log.info("remote-ws connecting", { connectionId, gen: g.id, endpoint: endpoint.replace(/token=[^&]+/, "token=***") })
|
||||
let socket: WebSocket
|
||||
try {
|
||||
socket = new WebSocket(endpoint)
|
||||
} catch (err) {
|
||||
if (closed) return
|
||||
options.log.warn("remote-ws constructor threw, will retry", { gen: g.id, error: String(err) })
|
||||
scheduleRetry(g)
|
||||
return
|
||||
}
|
||||
ws = socket
|
||||
|
||||
socket.onclose = (event) => {
|
||||
if (ws !== socket) return
|
||||
options.log.info("remote-ws closed", { code: event.code, reason: event.reason })
|
||||
ws = undefined
|
||||
stopHeartbeat()
|
||||
stopWatchdog()
|
||||
socket.onopen = () => {
|
||||
if (g.settled || ws !== socket || closed) {
|
||||
socket.close()
|
||||
return
|
||||
}
|
||||
g.opened = true
|
||||
stopConnectDeadline()
|
||||
options.log.info("remote-ws connected", { gen: g.id, buffered: buffer.length })
|
||||
void withContext(() => options.onOpen?.())
|
||||
backoff = 1000
|
||||
for (const msg of buffer) socket.send(msg)
|
||||
buffer.length = 0
|
||||
activity = now()
|
||||
startHeartbeat()
|
||||
startWatchdog()
|
||||
if (waiters.length > 0) requestCycle()
|
||||
}
|
||||
|
||||
socket.onmessage = (event) => {
|
||||
if (g.settled || ws !== socket || closed) return
|
||||
activity = now()
|
||||
const raw = String(event.data)
|
||||
let json: unknown
|
||||
try {
|
||||
json = JSON.parse(raw)
|
||||
} catch {
|
||||
options.log.warn("remote-ws invalid JSON", { bytes: raw.length })
|
||||
return
|
||||
}
|
||||
const preview = RemoteProtocol.Preview.safeParse(json)
|
||||
options.log.info("remote-ws received", { bytes: raw.length, ...preview.data })
|
||||
const parsed = RemoteProtocol.Inbound.safeParse(json)
|
||||
if (!parsed.success) {
|
||||
options.log.warn("remote-ws message parse failed", { error: parsed.error })
|
||||
return
|
||||
}
|
||||
options.onMessage?.(parsed.data)
|
||||
}
|
||||
|
||||
socket.onclose = (event) => {
|
||||
if (ws !== socket) return
|
||||
stopConnectDeadline()
|
||||
options.log.info("remote-ws closed", { code: event.code, reason: event.reason, gen: g.id })
|
||||
ws = undefined
|
||||
stopHeartbeat()
|
||||
stopWatchdog()
|
||||
if (closed) return
|
||||
if (event.code === 4401 || event.code === 4403 || event.code === 4409) {
|
||||
options.log.warn("remote-ws closed permanently", {
|
||||
code: event.code,
|
||||
reason: event.reason,
|
||||
})
|
||||
const pending = waiters
|
||||
waiters = []
|
||||
rejectWaiters(pending, new Error("remote-ws connection permanently closed"))
|
||||
void withContext(() => options.onClose?.(event.code, event.reason))
|
||||
return
|
||||
}
|
||||
if (g.opened) void withContext(() => options.onDisconnect?.())
|
||||
scheduleRetry(g)
|
||||
}
|
||||
|
||||
socket.onerror = (event) => {
|
||||
if (g.settled || ws !== socket || closed) return
|
||||
options.log.error("remote-ws error", { error: event })
|
||||
}
|
||||
} catch (err) {
|
||||
if (closed) return
|
||||
if (event.code === 4401 || event.code === 4403 || event.code === 4409) {
|
||||
options.log.warn("remote-ws closed permanently", {
|
||||
code: event.code,
|
||||
reason: event.reason,
|
||||
})
|
||||
void withContext(() => options.onClose?.(event.code, event.reason))
|
||||
return
|
||||
}
|
||||
void withContext(() => options.onDisconnect?.())
|
||||
schedule()
|
||||
}
|
||||
|
||||
socket.onerror = (event) => {
|
||||
if (ws !== socket || closed) return
|
||||
options.log.error("remote-ws error", { error: event })
|
||||
options.log.warn("remote-ws open threw, will retry", { gen: g.id, error: String(err) })
|
||||
scheduleRetry(g)
|
||||
}
|
||||
}
|
||||
|
||||
function schedule() {
|
||||
if (closed) return
|
||||
timer = setTimeout(() => open(), backoff)
|
||||
timer = timers.setTimeout(() => open(), backoff)
|
||||
backoff = Math.min(backoff * 2, 60000)
|
||||
}
|
||||
|
||||
@@ -206,8 +490,12 @@ export namespace RemoteWS {
|
||||
queued = false
|
||||
stopHeartbeat()
|
||||
stopWatchdog()
|
||||
if (timer) clearTimeout(timer)
|
||||
stopConnectDeadline()
|
||||
if (timer) timers.clearTimeout(timer)
|
||||
if (ws) ws.close()
|
||||
const pending = waiters
|
||||
waiters = []
|
||||
rejectWaiters(pending, new Error("remote-ws connection closed"))
|
||||
}
|
||||
|
||||
void open()
|
||||
|
||||
@@ -744,4 +744,29 @@ describe("AttachedState", () => {
|
||||
await replacement
|
||||
expect([...state.union()].sort()).toEqual(["ses_x"])
|
||||
})
|
||||
|
||||
// AC6d: announce(id) must forward { requireSessionId: id } to the
|
||||
// heartbeat callback so the relay only resolves the attach once a fresh
|
||||
// heartbeat whose payload contains that id was actually sent. Presence
|
||||
// fire-and-forget heartbeats (from setPresence) continue to call
|
||||
// without an id and resolve on any fresh send.
|
||||
test("announce(id) forwards { requireSessionId: id } to the heartbeat callback", async () => {
|
||||
const calls: Array<{ requireSessionId?: string }> = []
|
||||
const state = AttachedState.create({
|
||||
heartbeat: (opts) => {
|
||||
calls.push(opts ? { ...opts } : {})
|
||||
return Promise.resolve()
|
||||
},
|
||||
log: nolog,
|
||||
})
|
||||
|
||||
// setPresence fires a fire-and-forget heartbeat with NO id.
|
||||
state.setPresence(["ses_a"])
|
||||
await Promise.resolve()
|
||||
|
||||
// announce(id) forwards the id to the awaited heartbeat.
|
||||
await state.announce("ses_b")
|
||||
|
||||
expect(calls).toEqual([{}, { requireSessionId: "ses_b" }])
|
||||
})
|
||||
})
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user