fix(core): recover hub sessions after daemon replacement

This commit is contained in:
Saoud Rizwan
2026-05-22 16:03:02 -07:00
parent 0157ed9efb
commit 4e146c312e
6 changed files with 367 additions and 64 deletions
@@ -10,6 +10,17 @@ const getClientIdMock = vi.hoisted(() => vi.fn(() => "client-1"));
const restartLocalHubIfIdleAfterStartupTimeoutMock = vi.hoisted(() => vi.fn());
vi.mock("../client", () => ({
HubCommandError: class HubCommandError extends Error {
readonly command: string;
readonly code: string | undefined;
constructor(command: string, code: string | undefined, message: string) {
super(message);
this.name = "HubCommandError";
this.command = command;
this.code = code;
}
},
NodeHubClient: class {
private readonly url: string;
@@ -52,6 +63,31 @@ function createConfig() {
};
}
function createRunResult(text = "Hey!") {
return {
text,
usage: {
inputTokens: 1,
outputTokens: 1,
cacheReadTokens: 0,
cacheWriteTokens: 0,
totalCost: 0,
},
messages: [],
toolCalls: [],
iterations: 1,
finishReason: "completed" as const,
model: {
id: "anthropic/claude-haiku-4.5",
provider: "cline",
info: {},
},
startedAt: new Date("2026-04-21T00:00:00.000Z"),
endedAt: new Date("2026-04-21T00:00:01.000Z"),
durationMs: 1000,
};
}
function agentDoneEvents(events: unknown[]) {
return events.filter(
(
@@ -181,28 +217,7 @@ describe("HubRuntimeHost", () => {
it("starts runs only through send", async () => {
subscribeMock.mockReturnValue(() => {});
const result = {
text: "Hey!",
usage: {
inputTokens: 1,
outputTokens: 1,
cacheReadTokens: 0,
cacheWriteTokens: 0,
totalCost: 0,
},
messages: [],
toolCalls: [],
iterations: 1,
finishReason: "completed",
model: {
id: "anthropic/claude-haiku-4.5",
provider: "cline",
info: {},
},
startedAt: new Date("2026-04-21T00:00:00.000Z"),
endedAt: new Date("2026-04-21T00:00:01.000Z"),
durationMs: 1000,
};
const result = createRunResult();
commandMock.mockResolvedValue({ ok: true, payload: { result } });
const { HubRuntimeHost } = await import("./hub-runtime-host");
@@ -233,6 +248,89 @@ describe("HubRuntimeHost", () => {
expect(sent).toEqual(result);
});
it("recreates a missing hub session with the same id and retries the turn", async () => {
subscribeMock.mockReturnValue(() => {});
const { HubCommandError } = await import("../client");
const result = createRunResult("Recovered");
let createAttempts = 0;
let runAttempts = 0;
commandMock.mockImplementation((command: string) => {
if (command === "session.create") {
createAttempts += 1;
return Promise.resolve({
payload: {
session: {
sessionId: "sess-1",
status: "running",
createdAt: Date.now(),
updatedAt: Date.now(),
workspaceRoot: "/tmp/project",
cwd: "/tmp/project",
},
},
});
}
if (command === "session.messages") {
return Promise.resolve({ payload: { messages: [] } });
}
if (command === "run.start") {
runAttempts += 1;
if (runAttempts === 1) {
return Promise.reject(
new HubCommandError(
"run.start",
"session_not_found",
"session not found: sess-1",
),
);
}
return Promise.resolve({ payload: { result } });
}
return Promise.resolve({ payload: {} });
});
const askQuestion = vi.fn(
async (
_question: string,
_options: string[],
_context: AgentToolContext,
) => "Use the SDK",
);
const { HubRuntimeHost } = await import("./hub-runtime-host");
const host = new HubRuntimeHost({ url: "ws://127.0.0.1:25463/hub" });
const started = await host.startSession({
config: createConfig(),
source: SessionSource.CLI,
interactive: true,
capabilities: { toolExecutors: { askQuestion } },
});
const sent = await host.runTurn({
sessionId: started.sessionId,
prompt: "Hey",
mode: "act",
});
const createCalls = commandMock.mock.calls.filter(
(call) => call[0] === "session.create",
);
expect(sent).toEqual(result);
expect(createAttempts).toBe(2);
expect(runAttempts).toBe(2);
expect(createCalls[1]?.[1]).toMatchObject({
sessionConfig: expect.objectContaining({ sessionId: "sess-1" }),
runtimeOptions: {
clientContributions: [
{
kind: "toolExecutor",
executor: "askQuestion",
capabilityName: "tool_executor.askQuestion",
},
],
},
});
});
it("projects canonical hub snapshots from replies and lifecycle events", async () => {
let onEvent: ((event: HubEventEnvelope) => void) | undefined;
subscribeMock.mockImplementation((listener) => {
@@ -1,3 +1,4 @@
import type { Message } from "@cline/llms";
import type {
AgentEvent,
AgentFinishReason,
@@ -65,6 +66,7 @@ import type {
} from "../../types/events";
import type { SessionRecord } from "../../types/sessions";
import {
HubCommandError,
type HubClientOptions,
isHubCommandTimeoutError,
NodeHubClient,
@@ -146,6 +148,11 @@ interface ClientContributionRegistration {
handlers: Map<string, ClientContributionHandler>;
}
interface RetainedStartContext {
input: StartSessionInput;
capabilities: RuntimeCapabilities;
}
function addClientContribution(
registration: ClientContributionRegistration,
contribution: HubClientContribution,
@@ -687,6 +694,8 @@ export class HubRuntimeHost implements RuntimeHost {
string,
Map<string, ClientContributionHandler>
>();
private readonly sessionStartContexts = new Map<string, RetainedStartContext>();
private readonly sessionRecoveryPromises = new Map<string, Promise<boolean>>();
private readonly sessionSubscriptions = new Map<string, () => void>();
private readonly pendingApprovalToolCallIds = new Set<string>();
private readonly agentDoneEmittedForCurrentRunBySession = new Set<string>();
@@ -750,6 +759,72 @@ export class HubRuntimeHost implements RuntimeHost {
return true;
}
private buildSessionCreatePayload(
input: StartSessionInput,
sessionId: string,
clientContributions: ClientContributionRegistration,
): Record<string, unknown> {
return {
workspaceRoot: input.config.workspaceRoot?.trim() || input.config.cwd,
cwd: input.config.cwd,
sessionConfig: toJsonRecord({
...(input.config as Record<string, unknown>),
sessionId,
}),
metadata: {
...(input.sessionMetadata ?? {}),
source: input.source ?? SessionSource.CORE,
provider: input.config.providerId,
model: input.config.modelId,
enableTools: input.config.enableTools,
enableSpawn: input.config.enableSpawnAgent,
enableTeams: input.config.enableAgentTeams,
teamName: input.config.teamName,
prompt: input.prompt,
interactive: input.interactive === true,
},
runtimeOptions: {
...(clientContributions.manifest.length > 0
? { clientContributions: clientContributions.manifest }
: {}),
...(input.localRuntime?.configExtensions
? { configExtensions: input.localRuntime.configExtensions }
: {}),
},
toolPolicies: toJsonRecord(
input.toolPolicies as Record<string, unknown> | undefined,
),
initialMessages: input.initialMessages,
};
}
private async sendCreateSessionCommand(
input: StartSessionInput,
sessionId: string,
clientContributions: ClientContributionRegistration,
): Promise<Awaited<ReturnType<NodeHubClient["command"]>>> {
return await this.client.command(
"session.create",
this.buildSessionCreatePayload(input, sessionId, clientContributions),
);
}
private retainSessionStartContext(
sessionId: string,
input: StartSessionInput,
capabilities: RuntimeCapabilities,
): void {
this.sessionStartContexts.set(sessionId, {
input: {
...input,
initialMessages: undefined,
prompt: undefined,
config: { ...input.config, sessionId },
},
capabilities,
});
}
private registerPlannedSession(
sessionId: string,
capabilities: RuntimeCapabilities,
@@ -773,6 +848,8 @@ export class HubRuntimeHost implements RuntimeHost {
private cleanupPlannedSession(sessionId: string): void {
this.sessionCapabilities.delete(sessionId);
this.sessionClientContributionHandlers.delete(sessionId);
this.sessionStartContexts.delete(sessionId);
this.sessionRecoveryPromises.delete(sessionId);
this.disposeSessionSubscription(sessionId);
}
@@ -788,39 +865,6 @@ export class HubRuntimeHost implements RuntimeHost {
);
const plannedSessionId =
input.config.sessionId?.trim() || createSessionId();
const sendCreateCommand = () =>
this.client.command("session.create", {
workspaceRoot: input.config.workspaceRoot?.trim() || input.config.cwd,
cwd: input.config.cwd,
sessionConfig: toJsonRecord({
...(input.config as Record<string, unknown>),
sessionId: plannedSessionId,
}),
metadata: {
...(input.sessionMetadata ?? {}),
source: input.source ?? SessionSource.CORE,
provider: input.config.providerId,
model: input.config.modelId,
enableTools: input.config.enableTools,
enableSpawn: input.config.enableSpawnAgent,
enableTeams: input.config.enableAgentTeams,
teamName: input.config.teamName,
prompt: input.prompt,
interactive: input.interactive === true,
},
runtimeOptions: {
...(clientContributions.manifest.length > 0
? { clientContributions: clientContributions.manifest }
: {}),
...(input.localRuntime?.configExtensions
? { configExtensions: input.localRuntime.configExtensions }
: {}),
},
toolPolicies: toJsonRecord(
input.toolPolicies as Record<string, unknown> | undefined,
),
initialMessages: input.initialMessages,
});
this.registerPlannedSession(
plannedSessionId,
capabilities,
@@ -828,7 +872,11 @@ export class HubRuntimeHost implements RuntimeHost {
);
let reply: Awaited<ReturnType<NodeHubClient["command"]>>;
try {
reply = await sendCreateCommand();
reply = await this.sendCreateSessionCommand(
input,
plannedSessionId,
clientContributions,
);
} catch (error) {
this.cleanupPlannedSession(plannedSessionId);
if (await this.recoverLocalHubStartupDeadlock(error)) {
@@ -838,7 +886,11 @@ export class HubRuntimeHost implements RuntimeHost {
clientContributions.handlers,
);
try {
reply = await sendCreateCommand();
reply = await this.sendCreateSessionCommand(
input,
plannedSessionId,
clientContributions,
);
} catch (retryError) {
this.cleanupPlannedSession(plannedSessionId);
throw retryError;
@@ -862,6 +914,7 @@ export class HubRuntimeHost implements RuntimeHost {
clientContributions.handlers,
);
}
this.retainSessionStartContext(sessionId, input, capabilities);
return {
sessionId,
@@ -1044,7 +1097,82 @@ export class HubRuntimeHost implements RuntimeHost {
};
}
async runTurn(input: SendSessionInput): Promise<AgentResult | undefined> {
private async readMessagesForRecovery(sessionId: string): Promise<Message[]> {
try {
return await this.readSessionMessages(sessionId);
} catch {
return [];
}
}
private async recoverMissingActiveSession(
sessionId: string,
): Promise<boolean> {
const existing = this.sessionRecoveryPromises.get(sessionId);
if (existing) {
return await existing;
}
const recovery = this.recoverMissingActiveSessionOnce(sessionId).finally(
() => {
this.sessionRecoveryPromises.delete(sessionId);
},
);
this.sessionRecoveryPromises.set(sessionId, recovery);
return await recovery;
}
private async recoverMissingActiveSessionOnce(
sessionId: string,
): Promise<boolean> {
const retained = this.sessionStartContexts.get(sessionId);
if (!retained) {
return false;
}
const messages = await this.readMessagesForRecovery(sessionId);
const startInput = {
...retained.input,
...(messages.length > 0 ? { initialMessages: messages } : {}),
};
const clientContributions = buildClientContributionRegistration(
startInput.localRuntime,
retained.capabilities,
);
this.registerPlannedSession(
sessionId,
retained.capabilities,
clientContributions.handlers,
);
try {
const reply = await this.sendCreateSessionCommand(
startInput,
sessionId,
clientContributions,
);
const snapshot = parseCoreSessionSnapshot(reply.payload?.snapshot);
const session = reply.payload?.session as HubSessionRecord | undefined;
return (snapshot?.sessionId ?? session?.sessionId)?.trim() === sessionId;
} catch (error) {
captureSdkError(this.telemetry, {
component: "core",
operation: "hub.runtime_host.recover_missing_session",
error,
severity: "warn",
handled: true,
context: { sessionId },
});
return false;
}
}
private isRecoverableSessionNotFoundError(
error: unknown,
): error is HubCommandError {
return error instanceof HubCommandError && error.code === "session_not_found";
}
private async sendRunTurnCommand(
input: SendSessionInput,
): Promise<AgentResult | undefined> {
this.ensureSessionSubscription(input.sessionId);
const reply = await this.client.command(
"run.start",
@@ -1075,6 +1203,20 @@ export class HubRuntimeHost implements RuntimeHost {
return reply.payload?.result as AgentResult | undefined;
}
async runTurn(input: SendSessionInput): Promise<AgentResult | undefined> {
try {
return await this.sendRunTurnCommand(input);
} catch (error) {
if (
!this.isRecoverableSessionNotFoundError(error) ||
!(await this.recoverMissingActiveSession(input.sessionId))
) {
throw error;
}
}
return await this.sendRunTurnCommand(input);
}
private async requestPendingPromptsList(
input: Parameters<PendingPromptsServiceApi["list"]>[0],
): Promise<SessionPendingPrompt[]> {
@@ -1159,6 +1301,9 @@ export class HubRuntimeHost implements RuntimeHost {
async stopSession(sessionId: string): Promise<void> {
this.sessionCapabilities.delete(sessionId);
this.sessionClientContributionHandlers.delete(sessionId);
this.sessionStartContexts.delete(sessionId);
this.sessionRecoveryPromises.delete(sessionId);
this.disposeSessionSubscription(sessionId);
await this.client.command("session.detach", { sessionId }, sessionId);
}
@@ -1174,6 +1319,9 @@ export class HubRuntimeHost implements RuntimeHost {
}
this.sessionSubscriptions.clear();
this.sessionCapabilities.clear();
this.sessionClientContributionHandlers.clear();
this.sessionStartContexts.clear();
this.sessionRecoveryPromises.clear();
this.agentDoneEmittedForCurrentRunBySession.clear();
for (const controller of this.activeCapabilityAbortControllers.values()) {
controller.abort("Hub runtime host disposed.");
@@ -161,6 +161,38 @@ describe("run handlers", () => {
await expect(promise).resolves.toMatchObject({ ok: true });
});
it("returns typed session_not_found errors without publishing run.failed", async () => {
const missingSessionError = Object.assign(
new Error("session not found: session-1"),
{ code: "session_not_found" as const },
);
const ctx = createContext({
runTurn: vi.fn().mockRejectedValue(missingSessionError),
});
const reply = await handleSessionInput(ctx, {
version: "v1",
command: "run.start",
requestId: "req-missing-session",
sessionId: "session-1",
payload: { sessionId: "session-1", prompt: "go" },
});
expect(reply).toMatchObject({
ok: false,
error: {
code: "session_not_found",
message: "session not found: session-1",
},
});
expect(ctx.events).not.toContainEqual(
expect.objectContaining({
event: "run.failed",
sessionId: "session-1",
}),
);
});
it("treats abort as applied when the runtime abort hook rejects", async () => {
const abort = vi.fn().mockRejectedValue(new Error("Run aborted"));
const ctx = createContext({ abort });
@@ -64,6 +64,12 @@ function parseRunTimeoutMs(
return undefined;
}
function errorCode(error: unknown): string | undefined {
return error && typeof error === "object" && "code" in error
? String((error as { code?: unknown }).code)
: undefined;
}
async function runTurnWithRuntimeHealth(
ctx: HubTransportContext,
envelope: HubCommandEnvelope,
@@ -222,6 +228,13 @@ export async function handleSessionInput(
) {
ctx.suppressNextTerminalEventBySession.delete(sessionId);
}
if (errorCode(error) === "session_not_found") {
return errorReply(
envelope,
"session_not_found",
error instanceof Error ? error.message : String(error),
);
}
ctx.publish(
ctx.buildEvent(
"run.failed",
@@ -313,9 +313,18 @@ describe("LocalRuntimeHost", () => {
telemetry,
});
await expect(
manager.runTurn({ sessionId: "missing-session", prompt: "hi" }),
).rejects.toThrow("session not found: missing-session");
let thrown: unknown;
try {
await manager.runTurn({ sessionId: "missing-session", prompt: "hi" });
} catch (error) {
thrown = error;
}
expect(thrown).toBeInstanceOf(Error);
expect(thrown).toMatchObject({
code: "session_not_found",
message: "session not found: missing-session",
});
expect(adapter.emit).toHaveBeenCalledWith(
"sdk.error",
@@ -4963,7 +4972,7 @@ describe("LocalRuntimeHost", () => {
}),
};
const agent = {
// No `submit_and_exit` in toolCalls represents a non-interactive
// No `submit_and_exit` in toolCalls - represents a non-interactive
// run that finished cleanly without invoking the completion tool.
run: vi.fn().mockResolvedValue(createResult({ toolCalls: [] })),
continue: vi.fn(),
@@ -5071,7 +5080,7 @@ describe("LocalRuntimeHost", () => {
shutdown: vi.fn(),
}),
};
// Non-interactive: executeAgentTurn finalizeSingleRun shutdownSession,
// Non-interactive: executeAgentTurn to finalizeSingleRun to shutdownSession,
// all in the same `startSession(...)` call. We must see exactly one emission.
const agent = {
run: vi.fn().mockResolvedValue(
@@ -1573,7 +1573,10 @@ export class LocalRuntimeHost implements RuntimeHost {
private getSessionOrThrow(sessionId: string): ActiveSession {
const session = this.sessions.get(sessionId);
if (!session) {
const error = new Error(`session not found: ${sessionId}`);
const error = Object.assign(
new Error(`session not found: ${sessionId}`),
{ code: "session_not_found" as const },
);
captureSdkError(this.defaultTelemetry, {
component: "core",
operation: "session.active_lookup",