fix(hub): retire stale hubs before respawning in ensureCompatibleLocalHubUrl

Stacked on #12329. With the buildId gate restored there, an upgraded
client that calls ensureCompatibleLocalHubUrl while the old daemon still
holds the configured port hits a dead end: the raw spawn dies on
EADDRINUSE and the discovery poll spins for its full 8s timeout before
returning undefined (Greptile P2 on #12329) — the stale hub keeps
running unless the background prewarm happens to win the race.

Delegate to ensureDetachedHubServer, which retires the incompatible hub
(graceful /shutdown -> SIGTERM -> port-clear wait) before spawning and
verifying the replacement. The client-local blind-spawn helpers
(waitForCompatibleHubUrl and its startup constants) lose their only
caller and are removed.

The delegation relies on #12329 keeping the discovery record on
build_mismatch: the record carries the authToken/pid retirement needs;
a regression test pins that the client never clears it before the
daemon retires the hub.

sdk/packages/core/src/hub/client/index.test.ts

@@ -543,7 +543,9 @@ describe("NodeHubClient", () => {
 			globalThis as unknown as { WebSocket?: typeof RecoveryWebSocket }
 		).WebSocket = RecoveryWebSocket;
 		vi.doMock("../daemon", () => ({
-			spawnDetachedHubServerWithRetry: vi.fn(async () => undefined),
+			ensureDetachedHubServer: vi.fn(async () => {
+				throw new Error("unexpected ensureDetachedHubServer call");
+			}),
 		}));
 		vi.doMock("../discovery/workspace", () => ({
 			resolveProductionHubOwnerContext: () => ({
@@ -698,7 +700,9 @@ describe("NodeHubClient", () => {
 			globalThis as unknown as { WebSocket?: typeof ExplicitEndpointWebSocket }
 		).WebSocket = ExplicitEndpointWebSocket;
 		vi.doMock("../daemon", () => ({
-			spawnDetachedHubServerWithRetry: vi.fn(async () => undefined),
+			ensureDetachedHubServer: vi.fn(async () => {
+				throw new Error("unexpected ensureDetachedHubServer call");
+			}),
 		}));
 		vi.doMock("../discovery/workspace", () => ({
 			resolveProductionHubOwnerContext: () => ({
@@ -810,7 +814,7 @@ describe("resolveCompatibleLocalHubUrl", () => {
 		expect(clearHubDiscoveryMock).not.toHaveBeenCalled();
 	});
 
-	it("keeps discovery on build mismatch when protocol is compatible", async () => {
+	it("returns undefined on build mismatch but keeps discovery for retirement", async () => {
 		const clearHubDiscoveryMock = vi.fn();
 		vi.doMock("../discovery/workspace", () => ({
 			resolveProductionHubOwnerContext: () => ({
@@ -856,13 +860,11 @@ describe("resolveCompatibleLocalHubUrl", () => {
 
 		const { resolveCompatibleLocalHubUrl } = await import(".");
 
-		await expect(resolveCompatibleLocalHubUrl()).resolves.toBe(
-			"ws://127.0.0.1:59999/hub",
-		);
+		await expect(resolveCompatibleLocalHubUrl()).resolves.toBeUndefined();
 		expect(clearHubDiscoveryMock).not.toHaveBeenCalled();
 	});
 
-	it("keeps discovery when a hub omits build metadata but has compatible protocol", async () => {
+	it("returns undefined and keeps discovery when a hub omits build metadata", async () => {
 		const clearHubDiscoveryMock = vi.fn();
 		vi.doMock("../discovery/workspace", () => ({
 			resolveProductionHubOwnerContext: () => ({
@@ -906,9 +908,7 @@ describe("resolveCompatibleLocalHubUrl", () => {
 
 		const { resolveCompatibleLocalHubUrl } = await import(".");
 
-		await expect(resolveCompatibleLocalHubUrl()).resolves.toBe(
-			"ws://127.0.0.1:59999/hub",
-		);
+		await expect(resolveCompatibleLocalHubUrl()).resolves.toBeUndefined();
 		expect(clearHubDiscoveryMock).not.toHaveBeenCalled();
 	});
 
@@ -963,26 +963,15 @@ describe("resolveCompatibleLocalHubUrl", () => {
 		);
 	});
 
-	it("starts missing local hubs through the retrying daemon spawn API", async () => {
+	it("starts missing local hubs through the daemon ensure API", async () => {
 		vi.stubGlobal("WebSocket", MockWebSocket);
-		const spawnDetachedHubServerWithRetryMock = vi.fn(async () => undefined);
-		const record = {
-			hubId: "hub-test",
-			protocolVersion: "v1",
-			buildId: "test-build",
-			authToken: "token",
-			host: "127.0.0.1",
-			port: 25464,
+		const ensureDetachedHubServerMock = vi.fn(async () => ({
 			url: "ws://127.0.0.1:25464/hub",
-			startedAt: new Date().toISOString(),
-			updatedAt: new Date().toISOString(),
-		};
-		const readHubDiscoveryMock = vi
-			.fn()
-			.mockResolvedValueOnce(undefined)
-			.mockResolvedValueOnce(record);
+			authToken: "token",
+		}));
+		const readHubDiscoveryMock = vi.fn().mockResolvedValue(undefined);
 		vi.doMock("../daemon", () => ({
-			spawnDetachedHubServerWithRetry: spawnDetachedHubServerWithRetryMock,
+			ensureDetachedHubServer: ensureDetachedHubServerMock,
 		}));
 		vi.doMock("../discovery/workspace", () => ({
 			resolveProductionHubOwnerContext: () => ({
@@ -1001,7 +990,7 @@ describe("resolveCompatibleLocalHubUrl", () => {
 				...actual,
 				resolveHubBuildId: () => "test-build",
 				readHubDiscovery: readHubDiscoveryMock,
-				probeHubServer: vi.fn(async () => record),
+				probeHubServer: vi.fn(async () => undefined),
 				clearHubDiscovery: vi.fn(async () => undefined),
 			};
 		});
@@ -1014,18 +1003,109 @@ describe("resolveCompatibleLocalHubUrl", () => {
 				cwd: "/tmp/project",
 			}),
 		).resolves.toBe("ws://127.0.0.1:25464/hub");
-		expect(spawnDetachedHubServerWithRetryMock).toHaveBeenCalledWith(
-			"/tmp/project",
-		);
-		expect(
-			spawnDetachedHubServerWithRetryMock.mock.invocationCallOrder[0],
-		).toBeGreaterThan(readHubDiscoveryMock.mock.invocationCallOrder[0]);
-		expect(
-			spawnDetachedHubServerWithRetryMock.mock.invocationCallOrder[0],
-		).toBeLessThan(readHubDiscoveryMock.mock.invocationCallOrder[1]);
+		expect(ensureDetachedHubServerMock).toHaveBeenCalledWith("/tmp/project");
 	});
 
-	it("waits on shared discovery after spawning in development builds", async () => {
+	it("replaces a stale-build hub through the daemon ensure API without dropping its discovery record", async () => {
+		vi.stubGlobal("WebSocket", MockWebSocket);
+		const clearHubDiscoveryMock = vi.fn();
+		const ensureDetachedHubServerMock = vi.fn(async () => ({
+			url: "ws://127.0.0.1:25465/hub",
+			authToken: "new-token",
+		}));
+		const staleRecord = {
+			hubId: "hub-test",
+			protocolVersion: "v1",
+			buildId: "old-build",
+			authToken: "old-token",
+			host: "127.0.0.1",
+			port: 25464,
+			url: "ws://127.0.0.1:25464/hub",
+			startedAt: new Date().toISOString(),
+			updatedAt: new Date().toISOString(),
+		};
+		vi.doMock("../daemon", () => ({
+			ensureDetachedHubServer: ensureDetachedHubServerMock,
+		}));
+		vi.doMock("../discovery/workspace", () => ({
+			resolveProductionHubOwnerContext: () => ({
+				ownerId: "hub-test",
+				discoveryPath: "/tmp/hub-discovery.json",
+			}),
+			resolveSharedHubOwnerContext: () => ({
+				ownerId: "hub-test",
+				discoveryPath: "/tmp/hub-discovery.json",
+			}),
+		}));
+		vi.doMock("../discovery", async () => {
+			const actual =
+				await vi.importActual<typeof import("../discovery")>("../discovery");
+			return {
+				...actual,
+				resolveHubBuildId: () => "current-build",
+				readHubDiscovery: vi.fn(async () => staleRecord),
+				probeHubServer: vi.fn(async () => staleRecord),
+				clearHubDiscovery: vi.fn(async (...args: unknown[]) => {
+					clearHubDiscoveryMock(...args);
+				}),
+			};
+		});
+
+		const { ensureCompatibleLocalHubUrl } = await import(".");
+
+		await expect(
+			ensureCompatibleLocalHubUrl({
+				workspaceRoot: "/tmp/project",
+				cwd: "/tmp/project",
+			}),
+		).resolves.toBe("ws://127.0.0.1:25465/hub");
+		expect(ensureDetachedHubServerMock).toHaveBeenCalledWith("/tmp/project");
+		// The stale record must survive until the daemon retires the hub; it
+		// carries the authToken/pid the retirement path needs.
+		expect(clearHubDiscoveryMock).not.toHaveBeenCalled();
+	});
+
+	it("returns undefined when the daemon ensure fails", async () => {
+		vi.stubGlobal("WebSocket", MockWebSocket);
+		const ensureDetachedHubServerMock = vi.fn(async () => {
+			throw new Error("could not be retired automatically");
+		});
+		vi.doMock("../daemon", () => ({
+			ensureDetachedHubServer: ensureDetachedHubServerMock,
+		}));
+		vi.doMock("../discovery/workspace", () => ({
+			resolveProductionHubOwnerContext: () => ({
+				ownerId: "hub-test",
+				discoveryPath: "/tmp/hub-discovery.json",
+			}),
+			resolveSharedHubOwnerContext: () => ({
+				ownerId: "hub-test",
+				discoveryPath: "/tmp/hub-discovery.json",
+			}),
+		}));
+		vi.doMock("../discovery", async () => {
+			const actual =
+				await vi.importActual<typeof import("../discovery")>("../discovery");
+			return {
+				...actual,
+				readHubDiscovery: vi.fn(async () => undefined),
+				probeHubServer: vi.fn(async () => undefined),
+				clearHubDiscovery: vi.fn(async () => undefined),
+			};
+		});
+
+		const { ensureCompatibleLocalHubUrl } = await import(".");
+
+		await expect(
+			ensureCompatibleLocalHubUrl({
+				workspaceRoot: "/tmp/project",
+				cwd: "/tmp/project",
+			}),
+		).resolves.toBeUndefined();
+		expect(ensureDetachedHubServerMock).toHaveBeenCalledWith("/tmp/project");
+	});
+
+	it("resolves the shared-owner discovery hub in development builds", async () => {
 		vi.stubGlobal("WebSocket", MockWebSocket);
 		const originalBuildEnv = process.env.CLINE_BUILD_ENV;
 		process.env.CLINE_BUILD_ENV = "development";
@@ -1062,6 +1142,7 @@ describe("resolveCompatibleLocalHubUrl", () => {
 				await vi.importActual<typeof import("../discovery")>("../discovery");
 			return {
 				...actual,
+				resolveHubBuildId: () => "test-build",
 				readHubDiscovery: readHubDiscoveryMock,
 				probeHubServer: vi.fn(async () => record),
 				clearHubDiscovery: vi.fn(async () => undefined),
@@ -1115,7 +1196,9 @@ describe("resolveCompatibleLocalHubUrl", () => {
 			}),
 		}));
 		vi.doMock("../daemon", () => ({
-			spawnDetachedHubServerWithRetry: vi.fn(async () => undefined),
+			ensureDetachedHubServer: vi.fn(async () => {
+				throw new Error("unexpected ensureDetachedHubServer call");
+			}),
 		}));
 		vi.doMock("../discovery", async () => {
 			const actual =

sdk/packages/core/src/hub/client/index.ts

@@ -13,12 +13,13 @@ import {
 	SESSION_NOT_FOUND_ERROR_CODE,
 	SessionNotFoundError,
 } from "../../runtime/host/runtime-host";
-import { spawnDetachedHubServerWithRetry } from "../daemon";
+import { ensureDetachedHubServer } from "../daemon";
 import {
 	clearHubDiscovery,
 	type HubOwnerContext,
 	probeHubServer,
 	readHubDiscovery,
+	resolveHubBuildId,
 } from "../discovery";
 import {
 	resolveProductionHubOwnerContext,
@@ -178,8 +179,6 @@ export interface LocalHubResolutionOptions {
 	cwd?: string;
 }
 
-const HUB_STARTUP_TIMEOUT_MS = 8_000;
-const HUB_STARTUP_POLL_MS = 200;
 const GLOBAL_SUBSCRIPTION_KEY = "*";
 const HUB_CONNECT_TIMEOUT_MS = 8_000;
 const HUB_AUTH_PROTOCOL_PREFIX = "cline-hub-auth.";
@@ -831,7 +830,7 @@ type HubProbeResult =
 			url: string;
 	  }
 	| {
-			status: "unreachable" | "protocol_mismatch";
+			status: "unreachable" | "protocol_mismatch" | "build_mismatch";
 			url: string;
 	  };
 
@@ -860,6 +859,16 @@ async function probeCompatibleHubUrl(
 			url: normalized,
 		};
 	}
+	// A protocol-compatible hub from an older build keeps serving stale code
+	// after an upgrade; report it so callers stop reusing it and the daemon
+	// ensure/prewarm paths retire it. Missing/blank buildId counts as stale.
+	const recordBuildId = record.buildId?.trim();
+	if (!recordBuildId || recordBuildId !== resolveHubBuildId()) {
+		return {
+			status: "build_mismatch",
+			url: normalized,
+		};
+	}
 	if (
 		options?.verifyConnection === true &&
 		!(await verifyHubConnection(normalized, {
@@ -879,26 +888,6 @@ async function probeCompatibleHubUrl(
 	};
 }
 
-async function waitForCompatibleHubUrl(
-	owner: HubOwnerContext,
-): Promise<string | undefined> {
-	const deadline = Date.now() + HUB_STARTUP_TIMEOUT_MS;
-	while (Date.now() < deadline) {
-		const record = await readHubDiscovery(owner.discoveryPath);
-		if (record?.url) {
-			const compatible = await probeCompatibleHubUrl(record.url, {
-				verifyConnection: true,
-				authToken: record.authToken,
-			});
-			if (compatible.status === "compatible") {
-				return rememberRecoverableLocalHubUrl(compatible.url, record.authToken);
-			}
-		}
-		await new Promise((resolve) => setTimeout(resolve, HUB_STARTUP_POLL_MS));
-	}
-	return undefined;
-}
-
 async function waitForHubToRetire(url: string): Promise<boolean> {
 	const deadline = Date.now() + HUB_RECOVERY_RETIRE_TIMEOUT_MS;
 	while (Date.now() < deadline) {
@@ -994,6 +983,10 @@ export async function resolveCompatibleLocalHubUrl(
 	if (compatible.status === "compatible") {
 		return rememberRecoverableLocalHubUrl(compatible.url, record.authToken);
 	}
+	// Keep the discovery record on build_mismatch: it carries the authToken
+	// and pid the daemon ensure/prewarm paths need to retire the stale hub
+	// gracefully. Clearing it here would leave retirement with only an
+	// unauthenticated /health probe, which cannot stop the old daemon.
 	if (compatible.status === "protocol_mismatch") {
 		await clearHubDiscovery(owner.discoveryPath).catch(() => undefined);
 	}
@@ -1016,9 +1009,17 @@ export async function ensureCompatibleLocalHubUrl(
 	if (options.endpoint?.trim()) {
 		return undefined;
 	}
-	const owner = resolveDefaultHubOwnerContext();
-	await spawnDetachedHubServerWithRetry(options.workspaceRoot ?? process.cwd());
-	return await waitForCompatibleHubUrl(owner);
+	// Delegate to the daemon module's ensure so a stale hub still holding the
+	// configured port is retired before the replacement spawns; a raw spawn
+	// would die on the occupied port and this would time out to undefined.
+	try {
+		const ensured = await ensureDetachedHubServer(
+			options.workspaceRoot ?? process.cwd(),
+		);
+		return ensured.url;
+	} catch {
+		return undefined;
+	}
 }
 
 export async function requestHubShutdown(

sdk/packages/core/src/hub/daemon/index.test.ts

@@ -303,7 +303,7 @@ describe("ensureDetachedHubServer", () => {
 		}
 	});
 
-	it("reuses a protocol-compatible healthy hub from a different build", async () => {
+	it("retires a healthy hub from a different build and starts a replacement", async () => {
 		const kill = vi.spyOn(process, "kill").mockImplementation(() => true);
 		try {
 			readHubDiscovery
@@ -311,13 +311,24 @@ describe("ensureDetachedHubServer", () => {
 					url: "ws://127.0.0.1:25463/hub",
 					authToken: "old-token",
 				})
-				.mockResolvedValueOnce(undefined);
-			probeHubServer.mockResolvedValueOnce({
-				url: "ws://127.0.0.1:25463/hub",
-				protocolVersion: "v1",
-				buildId: "old-build",
-				pid: 12345,
-			});
+				.mockResolvedValueOnce({
+					url: "ws://127.0.0.1:25463/hub",
+					authToken: "new-token",
+				});
+			probeHubServer
+				.mockResolvedValueOnce({
+					url: "ws://127.0.0.1:25463/hub",
+					protocolVersion: "v1",
+					buildId: "old-build",
+					pid: 12345,
+				})
+				.mockResolvedValueOnce(undefined)
+				.mockResolvedValueOnce(undefined)
+				.mockResolvedValueOnce({
+					url: "ws://127.0.0.1:25463/hub",
+					protocolVersion: "v1",
+					buildId: "current-build",
+				});
 			verifyHubConnection.mockResolvedValueOnce(true);
 
 			const { ensureDetachedHubServer } = await import(".");
@@ -325,12 +336,15 @@ describe("ensureDetachedHubServer", () => {
 
 			expect(result).toEqual({
 				url: "ws://127.0.0.1:25463/hub",
-				authToken: "old-token",
+				authToken: "new-token",
 			});
-			expect(requestHubShutdown).not.toHaveBeenCalled();
-			expect(clearHubDiscovery).not.toHaveBeenCalled();
-			expect(kill).not.toHaveBeenCalled();
-			expect(spawn).not.toHaveBeenCalled();
+			expect(requestHubShutdown).toHaveBeenCalledWith(
+				"ws://127.0.0.1:25463/hub",
+				"old-token",
+			);
+			expect(kill).toHaveBeenCalledWith(12345, "SIGTERM");
+			expect(clearHubDiscovery).toHaveBeenCalledWith("/tmp/hub-discovery.json");
+			expect(spawn).toHaveBeenCalledOnce();
 			expect(verifyHubConnection).toHaveBeenCalledOnce();
 		} finally {
 			kill.mockRestore();

sdk/packages/core/src/hub/daemon/index.ts

@@ -22,6 +22,7 @@ import {
 	probeHubServer,
 	readHubDiscovery,
 	resolveClineDataDir,
+	resolveHubBuildId,
 } from "../discovery";
 import {
 	type HubEndpointOverrides,
@@ -66,7 +67,14 @@ function resolveDefaultHubOwnerContext() {
 }
 
 function isCompatibleHubRecord(record: HubServerProbeRecord): boolean {
-	return isHubProtocolCompatible(record).compatible;
+	if (!isHubProtocolCompatible(record).compatible) {
+		return false;
+	}
+	// A protocol-compatible hub from an older build keeps serving stale code
+	// after an upgrade, so require the running hub's build to match ours.
+	// Missing/blank buildId means a pre-buildId daemon and is also retired.
+	const recordBuildId = record.buildId?.trim();
+	return !!recordBuildId && recordBuildId === resolveHubBuildId();
 }
 
 function withMatchingDiscoveryRetirementMetadata(

sdk/packages/core/src/hub/server/hub-websocket-server.ts

@@ -364,6 +364,11 @@ export async function startHubWebSocketServer(
 				minClientProtocolVersion: versionPayload.minClientProtocolVersion,
 				maxClientProtocolVersion: versionPayload.maxClientProtocolVersion,
 				coreVersion: versionPayload.coreVersion,
+				// Unauthenticated probes rely on /health to judge build
+				// compatibility; without buildId every probe would classify this
+				// hub as a stale build. No new exposure: buildId defaults to
+				// coreVersion (above) and /version already serves it untokened.
+				buildId: versionPayload.buildId,
 				host,
 				port,
 				url,
@@ -605,6 +610,10 @@ export async function ensureHubWebSocketServer(
 			if (
 				healthy?.url &&
 				isHubProtocolCompatible(healthy).compatible &&
+				// Never reuse a hub from a different build: it keeps serving
+				// stale code after an upgrade. The authenticated probe reads
+				// /status, which always reports buildId.
+				healthy.buildId?.trim() === resolveHubBuildId() &&
 				(await verifyHubConnection(healthy.url, {
 					authToken: discovered.authToken,
 				}))

sdk/packages/core/src/hub/server/index.test.ts

@@ -11,6 +11,7 @@ import {
 	clearHubDiscovery,
 	createInMemoryHubOwnerContext,
 	readHubDiscovery,
+	resolveHubBuildId,
 	toHubHealthUrl,
 	writeHubDiscovery,
 } from "../discovery";
@@ -205,6 +206,55 @@ describe("hub server startup", () => {
 		}
 	});
 
+	it("reports its buildId on the unauthenticated health endpoint", async () => {
+		const owner = createInMemoryHubOwnerContext("hub-server-test-health-build");
+		const result = await ensureHubWebSocketServer({
+			owner,
+			host: "127.0.0.1",
+			port: 0,
+			pathname: "/hub",
+			runtimeHandlers: createLocalHubScheduleRuntimeHandlers(),
+		});
+		servers.add(requireServer(result.server));
+
+		const health = await fetch(new URL("/health", toHubHealthUrl(result.url)));
+		expect(health.status).toBe(200);
+		const payload = (await health.json()) as { buildId?: string };
+		expect(payload.buildId).toBe(resolveHubBuildId());
+	});
+
+	it("does not reuse a discovered hub from a different build", async () => {
+		const owner = createInMemoryHubOwnerContext("hub-server-test-stale-build");
+		vi.stubEnv("CLINE_HUB_BUILD_ID", "old-build");
+		try {
+			const stale = await startHubWebSocketServer({
+				owner,
+				host: "127.0.0.1",
+				port: 0,
+				pathname: "/hub",
+				runtimeHandlers: createLocalHubScheduleRuntimeHandlers(),
+			});
+			servers.add(stale);
+
+			vi.stubEnv("CLINE_HUB_BUILD_ID", "new-build");
+			const result = await ensureHubWebSocketServer({
+				owner,
+				host: "127.0.0.1",
+				port: 0,
+				pathname: "/hub",
+				allowPortFallback: true,
+				runtimeHandlers: createLocalHubScheduleRuntimeHandlers(),
+			});
+
+			expect(result.action).toBe("started");
+			expect(result.url).not.toBe(stale.url);
+			servers.add(requireServer(result.server));
+		} finally {
+			vi.unstubAllEnvs();
+			await clearHubDiscovery(owner.discoveryPath);
+		}
+	});
+
 	it("shuts down active server through the shutdown endpoint", async () => {
 		const owner = createInMemoryHubOwnerContext("hub-server-test-shutdown");
 		const result = await ensureHubWebSocketServer({