Fix checkpoint initialization to take less time

src/integrations/checkpoints/CheckpointGitOperations.ts

@@ -1,4 +1,5 @@
 import { fileExistsAtPath } from "@utils/fs"
+import { retryWithBackoff } from "@utils/retry"
 import fs from "fs/promises"
 import { globby } from "globby"
 import * as path from "path"
@@ -57,6 +58,11 @@ export class GitOperations {
 	 */
 	public async initShadowGit(gitPath: string, cwd: string, taskId: string): Promise<string> {
 		Logger.info(`Initializing shadow git`)
+		// Clean up any leftover .git_disabled directories from a previous crash/interruption.
+		// If addCheckpointFiles() was interrupted mid disable/enable cycle, nested repos may still be disabled.
+		await this.renameNestedGitRepos(false).catch((error) => {
+			Logger.warn("CheckpointTracker failed best-effort nested git cleanup during shadow git init:", error)
+		})
 
 		// If repo exists, just verify worktree
 		if (await fileExistsAtPath(gitPath)) {
@@ -144,7 +150,7 @@ export class GitOperations {
 		const gitPaths = await globby("**/.git" + (disable ? "" : GIT_DISABLED_SUFFIX), {
 			cwd: this.cwd,
 			onlyDirectories: true,
-			ignore: [".git"], // Ignore root level .git
+			ignore: [".git", "**/node_modules/**"], // Ignore root level .git and node_modules (can contain recursive .git dirs that cause 10s+ scans)
 			dot: true,
 			markDirectories: false,
 			suppressErrors: true,
@@ -165,6 +171,11 @@ export class GitOperations {
 				Logger.log(`CheckpointTracker ${disable ? "disabled" : "enabled"} nested git repo ${gitPath}`)
 			} catch (error) {
 				Logger.error(`CheckpointTracker failed to ${disable ? "disable" : "enable"} nested git repo ${gitPath}:`, error)
+				throw new Error(
+					`Failed to ${disable ? "disable" : "enable"} nested git repo ${gitPath}: ${
+						error instanceof Error ? error.message : String(error)
+					}`,
+				)
 			}
 		}
 	}
@@ -209,7 +220,18 @@ export class GitOperations {
 		} catch (_error) {
 			return { success: false }
 		} finally {
-			await this.renameNestedGitRepos(false)
+			await retryWithBackoff(() => this.renameNestedGitRepos(false), {
+				operationName: "CheckpointTracker re-enable nested git repos",
+				maxAttempts: 3,
+				baseDelayMs: 50,
+				onRetry: (_error, attempt, maxAttempts, delayMs) => {
+					Logger.warn(
+						`CheckpointTracker re-enable nested git repos failed on attempt ${attempt}/${maxAttempts}. Retrying in ${delayMs}ms`,
+					)
+				},
+			}).catch((error) => {
+				Logger.error("CheckpointTracker failed to re-enable nested git repos after retries:", error)
+			})
 		}
 	}
 }

src/utils/retry.test.ts

@@ -0,0 +1,112 @@
+import { describe, it } from "mocha"
+import sinon from "sinon"
+import "should"
+import { retryWithBackoff } from "./retry"
+
+describe("retryWithBackoff", () => {
+	it("returns immediately when operation succeeds on first attempt", async () => {
+		const operation = sinon.stub().resolves("ok")
+		const onRetry = sinon.stub()
+
+		const result = await retryWithBackoff<string>(operation, {
+			operationName: "Immediate success",
+			maxAttempts: 3,
+			baseDelayMs: 10,
+			onRetry,
+		})
+
+		result.should.equal("ok")
+		operation.callCount.should.equal(1)
+		onRetry.callCount.should.equal(0)
+	})
+
+	it("retries with exponential backoff until success", async () => {
+		const clock = sinon.useFakeTimers()
+		try {
+			let attempt = 0
+			const onRetry = sinon.stub()
+
+			const resultPromise = retryWithBackoff<string>(
+				async () => {
+					attempt++
+					if (attempt < 3) {
+						throw new Error(`fail ${attempt}`)
+					}
+					return "ok"
+				},
+				{
+					operationName: "Backoff retry",
+					maxAttempts: 4,
+					baseDelayMs: 100,
+					onRetry,
+				},
+			)
+
+			await Promise.resolve()
+			attempt.should.equal(1)
+
+			await clock.tickAsync(100)
+			attempt.should.equal(2)
+
+			await clock.tickAsync(200)
+			const result = await resultPromise
+			result.should.equal("ok")
+
+			attempt.should.equal(3)
+			onRetry.callCount.should.equal(2)
+			onRetry.getCall(0).args[1].should.equal(1)
+			onRetry.getCall(0).args[3].should.equal(100)
+			onRetry.getCall(1).args[1].should.equal(2)
+			onRetry.getCall(1).args[3].should.equal(200)
+		} finally {
+			clock.restore()
+		}
+	})
+
+	it("stops retrying when shouldRetry returns false", async () => {
+		const operation = sinon.stub().rejects(new Error("stop"))
+		const shouldRetry = sinon.stub().returns(false)
+
+		let errorMessage = ""
+		try {
+			await retryWithBackoff(operation, {
+				operationName: "Should retry gate",
+				maxAttempts: 5,
+				baseDelayMs: 10,
+				shouldRetry,
+			})
+		} catch (error) {
+			errorMessage = error instanceof Error ? error.message : String(error)
+		}
+
+		operation.callCount.should.equal(1)
+		shouldRetry.callCount.should.equal(1)
+		errorMessage.should.containEql("Should retry gate failed after 5 attempts")
+		errorMessage.should.containEql("stop")
+	})
+
+	it("throws after max attempts with operation name and last error", async () => {
+		let attempt = 0
+
+		let errorMessage = ""
+		try {
+			await retryWithBackoff(
+				async () => {
+					attempt++
+					throw new Error(`fail ${attempt}`)
+				},
+				{
+					operationName: "Always fails",
+					maxAttempts: 3,
+					baseDelayMs: 1,
+				},
+			)
+		} catch (error) {
+			errorMessage = error instanceof Error ? error.message : String(error)
+		}
+
+		attempt.should.equal(3)
+		errorMessage.should.containEql("Always fails failed after 3 attempts")
+		errorMessage.should.containEql("fail 3")
+	})
+})

src/utils/retry.ts

@@ -1,3 +1,69 @@
+export interface RetryWithBackoffOptions {
+	maxAttempts?: number
+	baseDelayMs?: number
+	maxDelayMs?: number
+	multiplier?: number
+	operationName?: string
+	shouldRetry?: (error: unknown, attempt: number) => boolean
+	onRetry?: (error: unknown, attempt: number, maxAttempts: number, delayMs: number) => void | Promise<void>
+}
+
+/**
+ * Retries an async operation with exponential backoff.
+ *
+ * Flow:
+ * 1. Try `operation()` immediately.
+ * 2. If it succeeds, return the result right away.
+ * 3. If it fails, decide whether to retry:
+ *    - stop if this was the last attempt
+ *    - stop if `shouldRetry(error, attempt)` returns false
+ * 4. If retrying, compute delay using exponential growth:
+ *    `baseDelayMs * multiplier^(attempt - 1)`, capped by `maxDelayMs`.
+ * 5. Call optional `onRetry(...)`, wait for the delay, and try again.
+ * 6. If all attempts fail, throw one final error with `operationName` and the last error message.
+ *
+ * Example timing with `maxAttempts=3`, `baseDelayMs=50`, `multiplier=2`:
+ * - Attempt 1 fails -> wait 50ms
+ * - Attempt 2 fails -> wait 100ms
+ * - Attempt 3 fails -> throw final error
+ * Total backoff wait before final failure: 150ms (plus operation runtime).
+ */
+export async function retryWithBackoff<T>(operation: () => Promise<T>, options: RetryWithBackoffOptions = {}): Promise<T> {
+	const {
+		maxAttempts = 3,
+		baseDelayMs = 50,
+		maxDelayMs = Number.POSITIVE_INFINITY,
+		multiplier = 2,
+		operationName = "Operation",
+		shouldRetry = () => true,
+		onRetry,
+	} = options
+
+	let lastError: unknown
+
+	for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+		try {
+			return await operation()
+		} catch (error) {
+			lastError = error
+			const isLastAttempt = attempt === maxAttempts
+			if (isLastAttempt || !shouldRetry(error, attempt)) {
+				break
+			}
+
+			const delayMs = Math.min(baseDelayMs * multiplier ** (attempt - 1), maxDelayMs)
+			await onRetry?.(error, attempt, maxAttempts, delayMs)
+			await new Promise((resolve) => setTimeout(resolve, delayMs))
+		}
+	}
+
+	throw new Error(
+		`${operationName} failed after ${maxAttempts} attempts: ${
+			lastError instanceof Error ? lastError.message : String(lastError)
+		}`,
+	)
+}
+
 /**
  * TypeScript equivalent of the Go common.RetryOperation utility
  * Performs an operation with retry logic and timeout handling