fix(core): Harden Daytona snapshot publishing with pruning and error recovery (no-changelog) (#36600)

.github/workflows/release-build-daytona-snapshot.yml

@@ -35,7 +35,18 @@ jobs:
   build-snapshot:
     name: Build versioned Daytona snapshot (${{ matrix.environment }})
     runs-on: ubuntu-latest
-    timeout-minutes: 30
+    # Serialize builds per version+environment: overlapping creates for the same
+    # snapshot name make Daytona reject with "an operation is already in progress"
+    # and leave the record in a failed state. Queue instead of cancel — a live
+    # build must finish, and the newer run then lands on the idempotent path.
+    concurrency:
+      group: daytona-snapshot-${{ inputs.n8n_version }}-${{ matrix.environment }}
+      cancel-in-progress: false
+    # Checkout + monorepo build (~10 min) run before the snapshot script, whose
+    # own budgets are 30 min create/verify + 5 min prune — keep headroom beyond
+    # that sum so the script's in-process timeouts always fire before the
+    # runner's hard kill.
+    timeout-minutes: 50
     # Dev is best-effort: a dev failure is reported but does not block the release.
     # Only the prod publish gates the release.
     continue-on-error: ${{ matrix.environment == 'dev' }}
@@ -60,4 +71,11 @@ jobs:
           DAYTONA_API_KEY: ${{ matrix.environment == 'dev' && secrets.DAYTONA_API_KEY_DEV || matrix.environment == 'prod' && secrets.DAYTONA_API_KEY_PROD || '' }}
           # Fall back to the shared DAYTONA_API_URL when an env-specific URL isn't set.
           DAYTONA_API_URL: ${{ matrix.environment == 'dev' && secrets.DAYTONA_API_URL_DEV || matrix.environment == 'prod' && secrets.DAYTONA_API_URL_PROD || secrets.DAYTONA_API_URL }}
+          # Prune versioned snapshots not used in 20 days (lastUsedAt-based, so
+          # versions still in use survive regardless of release cadence).
+          DAYTONA_SNAPSHOT_MAX_AGE_DAYS: '20'
+          # Hard count cap as a quota backstop (LRU eviction). Dev has a
+          # 30-snapshot org quota shared with ad-hoc testing, so keep it well
+          # below that.
+          DAYTONA_SNAPSHOT_RETENTION: ${{ matrix.environment == 'dev' && '15' || '100' }}
         run: node packages/@n8n/instance-ai/scripts/build-snapshot.cjs --version "$N8N_VERSION"

packages/@n8n/instance-ai/scripts/build-snapshot.cjs

@@ -21,6 +21,15 @@
  * Required env vars:
  *   DAYTONA_API_KEY   admin key with snapshot.create permissions
  *   DAYTONA_API_URL   Daytona API base URL (optional — SDK default used if absent)
+ *   DAYTONA_SNAPSHOT_MAX_AGE_DAYS
+ *                     prune versioned snapshots not used within this many days
+ *                     (lastUsedAt, falling back to createdAt). Runs after a
+ *                     successful publish and on quota-exceeded errors.
+ *                     0 disables age pruning. Default: 20.
+ *   DAYTONA_SNAPSHOT_RETENTION
+ *                     hard cap on versioned snapshots per org (quota backstop);
+ *                     least-recently-used ones are evicted beyond this count.
+ *                     0 disables the cap. Default: 10.
  *
  * Usage:
  *   node packages/@n8n/instance-ai/scripts/build-snapshot.cjs --version 1.123.0
@@ -38,6 +47,24 @@ function parseVersion(argv) {
 	return process.env.N8N_VERSION;
 }
 
+const DEFAULT_SNAPSHOT_RETENTION = 10;
+const DEFAULT_SNAPSHOT_MAX_AGE_DAYS = 20;
+
+/**
+ * Read a non-negative integer env var. These are tuning knobs — a malformed
+ * value warns and falls back to the default instead of failing the release.
+ */
+function readNonNegativeIntEnv(name, defaultValue) {
+	const rawValue = process.env[name];
+	if (rawValue === undefined || rawValue === '') return defaultValue;
+	const parsed = Number.parseInt(rawValue, 10);
+	if (!Number.isInteger(parsed) || parsed < 0 || String(parsed) !== rawValue.trim()) {
+		console.warn(`Invalid ${name} "${rawValue}" — using default ${defaultValue}`);
+		return defaultValue;
+	}
+	return parsed;
+}
+
 const consoleLogger = {
 	info: (msg, meta) => console.log(JSON.stringify({ level: 'info', msg, ...meta })),
 	warn: (msg, meta) => console.warn(JSON.stringify({ level: 'warn', msg, ...meta })),
@@ -59,6 +86,12 @@ async function main() {
 	}
 	const apiUrl = process.env.DAYTONA_API_URL || undefined;
 
+	const retention = readNonNegativeIntEnv('DAYTONA_SNAPSHOT_RETENTION', DEFAULT_SNAPSHOT_RETENTION);
+	const maxAgeDays = readNonNegativeIntEnv(
+		'DAYTONA_SNAPSHOT_MAX_AGE_DAYS',
+		DEFAULT_SNAPSHOT_MAX_AGE_DAYS,
+	);
+
 	const daytona = new Daytona({ apiKey, apiUrl });
 	const baseImage = process.env.SANDBOX_IMAGE || undefined;
 	const manager = new SnapshotManager(baseImage, consoleLogger, version);
@@ -66,14 +99,23 @@ async function main() {
 	const name = await manager.createSnapshot(daytona, {
 		timeout: 1800,
 		onLogs: (chunk) => process.stdout.write(`${chunk}\n`),
+		retention: retention > 0 ? retention : undefined,
+		maxAgeDays: maxAgeDays > 0 ? maxAgeDays : undefined,
 	});
 
 	consoleLogger.info('Snapshot ready', { name });
 }
 
-main().catch((error) => {
-	consoleLogger.error('Snapshot creation failed', {
-		error: error instanceof Error ? error.message : String(error),
-	});
-	process.exit(1);
-});
+main().then(
+	// Exit explicitly: a Daytona request abandoned by its deadline (e.g. a hung
+	// prune call) would otherwise keep the event loop alive until the CI runner
+	// kills the job despite a successful publish. The empty write drains any
+	// buffered stdout (e.g. streamed build logs) before the forced exit.
+	() => process.stdout.write('', () => process.exit(0)),
+	(error) => {
+		consoleLogger.error('Snapshot creation failed', {
+			error: error instanceof Error ? error.message : String(error),
+		});
+		process.exit(1);
+	},
+);

packages/@n8n/instance-ai/src/workspace/__tests__/snapshot-manager.test.ts

@@ -87,9 +87,27 @@ interface CreateSnapshotParams {
 	image: { dockerfile: string };
 }
 
+interface FakeSnapshot {
+	name: string;
+	state: string;
+	errorReason?: string;
+	createdAt?: string;
+	lastUsedAt?: string;
+}
+
+interface FakeSnapshotList {
+	items: FakeSnapshot[];
+	total: number;
+	page: number;
+	totalPages: number;
+}
+
 interface FakeSnapshotApi {
-	get: Mock<(...args: [string]) => Promise<{ name: string; state: string; errorReason?: string }>>;
+	get: Mock<(...args: [string]) => Promise<FakeSnapshot>>;
 	create: Mock<(...args: [CreateSnapshotParams, unknown?]) => Promise<{ name: string }>>;
+	list: Mock<(...args: [number?, number?]) => Promise<FakeSnapshotList>>;
+	delete: Mock<(...args: [FakeSnapshot]) => Promise<void>>;
+	activate: Mock<(...args: [FakeSnapshot]) => Promise<FakeSnapshot>>;
 }
 
 interface FakeDaytona {
@@ -136,13 +154,36 @@ function makeFakeDaytona(): FakeDaytona {
 	return {
 		snapshot: {
 			get: vi
-				.fn<(...args: [string]) => Promise<{ name: string; state: string; errorReason?: string }>>()
+				.fn<(...args: [string]) => Promise<FakeSnapshot>>()
 				.mockResolvedValue({ name: SNAPSHOT_NAME, state: 'active' }),
 			create: vi.fn<(...args: [CreateSnapshotParams, unknown?]) => Promise<{ name: string }>>(),
+			list: vi
+				.fn<(...args: [number?, number?]) => Promise<FakeSnapshotList>>()
+				.mockResolvedValue({ items: [], total: 0, page: 1, totalPages: 1 }),
+			delete: vi.fn<(...args: [FakeSnapshot]) => Promise<void>>().mockResolvedValue(undefined),
+			activate: vi
+				.fn<(...args: [FakeSnapshot]) => Promise<FakeSnapshot>>()
+				.mockImplementation(async (snapshot) => await Promise.resolve(snapshot)),
 		},
 	};
 }
 
+function snapshotPage(items: FakeSnapshot[], page = 1, totalPages = 1): FakeSnapshotList {
+	return { items, total: items.length, page, totalPages };
+}
+
+function daysAgo(days: number): string {
+	return new Date(Date.now() - days * 24 * 60 * 60 * 1000).toISOString();
+}
+
+/** After a prune, lookups of deleted snapshots must 404 for the removal wait. */
+function mockGetActiveOnlyFor(daytona: FakeDaytona, name: string): void {
+	daytona.snapshot.get.mockImplementation(async (requested) => {
+		if (requested !== name) throw new DaytonaNotFoundError(`Snapshot ${requested} not found`);
+		return await Promise.resolve({ name, state: 'active' });
+	});
+}
+
 describe('SnapshotManager.ensureImage', () => {
 	it('stages workspace files and builds a small COPY-based Daytona image descriptor', async () => {
 		const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0');
@@ -290,19 +331,170 @@ describe('SnapshotManager.createSnapshot', () => {
 		expect(result).toBe(SNAPSHOT_NAME);
 	});
 
-	it('throws when the created snapshot is in a failed state', async () => {
+	it('deletes an unusable snapshot, rebuilds once, and throws when the rebuild also fails', async () => {
 		const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0');
 		const daytona = makeFakeDaytona();
 		daytona.snapshot.create.mockResolvedValue({ name: SNAPSHOT_NAME });
-		daytona.snapshot.get.mockResolvedValue({
+		// Every build lands in a failed state; the record 404s while deleted.
+		let record: FakeSnapshot | undefined = {
 			name: SNAPSHOT_NAME,
 			state: 'build_failed',
 			errorReason: 'npm install exited 1',
+		};
+		daytona.snapshot.create.mockImplementation(async () => {
+			record = { name: SNAPSHOT_NAME, state: 'build_failed', errorReason: 'npm install exited 1' };
+			return await Promise.resolve({ name: SNAPSHOT_NAME });
+		});
+		daytona.snapshot.get.mockImplementation(async () => {
+			if (!record) throw new DaytonaNotFoundError('removed');
+			return await Promise.resolve(record);
+		});
+		daytona.snapshot.delete.mockImplementation(async () => {
+			record = undefined;
+			await Promise.resolve();
 		});
 
-		await expect(manager.createSnapshot(daytona as never)).rejects.toThrow(
-			`Versioned Daytona snapshot "${SNAPSHOT_NAME}" exists but is unusable (state: build_failed, reason: npm install exited 1)`,
-		);
+		await manager.ensureImage();
+		vi.useFakeTimers();
+		try {
+			const assertion = expect(manager.createSnapshot(daytona as never)).rejects.toThrow(
+				`Versioned Daytona snapshot "${SNAPSHOT_NAME}" exists but is unusable (state: build_failed, reason: npm install exited 1)`,
+			);
+			await vi.runAllTimersAsync();
+			await assertion;
+			expect(daytona.snapshot.delete).toHaveBeenCalledTimes(1);
+			expect(daytona.snapshot.create).toHaveBeenCalledTimes(2);
+		} finally {
+			vi.useRealTimers();
+		}
+	});
+
+	it('deletes the failed record left by a concurrent operation and retries the create', async () => {
+		// The 2.36.3 incident: the SDK's create poll saw the record land in `error`
+		// with "An operation is already in progress for this resource" and threw a
+		// synthesized DaytonaError without a statusCode. The failed record blocks
+		// every retry until deleted (previously a manual step in the Daytona UI).
+		const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0');
+		const daytona = makeFakeDaytona();
+		let record: FakeSnapshot | undefined;
+		daytona.snapshot.create
+			.mockImplementationOnce(async () => {
+				await Promise.resolve();
+				record = {
+					name: SNAPSHOT_NAME,
+					state: 'error',
+					errorReason: 'An operation is already in progress for this resource',
+				};
+				throw new DaytonaError(
+					`Failed to create snapshot. Name: ${SNAPSHOT_NAME} Reason: An operation is already in progress for this resource`,
+				);
+			})
+			.mockImplementationOnce(async () => {
+				record = { name: SNAPSHOT_NAME, state: 'active' };
+				return await Promise.resolve({ name: SNAPSHOT_NAME });
+			});
+		daytona.snapshot.get.mockImplementation(async () => {
+			if (!record) throw new DaytonaNotFoundError('removed');
+			return await Promise.resolve(record);
+		});
+		daytona.snapshot.delete.mockImplementation(async () => {
+			record = undefined;
+			await Promise.resolve();
+		});
+
+		await manager.ensureImage();
+		vi.useFakeTimers();
+		try {
+			const promise = manager.createSnapshot(daytona as never);
+			await vi.runAllTimersAsync();
+
+			await expect(promise).resolves.toBe(SNAPSHOT_NAME);
+			expect(daytona.snapshot.delete).toHaveBeenCalledTimes(1);
+			expect(daytona.snapshot.create).toHaveBeenCalledTimes(2);
+		} finally {
+			vi.useRealTimers();
+		}
+	});
+
+	it('recovers a re-run blocked by a leftover failed record', async () => {
+		const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0');
+		const daytona = makeFakeDaytona();
+		let record: FakeSnapshot | undefined = {
+			name: SNAPSHOT_NAME,
+			state: 'error',
+			errorReason: 'An operation is already in progress for this resource',
+		};
+		daytona.snapshot.create
+			.mockImplementationOnce(async () => {
+				await Promise.resolve();
+				throw new DaytonaError('already exists', 409);
+			})
+			.mockImplementationOnce(async () => {
+				record = { name: SNAPSHOT_NAME, state: 'active' };
+				return await Promise.resolve({ name: SNAPSHOT_NAME });
+			});
+		daytona.snapshot.get.mockImplementation(async () => {
+			if (!record) throw new DaytonaNotFoundError('removed');
+			return await Promise.resolve(record);
+		});
+		daytona.snapshot.delete.mockImplementation(async () => {
+			record = undefined;
+			await Promise.resolve();
+		});
+
+		await manager.ensureImage();
+		vi.useFakeTimers();
+		try {
+			const promise = manager.createSnapshot(daytona as never);
+			await vi.runAllTimersAsync();
+
+			await expect(promise).resolves.toBe(SNAPSHOT_NAME);
+			expect(daytona.snapshot.delete).toHaveBeenCalledTimes(1);
+			expect(daytona.snapshot.create).toHaveBeenCalledTimes(2);
+		} finally {
+			vi.useRealTimers();
+		}
+	});
+
+	it('bounds failed-record cleanups and surfaces the create failure', async () => {
+		const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0');
+		const daytona = makeFakeDaytona();
+		let record: FakeSnapshot | undefined;
+		// Every create attempt re-registers a record that lands in `error`.
+		daytona.snapshot.create.mockImplementation(async () => {
+			await Promise.resolve();
+			record = {
+				name: SNAPSHOT_NAME,
+				state: 'error',
+				errorReason: 'An operation is already in progress for this resource',
+			};
+			throw new DaytonaError(
+				`Failed to create snapshot. Name: ${SNAPSHOT_NAME} Reason: An operation is already in progress for this resource`,
+			);
+		});
+		daytona.snapshot.get.mockImplementation(async () => {
+			if (!record) throw new DaytonaNotFoundError('removed');
+			return await Promise.resolve(record);
+		});
+		daytona.snapshot.delete.mockImplementation(async () => {
+			record = undefined;
+			await Promise.resolve();
+		});
+
+		await manager.ensureImage();
+		vi.useFakeTimers();
+		try {
+			const assertion = expect(manager.createSnapshot(daytona as never)).rejects.toThrow(
+				/Failed to create snapshot/,
+			);
+			await vi.runAllTimersAsync();
+			await assertion;
+			// 1 initial attempt + 2 cleanup retries, then give up.
+			expect(daytona.snapshot.create).toHaveBeenCalledTimes(3);
+			expect(daytona.snapshot.delete).toHaveBeenCalledTimes(2);
+		} finally {
+			vi.useRealTimers();
+		}
 	});
 
 	it('waits for an existing snapshot that is still building', async () => {
@@ -327,12 +519,186 @@ describe('SnapshotManager.createSnapshot', () => {
 		}
 	});
 
-	it('throws on transient errors', async () => {
+	it('retries transient errors and throws after exhausting retries', async () => {
 		const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0');
 		const daytona = makeFakeDaytona();
 		daytona.snapshot.create.mockRejectedValue(new DaytonaError('upstream 500', 500));
 
-		await expect(manager.createSnapshot(daytona as never)).rejects.toThrow('upstream 500');
+		await manager.ensureImage();
+		vi.useFakeTimers();
+		try {
+			const assertion = expect(manager.createSnapshot(daytona as never)).rejects.toThrow(
+				'upstream 500',
+			);
+			await vi.runAllTimersAsync();
+			await assertion;
+			// 1 initial attempt + 3 transient retries
+			expect(daytona.snapshot.create).toHaveBeenCalledTimes(4);
+		} finally {
+			vi.useRealTimers();
+		}
+	});
+
+	it('recovers when a transient error is followed by already-exists on retry', async () => {
+		const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0');
+		const daytona = makeFakeDaytona();
+		daytona.snapshot.create
+			.mockRejectedValueOnce(new DaytonaError('<html>502 Bad Gateway</html>', 502))
+			.mockRejectedValueOnce(new DaytonaError('already exists', 409));
+
+		await manager.ensureImage();
+		vi.useFakeTimers();
+		try {
+			const promise = manager.createSnapshot(daytona as never);
+			await vi.runAllTimersAsync();
+
+			await expect(promise).resolves.toBe(SNAPSHOT_NAME);
+			expect(daytona.snapshot.create).toHaveBeenCalledTimes(2);
+		} finally {
+			vi.useRealTimers();
+		}
+	});
+
+	it('does not retry non-transient errors', async () => {
+		const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0');
+		const daytona = makeFakeDaytona();
+		daytona.snapshot.create.mockRejectedValue(new DaytonaError('invalid image', 400));
+
+		await expect(manager.createSnapshot(daytona as never)).rejects.toThrow('invalid image');
+		expect(daytona.snapshot.create).toHaveBeenCalledTimes(1);
+	});
+
+	it('reactivates an existing inactive snapshot instead of failing', async () => {
+		const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0');
+		const daytona = makeFakeDaytona();
+		daytona.snapshot.create.mockRejectedValue(new DaytonaError('already exists', 409));
+		daytona.snapshot.get
+			.mockResolvedValueOnce({ name: SNAPSHOT_NAME, state: 'inactive' })
+			.mockResolvedValue({ name: SNAPSHOT_NAME, state: 'active' });
+
+		await manager.ensureImage();
+		vi.useFakeTimers();
+		try {
+			const promise = manager.createSnapshot(daytona as never);
+			await vi.runAllTimersAsync();
+
+			await expect(promise).resolves.toBe(SNAPSHOT_NAME);
+			expect(daytona.snapshot.activate).toHaveBeenCalledTimes(1);
+			expect(daytona.snapshot.activate).toHaveBeenCalledWith(
+				expect.objectContaining({ name: SNAPSHOT_NAME, state: 'inactive' }),
+			);
+		} finally {
+			vi.useRealTimers();
+		}
+	});
+
+	it('retries activation on a transient error at the next settle window', async () => {
+		const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0');
+		const daytona = makeFakeDaytona();
+		daytona.snapshot.create.mockRejectedValue(new DaytonaError('already exists', 409));
+		let polls = 0;
+		daytona.snapshot.get.mockImplementation(
+			async () =>
+				// 8 inactive polls: attempt 1 fails on poll 1, the settle window
+				// elapses over polls 2-7, attempt 2 succeeds on poll 8.
+				await Promise.resolve({ name: SNAPSHOT_NAME, state: ++polls <= 8 ? 'inactive' : 'active' }),
+		);
+		daytona.snapshot.activate
+			.mockRejectedValueOnce(new DaytonaError('<html>502 Bad Gateway</html>', 502))
+			.mockImplementation(async (snapshot) => await Promise.resolve(snapshot));
+
+		await manager.ensureImage();
+		vi.useFakeTimers();
+		try {
+			const promise = manager.createSnapshot(daytona as never);
+			await vi.runAllTimersAsync();
+
+			await expect(promise).resolves.toBe(SNAPSHOT_NAME);
+			expect(daytona.snapshot.activate).toHaveBeenCalledTimes(2);
+		} finally {
+			vi.useRealTimers();
+		}
+	});
+
+	it('gives up after exhausting activation attempts on a stuck-inactive snapshot', async () => {
+		const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0');
+		const daytona = makeFakeDaytona();
+		daytona.snapshot.create.mockRejectedValue(new DaytonaError('already exists', 409));
+		daytona.snapshot.get.mockResolvedValue({ name: SNAPSHOT_NAME, state: 'inactive' });
+
+		await manager.ensureImage();
+		vi.useFakeTimers();
+		try {
+			const assertion = expect(manager.createSnapshot(daytona as never)).rejects.toThrow(
+				'remained inactive after 3 activation requests',
+			);
+			await vi.runAllTimersAsync();
+			await assertion;
+			expect(daytona.snapshot.activate).toHaveBeenCalledTimes(3);
+		} finally {
+			vi.useRealTimers();
+		}
+	});
+
+	it('waits after requesting activation and times out if the snapshot stays inactive', async () => {
+		const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0');
+		const daytona = makeFakeDaytona();
+		daytona.snapshot.create.mockRejectedValue(new DaytonaError('already exists', 409));
+		daytona.snapshot.get.mockResolvedValue({ name: SNAPSHOT_NAME, state: 'inactive' });
+
+		await manager.ensureImage();
+		vi.useFakeTimers();
+		try {
+			const assertion = expect(
+				manager.createSnapshot(daytona as never, { timeout: 1 }),
+			).rejects.toThrow('Timed out waiting');
+			await vi.runAllTimersAsync();
+			await assertion;
+			expect(daytona.snapshot.activate).toHaveBeenCalledTimes(1);
+		} finally {
+			vi.useRealTimers();
+		}
+	});
+
+	it('bounds a hung status poll with the overall deadline', async () => {
+		const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0');
+		const daytona = makeFakeDaytona();
+		daytona.snapshot.create.mockResolvedValue({ name: SNAPSHOT_NAME });
+		// A request that never settles (stalled transport, no error).
+		daytona.snapshot.get.mockImplementation(async () => await new Promise<never>(() => {}));
+
+		await manager.ensureImage();
+		vi.useFakeTimers();
+		try {
+			const assertion = expect(
+				manager.createSnapshot(daytona as never, { timeout: 60 }),
+			).rejects.toThrow('Timed out fetching state');
+			await vi.runAllTimersAsync();
+			await assertion;
+		} finally {
+			vi.useRealTimers();
+		}
+	});
+
+	it('tolerates a transient error while polling snapshot state', async () => {
+		const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0');
+		const daytona = makeFakeDaytona();
+		daytona.snapshot.create.mockResolvedValue({ name: SNAPSHOT_NAME });
+		daytona.snapshot.get
+			.mockRejectedValueOnce(new DaytonaError('<html>502 Bad Gateway</html>', 502))
+			.mockResolvedValue({ name: SNAPSHOT_NAME, state: 'active' });
+
+		await manager.ensureImage();
+		vi.useFakeTimers();
+		try {
+			const promise = manager.createSnapshot(daytona as never);
+			await vi.runAllTimersAsync();
+
+			await expect(promise).resolves.toBe(SNAPSHOT_NAME);
+			expect(daytona.snapshot.get).toHaveBeenCalledTimes(2);
+		} finally {
+			vi.useRealTimers();
+		}
 	});
 
 	it('throws when no version is configured', async () => {
@@ -357,6 +723,340 @@ describe('SnapshotManager.createSnapshot', () => {
 	});
 });
 
+describe('SnapshotManager snapshot pruning', () => {
+	it('prunes on quota exhaustion and retries the create once', async () => {
+		const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0');
+		const daytona = makeFakeDaytona();
+		daytona.snapshot.create
+			.mockRejectedValueOnce(new DaytonaError('Snapshot quota exceeded. Maximum allowed: 30'))
+			.mockResolvedValue({ name: SNAPSHOT_NAME });
+		daytona.snapshot.list.mockResolvedValue(
+			snapshotPage([
+				{ name: 'n8n/instance-ai:1.122.0', state: 'active' },
+				{ name: 'n8n/instance-ai:1.121.0', state: 'active' },
+				{ name: 'n8n/instance-ai:1.120.0', state: 'active' },
+				{ name: 'n8n/instance-ai:1.119.0', state: 'active' },
+				{ name: 'n8n/instance-ai:1.118.0', state: 'inactive' },
+			]),
+		);
+		mockGetActiveOnlyFor(daytona, SNAPSHOT_NAME);
+
+		const result = await manager.createSnapshot(daytona as never, { retention: 2 });
+
+		expect(result).toBe(SNAPSHOT_NAME);
+		expect(daytona.snapshot.create).toHaveBeenCalledTimes(2);
+		// The count backstop evicts beyond the newest-3 floor.
+		const deletedNames = daytona.snapshot.delete.mock.calls.map(([snapshot]) => snapshot.name);
+		expect(deletedNames).toContain('n8n/instance-ai:1.119.0');
+		expect(deletedNames).toContain('n8n/instance-ai:1.118.0');
+	});
+
+	it('waits for pruned snapshots to finish removing before retrying after a quota error', async () => {
+		const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0');
+		const daytona = makeFakeDaytona();
+		daytona.snapshot.create
+			.mockRejectedValueOnce(new DaytonaError('Snapshot quota exceeded. Maximum allowed: 30'))
+			.mockResolvedValue({ name: SNAPSHOT_NAME });
+		daytona.snapshot.list.mockResolvedValue(
+			snapshotPage([
+				{ name: 'n8n/instance-ai:1.122.0', state: 'active' },
+				{ name: 'n8n/instance-ai:1.121.0', state: 'active' },
+				{ name: 'n8n/instance-ai:1.120.0', state: 'active' },
+				{ name: 'n8n/instance-ai:1.119.0', state: 'active' },
+			]),
+		);
+		// The deleted snapshot lingers in `removing` before disappearing.
+		daytona.snapshot.get.mockImplementation(async (requested) => {
+			if (requested === SNAPSHOT_NAME)
+				return await Promise.resolve({ name: SNAPSHOT_NAME, state: 'active' });
+			if (daytona.snapshot.get.mock.calls.filter(([n]) => n === requested).length <= 1)
+				return await Promise.resolve({ name: requested, state: 'removing' });
+			throw new DaytonaNotFoundError(`Snapshot ${requested} not found`);
+		});
+
+		await manager.ensureImage();
+		vi.useFakeTimers();
+		try {
+			const promise = manager.createSnapshot(daytona as never, { retention: 3 });
+			await vi.runAllTimersAsync();
+
+			await expect(promise).resolves.toBe(SNAPSHOT_NAME);
+			// The retry happened only after the pruned snapshot was gone.
+			expect(daytona.snapshot.create).toHaveBeenCalledTimes(2);
+			const removalPolls = daytona.snapshot.get.mock.calls.filter(
+				([requested]) => requested === 'n8n/instance-ai:1.119.0',
+			);
+			expect(removalPolls.length).toBeGreaterThanOrEqual(2);
+		} finally {
+			vi.useRealTimers();
+		}
+	});
+
+	it('throws the quota error when no retention is configured', async () => {
+		const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0');
+		const daytona = makeFakeDaytona();
+		daytona.snapshot.create.mockRejectedValue(
+			new DaytonaError('Snapshot quota exceeded. Maximum allowed: 30'),
+		);
+
+		await expect(manager.createSnapshot(daytona as never)).rejects.toThrow('quota exceeded');
+		expect(daytona.snapshot.list).not.toHaveBeenCalled();
+		expect(daytona.snapshot.create).toHaveBeenCalledTimes(1);
+	});
+
+	it('treats an explicit retention of 0 as pruning disabled on the quota path', async () => {
+		const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0');
+		const daytona = makeFakeDaytona();
+		daytona.snapshot.create.mockRejectedValue(
+			new DaytonaError('Snapshot quota exceeded. Maximum allowed: 30'),
+		);
+
+		await expect(manager.createSnapshot(daytona as never, { retention: 0 })).rejects.toThrow(
+			'quota exceeded',
+		);
+		expect(daytona.snapshot.list).not.toHaveBeenCalled();
+	});
+
+	it('force-evicts the least-recently-used snapshot when quota is held below the retention window', async () => {
+		// Foreign snapshots can exhaust the org quota while our own count is
+		// within policy — the publish still needs one slot freed.
+		const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0');
+		const daytona = makeFakeDaytona();
+		daytona.snapshot.create
+			.mockRejectedValueOnce(new DaytonaError('Snapshot quota exceeded. Maximum allowed: 30'))
+			.mockResolvedValue({ name: SNAPSHOT_NAME });
+		daytona.snapshot.list.mockResolvedValue(
+			snapshotPage([
+				{ name: 'n8n/instance-ai:1.122.0', state: 'active', lastUsedAt: daysAgo(1) },
+				{ name: 'n8n/instance-ai:1.121.0', state: 'active', lastUsedAt: daysAgo(2) },
+				{ name: 'n8n/instance-ai:1.120.0', state: 'active', lastUsedAt: daysAgo(3) },
+				{ name: 'n8n/instance-ai:1.119.0', state: 'active', lastUsedAt: daysAgo(10) },
+			]),
+		);
+		mockGetActiveOnlyFor(daytona, SNAPSHOT_NAME);
+
+		const result = await manager.createSnapshot(daytona as never, { retention: 15 });
+
+		expect(result).toBe(SNAPSHOT_NAME);
+		expect(daytona.snapshot.create).toHaveBeenCalledTimes(2);
+		const deletedNames = daytona.snapshot.delete.mock.calls.map(([snapshot]) => snapshot.name);
+		expect(deletedNames).toEqual(['n8n/instance-ai:1.119.0']);
+	});
+
+	it('throws the quota error when pruning frees nothing', async () => {
+		const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0');
+		const daytona = makeFakeDaytona();
+		daytona.snapshot.create.mockRejectedValue(
+			new DaytonaError('Snapshot quota exceeded. Maximum allowed: 30'),
+		);
+		daytona.snapshot.list.mockResolvedValue(
+			snapshotPage([{ name: 'n8n/instance-ai:1.122.0', state: 'active' }]),
+		);
+
+		await expect(manager.createSnapshot(daytona as never, { retention: 5 })).rejects.toThrow(
+			'quota exceeded',
+		);
+		expect(daytona.snapshot.create).toHaveBeenCalledTimes(1);
+		expect(daytona.snapshot.delete).not.toHaveBeenCalled();
+	});
+
+	it('enforces the count cap after a successful publish, sparing the newest versions and in-progress builds', async () => {
+		const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0');
+		const daytona = makeFakeDaytona();
+		daytona.snapshot.create.mockResolvedValue({ name: SNAPSHOT_NAME });
+		daytona.snapshot.list.mockResolvedValue(
+			snapshotPage([
+				{ name: 'n8n/instance-ai:1.122.0', state: 'active' },
+				{ name: SNAPSHOT_NAME, state: 'active' },
+				{ name: 'n8n/instance-ai:1.121.0', state: 'inactive' },
+				{ name: 'n8n/instance-ai:1.121.0-abc123', state: 'active' },
+				{ name: 'n8n/instance-ai:1.120.0', state: 'building' },
+				{ name: 'someone-elses/snapshot:1.0.0', state: 'active' },
+			]),
+		);
+
+		await manager.createSnapshot(daytona as never, { retention: 2 });
+
+		// The newest 3 versions (1.123.0, 1.122.0, 1.121.0 — suffixed
+		// 1.121.0-abc123 ranks below plain 1.121.0) are floor-protected; the
+		// building and foreign snapshots are untouched.
+		const deletedNames = daytona.snapshot.delete.mock.calls.map(([snapshot]) => snapshot.name);
+		expect(deletedNames).toEqual(['n8n/instance-ai:1.121.0-abc123']);
+	});
+
+	it('age-prunes snapshots unused beyond maxAgeDays, keeping recently used and floor-protected ones', async () => {
+		const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0');
+		const daytona = makeFakeDaytona();
+		daytona.snapshot.create.mockResolvedValue({ name: SNAPSHOT_NAME });
+		daytona.snapshot.list.mockResolvedValue(
+			snapshotPage([
+				{ name: SNAPSHOT_NAME, state: 'active', createdAt: daysAgo(0), lastUsedAt: daysAgo(0) },
+				// Aged but within the newest-3 floor → kept.
+				{
+					name: 'n8n/instance-ai:1.122.0',
+					state: 'active',
+					createdAt: daysAgo(40),
+					lastUsedAt: daysAgo(25),
+				},
+				// Created long ago but recently used → kept.
+				{
+					name: 'n8n/instance-ai:1.121.0',
+					state: 'active',
+					createdAt: daysAgo(30),
+					lastUsedAt: daysAgo(2),
+				},
+				// Idle past the cutoff → pruned.
+				{
+					name: 'n8n/instance-ai:1.120.0',
+					state: 'inactive',
+					createdAt: daysAgo(40),
+					lastUsedAt: daysAgo(25),
+				},
+				// No lastUsedAt → createdAt fallback → pruned.
+				{ name: 'n8n/instance-ai:1.119.0', state: 'active', createdAt: daysAgo(25) },
+			]),
+		);
+
+		await manager.createSnapshot(daytona as never, { maxAgeDays: 20 });
+
+		const deletedNames = daytona.snapshot.delete.mock.calls.map(([snapshot]) => snapshot.name);
+		expect(deletedNames).toEqual(['n8n/instance-ai:1.120.0', 'n8n/instance-ai:1.119.0']);
+	});
+
+	it('does not let failed or suffixed snapshots consume rollback-floor slots', async () => {
+		const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0');
+		const daytona = makeFakeDaytona();
+		daytona.snapshot.create.mockResolvedValue({ name: SNAPSHOT_NAME });
+		daytona.snapshot.list.mockResolvedValue(
+			snapshotPage([
+				{ name: SNAPSHOT_NAME, state: 'active', lastUsedAt: daysAgo(0) },
+				// Failed build: deleted, and must not occupy a floor slot.
+				{ name: 'n8n/instance-ai:1.122.0', state: 'build_failed', lastUsedAt: daysAgo(1) },
+				// Suffixed build: not a rollback target; aged out → pruned.
+				{ name: 'n8n/instance-ai:1.121.0-pr1', state: 'active', lastUsedAt: daysAgo(25) },
+				// Idle plain releases: floor-protected because the failed and
+				// suffixed snapshots above don't count toward the newest-3 floor.
+				{ name: 'n8n/instance-ai:1.120.0', state: 'active', lastUsedAt: daysAgo(25) },
+				{ name: 'n8n/instance-ai:1.119.0', state: 'active', lastUsedAt: daysAgo(25) },
+			]),
+		);
+
+		await manager.createSnapshot(daytona as never, { maxAgeDays: 20 });
+
+		const deletedNames = daytona.snapshot.delete.mock.calls.map(([snapshot]) => snapshot.name);
+		expect(deletedNames).toEqual(['n8n/instance-ai:1.122.0', 'n8n/instance-ai:1.121.0-pr1']);
+	});
+
+	it('count-cap eviction is LRU: an old version still in use outlives an idle newer one', async () => {
+		const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0');
+		const daytona = makeFakeDaytona();
+		daytona.snapshot.create.mockResolvedValue({ name: SNAPSHOT_NAME });
+		daytona.snapshot.list.mockResolvedValue(
+			snapshotPage([
+				{ name: SNAPSHOT_NAME, state: 'active', lastUsedAt: daysAgo(0) },
+				{ name: 'n8n/instance-ai:1.122.0', state: 'active', lastUsedAt: daysAgo(1) },
+				{ name: 'n8n/instance-ai:1.121.0', state: 'active', lastUsedAt: daysAgo(1) },
+				// Oldest version but used yesterday (a pinned instance) → kept.
+				{ name: 'n8n/instance-ai:1.100.0', state: 'active', lastUsedAt: daysAgo(1) },
+				// Idle for two weeks → evicted first.
+				{ name: 'n8n/instance-ai:1.119.0', state: 'active', lastUsedAt: daysAgo(15) },
+				{ name: 'n8n/instance-ai:1.118.0', state: 'active', lastUsedAt: daysAgo(10) },
+			]),
+		);
+
+		await manager.createSnapshot(daytona as never, { retention: 5 });
+
+		const deletedNames = daytona.snapshot.delete.mock.calls.map(([snapshot]) => snapshot.name);
+		expect(deletedNames).toEqual(['n8n/instance-ai:1.119.0']);
+	});
+
+	it('deletes failed snapshots even within the retention window', async () => {
+		const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0');
+		const daytona = makeFakeDaytona();
+		daytona.snapshot.create.mockResolvedValue({ name: SNAPSHOT_NAME });
+		daytona.snapshot.list.mockResolvedValue(
+			snapshotPage([
+				{ name: SNAPSHOT_NAME, state: 'active' },
+				{ name: 'n8n/instance-ai:1.122.0', state: 'build_failed', errorReason: 'npm exit 1' },
+				{ name: 'n8n/instance-ai:1.121.0', state: 'active' },
+			]),
+		);
+
+		await manager.createSnapshot(daytona as never, { retention: 3 });
+
+		const deletedNames = daytona.snapshot.delete.mock.calls.map(([snapshot]) => snapshot.name);
+		expect(deletedNames).toEqual(['n8n/instance-ai:1.122.0']);
+	});
+
+	it('never deletes the snapshot being published', async () => {
+		// Republish of an old version that ranks outside the retention window.
+		const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0');
+		const daytona = makeFakeDaytona();
+		daytona.snapshot.create.mockResolvedValue({ name: SNAPSHOT_NAME });
+		daytona.snapshot.list.mockResolvedValue(
+			snapshotPage([
+				{ name: 'n8n/instance-ai:2.0.0', state: 'active' },
+				{ name: SNAPSHOT_NAME, state: 'active' },
+			]),
+		);
+
+		await manager.createSnapshot(daytona as never, { retention: 1 });
+
+		expect(daytona.snapshot.delete).not.toHaveBeenCalled();
+	});
+
+	it('paginates through the snapshot list', async () => {
+		const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0');
+		const daytona = makeFakeDaytona();
+		daytona.snapshot.create.mockResolvedValue({ name: SNAPSHOT_NAME });
+		daytona.snapshot.list
+			.mockResolvedValueOnce(snapshotPage([{ name: SNAPSHOT_NAME, state: 'active' }], 1, 2))
+			.mockResolvedValueOnce(
+				snapshotPage([{ name: 'n8n/instance-ai:1.100.0', state: 'build_failed' }], 2, 2),
+			);
+
+		await manager.createSnapshot(daytona as never, { retention: 1 });
+
+		expect(daytona.snapshot.list).toHaveBeenCalledTimes(2);
+		const deletedNames = daytona.snapshot.delete.mock.calls.map(([snapshot]) => snapshot.name);
+		expect(deletedNames).toEqual(['n8n/instance-ai:1.100.0']);
+	});
+
+	it('does not fail the publish when pruning fails', async () => {
+		const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0');
+		const daytona = makeFakeDaytona();
+		daytona.snapshot.create.mockResolvedValue({ name: SNAPSHOT_NAME });
+		daytona.snapshot.list.mockRejectedValue(new DaytonaError('boom', 500));
+
+		await expect(manager.createSnapshot(daytona as never, { retention: 2 })).resolves.toBe(
+			SNAPSHOT_NAME,
+		);
+	});
+
+	it('does not fail the publish when a single delete fails', async () => {
+		const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0');
+		const daytona = makeFakeDaytona();
+		daytona.snapshot.create.mockResolvedValue({ name: SNAPSHOT_NAME });
+		daytona.snapshot.list.mockResolvedValue(
+			snapshotPage([
+				{ name: SNAPSHOT_NAME, state: 'active' },
+				{ name: 'n8n/instance-ai:1.122.0', state: 'active' },
+				{ name: 'n8n/instance-ai:1.121.0', state: 'active' },
+				{ name: 'n8n/instance-ai:1.120.0', state: 'active' },
+				{ name: 'n8n/instance-ai:1.119.0', state: 'active' },
+			]),
+		);
+		daytona.snapshot.delete
+			.mockRejectedValueOnce(new DaytonaError('delete failed', 500))
+			.mockResolvedValue(undefined);
+
+		await expect(manager.createSnapshot(daytona as never, { retention: 1 })).resolves.toBe(
+			SNAPSHOT_NAME,
+		);
+		expect(daytona.snapshot.delete).toHaveBeenCalledTimes(2);
+	});
+});
+
 describe('SnapshotManager.snapshotName', () => {
 	it('returns null when no version is configured', () => {
 		const manager = new SnapshotManager(undefined, NOOP_LOGGER, undefined);

packages/@n8n/instance-ai/src/workspace/snapshot-manager.ts

@@ -34,13 +34,56 @@ import { loadInstanceAiRuntimeSkillSource } from '../skills/runtime-skills';
 export interface CreateSnapshotOptions {
 	timeout?: number;
 	onLogs?: (chunk: string) => void;
+	/**
+	 * Hard cap on versioned snapshots per organization (quota backstop). When
+	 * the count exceeds this after age pruning, the least-recently-used
+	 * snapshots are deleted until the cap is met. Unset disables the cap.
+	 */
+	retention?: number;
+	/**
+	 * Delete versioned snapshots not used within this many days
+	 * (`lastUsedAt`, falling back to `createdAt`). Snapshots still in use by
+	 * older n8n versions keep a fresh `lastUsedAt` and survive regardless of
+	 * release cadence. Unset disables age pruning.
+	 */
+	maxAgeDays?: number;
 }
 
+type DaytonaSnapshot = Awaited<ReturnType<Daytona['snapshot']['get']>>;
+
 const DAYTONA_WORKSPACE_BAKE_ROOT = '/tmp/n8n-workspace-bake';
 const SNAPSHOT_WORKSPACE_LAYOUT_DIRS = ['src', 'chunks', 'node-types'] as const;
 const SNAPSHOT_BUILDING_STATES = new Set(['building', 'pending', 'pulling']);
 const SNAPSHOT_VERIFY_POLL_MS = 5_000;
 const DEFAULT_SNAPSHOT_VERIFY_TIMEOUT_S = 1_800;
+const SNAPSHOT_NAME_PREFIX = 'n8n/instance-ai:';
+const MAX_TRANSIENT_CREATE_RETRIES = 3;
+const TRANSIENT_CREATE_RETRY_BACKOFF_MS = 5_000;
+/**
+ * Times a failed (`error`/`build_failed`) record for the target version is deleted and
+ * the create retried before giving up. Automates the manual "delete the broken snapshot
+ * in the Daytona UI and re-run" remediation.
+ */
+const MAX_FAILED_SNAPSHOT_CLEANUPS = 2;
+const SNAPSHOT_LIST_PAGE_SIZE = 100;
+const MAX_SNAPSHOT_LIST_PAGES = 20;
+// Bound for the post-publish prune: it runs after the snapshot is verified, so
+// a hung Daytona call must not stall a release (the SDK has no sane default
+// HTTP timeout).
+const SNAPSHOT_PRUNE_TIMEOUT_MS = 5 * 60_000;
+// Deleted snapshots pass through `removing` before their quota slot frees up.
+const SNAPSHOT_REMOVAL_WAIT_MS = 60_000;
+const SNAPSHOT_REMOVAL_POLL_MS = 2_000;
+const MAX_ACTIVATION_ATTEMPTS = 3;
+// Polls (5s apart) to let an activation request settle before re-requesting.
+const ACTIVATION_SETTLE_POLLS = 6;
+// States a snapshot can be safely deleted from. Never delete in-progress
+// builds (a concurrent release job may own them) or already-removing ones.
+const SNAPSHOT_DELETABLE_STATES = new Set(['active', 'inactive', 'error', 'build_failed']);
+const SNAPSHOT_FAILED_STATES = new Set(['error', 'build_failed']);
+// Rollback insurance: the newest versions are never age- or count-pruned even
+// when idle (e.g. a quiet dev org where nothing was used for weeks).
+const MIN_KEEP_NEWEST_VERSIONS = 3;
 
 function isAlreadyExistsError(error: unknown): error is TDaytonaError {
 	const { DaytonaError } = loadDaytona();
@@ -49,6 +92,91 @@ function isAlreadyExistsError(error: unknown): error is TDaytonaError {
 	return /already exists/i.test(error.message);
 }
 
+/**
+ * The SDK's `snapshot.create` polls the new record and synthesizes this error (no
+ * statusCode) when the record lands in `error`/`build_failed` — e.g. "Reason: An
+ * operation is already in progress for this resource" when concurrent publishes race.
+ * The failed record persists under the version's name and blocks every retry until
+ * it is deleted.
+ */
+function isCreateFailedStateError(error: unknown): boolean {
+	const { DaytonaError } = loadDaytona();
+	return error instanceof DaytonaError && /^failed to create snapshot\b/i.test(error.message);
+}
+
+/**
+ * Internal signal from {@link SnapshotManager.verifySnapshot}: the record exists in a
+ * state that can never become active. Carries the state so the publish loop can decide
+ * whether deleting and rebuilding is worthwhile (failed states only).
+ */
+class SnapshotUnusableError extends Error {
+	constructor(
+		message: string,
+		readonly snapshotState: string,
+	) {
+		super(message);
+	}
+}
+
+/**
+ * Gateway/availability errors worth retrying: 5xx/408/429 responses, plus the
+ * SDK's connection/timeout errors (matched by name — they carry no statusCode).
+ */
+function isTransientDaytonaError(error: unknown): boolean {
+	const { DaytonaError } = loadDaytona();
+	if (!(error instanceof DaytonaError)) return false;
+	if (error.name === 'DaytonaConnectionError' || error.name === 'DaytonaTimeoutError') return true;
+	const status = error.statusCode;
+	return status !== undefined && (status >= 500 || status === 408 || status === 429);
+}
+
+// The SDK has no dedicated error class or status mapping for quota rejections;
+// the message (e.g. "Snapshot quota exceeded. Maximum allowed: 30") is the only
+// signal. Deliberately narrow: a rewording degrades to the pre-existing
+// hard-fail rather than triggering pruning on unrelated quota-flavored errors.
+function isQuotaExceededError(error: unknown): boolean {
+	const { DaytonaError } = loadDaytona();
+	return error instanceof DaytonaError && /snapshot quota exceeded/i.test(error.message);
+}
+
+/**
+ * Order snapshots newest-version-first by the version in their name.
+ * A suffixed version (`2.23.0-<hash>`) sorts older than its plain release;
+ * names whose version segment is unparseable sort oldest.
+ */
+function compareSnapshotVersionsDesc(a: DaytonaSnapshot, b: DaytonaSnapshot): number {
+	return parseSnapshotVersionRank(b.name) - parseSnapshotVersionRank(a.name);
+}
+
+function parseSnapshotVersionRank(name: string): number {
+	const version = name.slice(SNAPSHOT_NAME_PREFIX.length);
+	const match = /^(\d+)\.(\d+)\.(\d+)(-.+)?$/.exec(version);
+	if (!match) return Number.NEGATIVE_INFINITY;
+	const [, major, minor, patch, suffix] = match;
+	// Scale leaves room for four-digit minor/patch; -0.5 ranks suffixed builds
+	// below their plain release.
+	return (
+		Number(major) * 1e8 + Number(minor) * 1e4 + Number(patch) + (suffix !== undefined ? -0.5 : 0)
+	);
+}
+
+// Plain release versions only — suffixed builds (`2.23.0-<hash>`) and failed
+// snapshots must not consume rollback-floor slots.
+function isPlainVersionName(name: string): boolean {
+	return /^\d+\.\d+\.\d+$/.test(name.slice(SNAPSHOT_NAME_PREFIX.length));
+}
+
+/**
+ * When a snapshot was last used, for LRU pruning. `lastUsedAt` is bumped by
+ * sandbox creation (verified against real org data); `createdAt` covers
+ * never-used snapshots. Unparseable timestamps count as just-used so bad data
+ * never causes a deletion.
+ */
+function lastUsedTime(snapshot: DaytonaSnapshot): number {
+	const time = new Date(snapshot.lastUsedAt ?? snapshot.createdAt).getTime();
+	return Number.isNaN(time) ? Number.POSITIVE_INFINITY : time;
+}
+
 export class SnapshotManager {
 	private cachedImage: Promise<Image> | null = null;
 
@@ -118,8 +246,9 @@ export class SnapshotManager {
 	/**
 	 * Create the versioned Daytona snapshot for the configured n8n version.
 	 * Treats 409 / "already exists" as success — re-runs against the same
-	 * version are idempotent. Throws on transient or unexpected errors so
-	 * callers can decide whether to retry, fall back, or fail loudly.
+	 * version are idempotent. Retries transient gateway errors, prunes old
+	 * versioned snapshots on quota exhaustion (when `retention` is set), and
+	 * reactivates an existing-but-inactive snapshot instead of failing.
 	 *
 	 * Single source of truth for snapshot creation in the CI release pipeline
 	 * (`scripts/build-snapshot.cjs`). Runtime never calls this.
@@ -130,45 +259,314 @@ export class SnapshotManager {
 			throw new Error('SnapshotManager: n8nVersion is required to derive a snapshot name');
 		}
 
-		try {
-			await daytona.snapshot.create({ name, image: await this.ensureImage() }, options);
-			this.logger.info('Created versioned Daytona snapshot; verifying it is usable', { name });
-		} catch (error) {
-			if (isAlreadyExistsError(error)) {
-				this.logger.info('Versioned Daytona snapshot already exists; verifying it is usable', {
+		// The SDK treats `timeout: 0` as "no timeout"; here it means the default
+		// total budget rather than an instantly-expired deadline.
+		const timeoutS =
+			options?.timeout !== undefined && options.timeout > 0
+				? options.timeout
+				: DEFAULT_SNAPSHOT_VERIFY_TIMEOUT_S;
+		const deadline = Date.now() + timeoutS * 1000;
+
+		// One rebuild when the published record turns out unusable: a build that
+		// finished in a failed state (e.g. broken by a concurrent operation) blocks
+		// this version until its record is deleted, so delete it and publish again.
+		for (let rebuilds = 0; ; rebuilds++) {
+			await this.createWithRecovery(daytona, name, deadline, options);
+			try {
+				await this.verifySnapshot(daytona, name, deadline);
+				break;
+			} catch (error) {
+				const rebuildable =
+					error instanceof SnapshotUnusableError &&
+					SNAPSHOT_FAILED_STATES.has(error.snapshotState) &&
+					rebuilds < 1 &&
+					Date.now() < deadline;
+				if (!rebuildable) throw error;
+				this.logger.warn('Published Daytona snapshot is unusable; deleting it and rebuilding', {
 					name,
+					state: error.snapshotState,
 				});
-			} else {
+				await this.reconcileFailedSnapshotRecord(daytona, name, deadline);
+			}
+		}
+		try {
+			// Best-effort and time-boxed: neither a prune failure nor a hung
+			// Daytona call may fail a release that just published a healthy
+			// snapshot.
+			await this.withDeadline(
+				this.pruneSnapshots(daytona, name, options),
+				Date.now() + SNAPSHOT_PRUNE_TIMEOUT_MS,
+				'Timed out pruning Daytona snapshots',
+			);
+		} catch (error) {
+			this.logger.warn('Snapshot pruning did not complete', {
+				error: error instanceof Error ? error.message : String(error),
+			});
+		}
+		return name;
+	}
+
+	/**
+	 * Run `snapshot.create` with recovery for the two failure modes CI has hit:
+	 * quota exhaustion (prune old versions once, then retry) and transient
+	 * gateway errors (bounded retries; a re-create after the first request
+	 * registered lands on the idempotent 409 path).
+	 *
+	 * The SDK's `timeout` option only bounds the initial POST — its status
+	 * polling loop is unbounded — so each attempt is raced against `deadline`.
+	 */
+	private async createWithRecovery(
+		daytona: Daytona,
+		name: string,
+		deadline: number,
+		options?: CreateSnapshotOptions,
+	): Promise<void> {
+		const createOptions = { timeout: options?.timeout, onLogs: options?.onLogs };
+		let transientRetries = 0;
+		let failedCleanups = 0;
+		let prunedForQuota = false;
+		for (;;) {
+			try {
+				const image = await this.withDeadline(
+					this.ensureImage(),
+					deadline,
+					`Timed out preparing the image for Daytona snapshot "${name}"`,
+				);
+				await this.withDeadline(
+					daytona.snapshot.create({ name, image }, createOptions),
+					deadline,
+					`Timed out creating Daytona snapshot "${name}"`,
+				);
+				this.logger.info('Created versioned Daytona snapshot; verifying it is usable', { name });
+				return;
+			} catch (error) {
+				if (isCreateFailedStateError(error)) {
+					// The build landed in `error`/`build_failed` (e.g. broken by a concurrent
+					// operation on the same name). The failed record blocks every retry of
+					// this version, so delete it and retry instead of requiring a manual
+					// deletion in the Daytona UI.
+					if (failedCleanups < MAX_FAILED_SNAPSHOT_CLEANUPS) {
+						const record = await this.reconcileFailedSnapshotRecord(daytona, name, deadline);
+						if (record !== 'usable') {
+							failedCleanups++;
+							await sleep(
+								Math.min(TRANSIENT_CREATE_RETRY_BACKOFF_MS * failedCleanups, deadline - Date.now()),
+							);
+							if (Date.now() >= deadline) throw error;
+							continue;
+						}
+					}
+					throw error;
+				}
+				if (isAlreadyExistsError(error)) {
+					// A pre-existing failed record is handled by the verify + rebuild loop.
+					this.logger.info('Versioned Daytona snapshot already exists; verifying it is usable', {
+						name,
+					});
+					return;
+				}
+				const canPrune = (options?.retention ?? 0) > 0 || (options?.maxAgeDays ?? 0) > 0;
+				if (isQuotaExceededError(error) && !prunedForQuota && canPrune) {
+					prunedForQuota = true;
+					this.logger.warn('Snapshot quota exceeded; pruning old versioned snapshots', { name });
+					// Quota may be held below the retention window (e.g. by foreign
+					// snapshots in the org), so force at least one LRU eviction.
+					const deleted = await this.withDeadline(
+						this.pruneSnapshots(daytona, name, options, { ensureAtLeastOne: true }),
+						deadline,
+						'Timed out pruning Daytona snapshots',
+					).catch((): DaytonaSnapshot[] => []);
+					if (deleted.length === 0) throw error;
+					// Deletion is asynchronous server-side (snapshots pass through
+					// `removing`); the quota slot only frees once they are gone.
+					await this.waitForSnapshotRemoval(daytona, deleted, deadline);
+					continue;
+				}
+				if (
+					isTransientDaytonaError(error) &&
+					transientRetries < MAX_TRANSIENT_CREATE_RETRIES &&
+					Date.now() < deadline
+				) {
+					transientRetries++;
+					this.logger.warn('Transient Daytona error during snapshot create; retrying', {
+						name,
+						attempt: transientRetries,
+						error: error instanceof Error ? error.message : String(error),
+					});
+					await sleep(
+						Math.min(TRANSIENT_CREATE_RETRY_BACKOFF_MS * transientRetries, deadline - Date.now()),
+					);
+					// The backoff may have consumed the remaining budget; surface the
+					// real error instead of a confusing deadline timeout.
+					if (Date.now() >= deadline) throw error;
+					continue;
+				}
 				throw error;
 			}
 		}
+	}
 
-		await this.verifySnapshot(daytona, name, options?.timeout);
-		return name;
+	/**
+	 * Look up the record for `name` and, when it sits in a failed state, delete it and
+	 * wait for the removal — a failed record permanently blocks republishing the version.
+	 * Returns 'cleaned' when a failed record was deleted, 'absent' when no record exists
+	 * (create can be retried directly), and 'usable' when the record is in any live state.
+	 */
+	private async reconcileFailedSnapshotRecord(
+		daytona: Daytona,
+		name: string,
+		deadline: number,
+	): Promise<'cleaned' | 'absent' | 'usable'> {
+		const { DaytonaNotFoundError } = loadDaytona();
+		let snapshot: DaytonaSnapshot;
+		try {
+			snapshot = await this.withDeadline(
+				daytona.snapshot.get(name),
+				deadline,
+				`Timed out fetching state of Daytona snapshot "${name}"`,
+			);
+		} catch (error) {
+			if (error instanceof DaytonaNotFoundError) return 'absent';
+			throw error;
+		}
+		if (!SNAPSHOT_FAILED_STATES.has(snapshot.state)) return 'usable';
+		this.logger.warn('Versioned Daytona snapshot is in a failed state; deleting it to retry', {
+			name,
+			state: snapshot.state,
+			...(snapshot.errorReason ? { reason: snapshot.errorReason } : {}),
+		});
+		await this.withDeadline(
+			daytona.snapshot.delete(snapshot),
+			deadline,
+			`Timed out deleting failed Daytona snapshot "${name}"`,
+		);
+		await this.waitForSnapshotRemoval(daytona, [snapshot], deadline);
+		return 'cleaned';
+	}
+
+	/**
+	 * Wait until pruned snapshots are actually gone (a 404 on lookup). Bounded
+	 * and best-effort: on timeout or an unexpected lookup result the create is
+	 * retried anyway and surfaces whatever is still wrong.
+	 */
+	private async waitForSnapshotRemoval(
+		daytona: Daytona,
+		snapshots: DaytonaSnapshot[],
+		deadline: number,
+	): Promise<void> {
+		const { DaytonaNotFoundError } = loadDaytona();
+		const waitDeadline = Math.min(deadline, Date.now() + SNAPSHOT_REMOVAL_WAIT_MS);
+		for (const snapshot of snapshots) {
+			for (;;) {
+				try {
+					// Race the lookup itself against the wait budget — the SDK's transport
+					// timeout is effectively unbounded, so a stalled request would
+					// otherwise hang past the deadline.
+					await this.withDeadline(
+						daytona.snapshot.get(snapshot.name),
+						waitDeadline,
+						`Timed out waiting for Daytona snapshot "${snapshot.name}" to be removed`,
+					);
+				} catch (error) {
+					if (!(error instanceof DaytonaNotFoundError)) {
+						this.logger.warn('Unexpected error while waiting for snapshot removal', {
+							name: snapshot.name,
+							error: error instanceof Error ? error.message : String(error),
+						});
+					}
+					break;
+				}
+				if (Date.now() >= waitDeadline) {
+					this.logger.warn('Timed out waiting for pruned snapshots to be removed', {
+						name: snapshot.name,
+					});
+					return;
+				}
+				await sleep(SNAPSHOT_REMOVAL_POLL_MS);
+			}
+		}
 	}
 
 	/**
 	 * Verify that the snapshot is actually usable. A snapshot build can finish in
 	 * `error`/`build_failed` state; returning before it is active would let a release
-	 * ship without a working snapshot. Waits out in-progress builds, throws on any
-	 * unusable state.
+	 * ship without a working snapshot. Waits out in-progress builds, reactivates an
+	 * `inactive` snapshot (Daytona deactivates idle ones), tolerates transient poll
+	 * errors, and throws on any unusable state.
 	 */
-	private async verifySnapshot(
-		daytona: Daytona,
-		name: string,
-		timeoutS = DEFAULT_SNAPSHOT_VERIFY_TIMEOUT_S,
-	): Promise<void> {
-		const deadline = Date.now() + timeoutS * 1000;
+	private async verifySnapshot(daytona: Daytona, name: string, deadline: number): Promise<void> {
+		let activationAttempts = 0;
+		// Start at the threshold so the first `inactive` poll requests activation
+		// immediately.
+		let pollsSinceActivation = ACTIVATION_SETTLE_POLLS;
 		for (;;) {
-			const snapshot = await daytona.snapshot.get(name);
+			let snapshot: DaytonaSnapshot;
+			try {
+				// Race every request against the deadline — the SDK's transport
+				// timeout is effectively unbounded, so a stalled request would
+				// otherwise hang the job past the budget.
+				snapshot = await this.withDeadline(
+					daytona.snapshot.get(name),
+					deadline,
+					`Timed out fetching state of Daytona snapshot "${name}"`,
+				);
+			} catch (error) {
+				if (!isTransientDaytonaError(error) || Date.now() >= deadline) throw error;
+				this.logger.warn('Transient Daytona error while polling snapshot state; retrying', {
+					name,
+					error: error instanceof Error ? error.message : String(error),
+				});
+				await sleep(SNAPSHOT_VERIFY_POLL_MS);
+				continue;
+			}
 			if (snapshot.state === 'active') {
 				this.logger.info('Versioned Daytona snapshot is active', { name });
 				return;
 			}
+			if (snapshot.state === 'inactive') {
+				// Re-request activation every settle window; transient failures
+				// retry on the next window instead of failing the release.
+				if (pollsSinceActivation >= ACTIVATION_SETTLE_POLLS) {
+					if (activationAttempts >= MAX_ACTIVATION_ATTEMPTS) {
+						throw new Error(
+							`Versioned Daytona snapshot "${name}" remained inactive after ${MAX_ACTIVATION_ATTEMPTS} activation requests`,
+						);
+					}
+					activationAttempts++;
+					pollsSinceActivation = 0;
+					this.logger.info('Versioned Daytona snapshot is inactive; requesting activation', {
+						name,
+						attempt: activationAttempts,
+					});
+					try {
+						await this.withDeadline(
+							daytona.snapshot.activate(snapshot),
+							deadline,
+							`Timed out requesting activation of Daytona snapshot "${name}"`,
+						);
+					} catch (error) {
+						if (!isTransientDaytonaError(error)) throw error;
+						this.logger.warn('Transient Daytona error during snapshot activation; will retry', {
+							name,
+							error: error instanceof Error ? error.message : String(error),
+						});
+					}
+				} else {
+					pollsSinceActivation++;
+				}
+				if (Date.now() >= deadline) {
+					throw new Error(
+						`Timed out waiting for existing Daytona snapshot "${name}" to become active (state: ${snapshot.state})`,
+					);
+				}
+				await sleep(SNAPSHOT_VERIFY_POLL_MS);
+				continue;
+			}
 			if (!SNAPSHOT_BUILDING_STATES.has(snapshot.state)) {
 				const reason = snapshot.errorReason ? `, reason: ${snapshot.errorReason}` : '';
-				throw new Error(
+				throw new SnapshotUnusableError(
 					`Versioned Daytona snapshot "${name}" exists but is unusable (state: ${snapshot.state}${reason})`,
+					snapshot.state,
 				);
 			}
 			if (Date.now() >= deadline) {
@@ -176,7 +574,7 @@ export class SnapshotManager {
 					`Timed out waiting for existing Daytona snapshot "${name}" to become active (state: ${snapshot.state})`,
 				);
 			}
-			this.logger.info('Waiting for existing Daytona snapshot to finish building', {
+			this.logger.info('Waiting for existing Daytona snapshot to become active', {
 				name,
 				state: snapshot.state,
 			});
@@ -184,6 +582,158 @@ export class SnapshotManager {
 		}
 	}
 
+	/**
+	 * Prune versioned snapshots. Age-based pruning (`maxAgeDays`, keyed on
+	 * `lastUsedAt`) is the primary policy; the `retention` count is a quota
+	 * backstop that evicts least-recently-used snapshots when the total still
+	 * exceeds it. Failed snapshots are always deleted. Only touches
+	 * `n8n/instance-ai:*` names, never the snapshot being published, never
+	 * in-progress states, and never the newest `MIN_KEEP_NEWEST_VERSIONS`
+	 * versions. Never throws: returns the snapshots actually deleted.
+	 */
+	private async pruneSnapshots(
+		daytona: Daytona,
+		protectedName: string,
+		options?: Pick<CreateSnapshotOptions, 'retention' | 'maxAgeDays'>,
+		{ ensureAtLeastOne = false }: { ensureAtLeastOne?: boolean } = {},
+	): Promise<DaytonaSnapshot[]> {
+		const retention = options?.retention;
+		const maxAgeDays = options?.maxAgeDays;
+		if (!retention && !maxAgeDays) return [];
+
+		let matching: DaytonaSnapshot[];
+		try {
+			matching = await this.listVersionedSnapshots(daytona);
+		} catch (error) {
+			this.logger.warn('Failed to list Daytona snapshots for pruning; skipping', {
+				error: error instanceof Error ? error.message : String(error),
+			});
+			return [];
+		}
+
+		// Floor slots must go to usable release versions — failed and suffixed
+		// snapshots don't count as rollback targets.
+		const newestVersionNames = new Set(
+			matching
+				.filter(
+					(snapshot) =>
+						!SNAPSHOT_FAILED_STATES.has(snapshot.state) && isPlainVersionName(snapshot.name),
+				)
+				.sort(compareSnapshotVersionsDesc)
+				.slice(0, MIN_KEEP_NEWEST_VERSIONS)
+				.map((snapshot) => snapshot.name),
+		);
+		const isDeletable = (snapshot: DaytonaSnapshot) =>
+			snapshot.name !== protectedName && SNAPSHOT_DELETABLE_STATES.has(snapshot.state);
+		const isPrunable = (snapshot: DaytonaSnapshot) =>
+			isDeletable(snapshot) && !newestVersionNames.has(snapshot.name);
+
+		const toDelete: DaytonaSnapshot[] = [];
+		const selected = new Set<string>();
+		const select = (snapshot: DaytonaSnapshot) => {
+			if (selected.has(snapshot.name)) return;
+			selected.add(snapshot.name);
+			toDelete.push(snapshot);
+		};
+
+		// Failed snapshots are quota dead weight; the newest-versions floor
+		// does not protect them (they are unusable anyway).
+		for (const snapshot of matching) {
+			if (SNAPSHOT_FAILED_STATES.has(snapshot.state) && isDeletable(snapshot)) select(snapshot);
+		}
+
+		if (maxAgeDays) {
+			const cutoff = Date.now() - maxAgeDays * 24 * 60 * 60 * 1000;
+			for (const snapshot of matching) {
+				if (!selected.has(snapshot.name) && isPrunable(snapshot) && lastUsedTime(snapshot) < cutoff)
+					select(snapshot);
+			}
+		}
+
+		if (retention) {
+			const remaining = matching.filter((snapshot) => !selected.has(snapshot.name));
+			let excess = remaining.length - retention;
+			const evictable = remaining
+				.filter(isPrunable)
+				.sort((a, b) => lastUsedTime(a) - lastUsedTime(b));
+			for (const snapshot of evictable) {
+				if (excess <= 0) break;
+				select(snapshot);
+				excess--;
+			}
+		}
+
+		// Under quota pressure the publish needs a free slot even when our own
+		// snapshots are within policy (e.g. foreign snapshots hold the quota):
+		// evict the least-recently-used prunable one.
+		if (ensureAtLeastOne && toDelete.length === 0) {
+			const lruCandidate = matching
+				.filter(isPrunable)
+				.sort((a, b) => lastUsedTime(a) - lastUsedTime(b))[0];
+			if (lruCandidate) {
+				this.logger.warn(
+					'Quota pressure: evicting least-recently-used snapshot despite retention policy',
+					{ name: lruCandidate.name },
+				);
+				select(lruCandidate);
+			}
+		}
+
+		const deleted: DaytonaSnapshot[] = [];
+		for (const snapshot of toDelete) {
+			try {
+				await daytona.snapshot.delete(snapshot);
+				deleted.push(snapshot);
+				this.logger.info('Pruned versioned Daytona snapshot', {
+					name: snapshot.name,
+					state: snapshot.state,
+					lastUsedAt: snapshot.lastUsedAt ?? null,
+				});
+			} catch (error) {
+				this.logger.warn('Failed to delete Daytona snapshot during pruning', {
+					name: snapshot.name,
+					error: error instanceof Error ? error.message : String(error),
+				});
+			}
+		}
+		return deleted;
+	}
+
+	private async listVersionedSnapshots(daytona: Daytona): Promise<DaytonaSnapshot[]> {
+		const matching: DaytonaSnapshot[] = [];
+		let page = 1;
+		for (let fetched = 0; fetched < MAX_SNAPSHOT_LIST_PAGES; fetched++) {
+			const result = await daytona.snapshot.list(page, SNAPSHOT_LIST_PAGE_SIZE);
+			matching.push(...result.items.filter((item) => item.name.startsWith(SNAPSHOT_NAME_PREFIX)));
+			if (result.items.length === 0 || result.page >= result.totalPages) break;
+			page = result.page + 1;
+		}
+		return matching;
+	}
+
+	private async withDeadline<T>(
+		promise: Promise<T>,
+		deadline: number,
+		timeoutMessage: string,
+	): Promise<T> {
+		let timer: NodeJS.Timeout | undefined;
+		try {
+			return await Promise.race([
+				promise,
+				new Promise<never>((_, reject) => {
+					timer = setTimeout(
+						() => reject(new Error(timeoutMessage)),
+						Math.max(deadline - Date.now(), 0),
+					);
+				}),
+			]);
+		} finally {
+			clearTimeout(timer);
+			// The losing promise keeps running; swallow its eventual rejection.
+			promise.catch(() => {});
+		}
+	}
+
 	/**
 	 * Derive the versioned snapshot name for the running n8n version, or null
 	 * when no version is configured. Existence is validated implicitly by the