diff --git a/.github/workflows/release-build-daytona-snapshot.yml b/.github/workflows/release-build-daytona-snapshot.yml index 4737d2d5df7..8bed6a2d7b2 100644 --- a/.github/workflows/release-build-daytona-snapshot.yml +++ b/.github/workflows/release-build-daytona-snapshot.yml @@ -35,7 +35,18 @@ jobs: build-snapshot: name: Build versioned Daytona snapshot (${{ matrix.environment }}) runs-on: ubuntu-latest - timeout-minutes: 30 + # Serialize builds per version+environment: overlapping creates for the same + # snapshot name make Daytona reject with "an operation is already in progress" + # and leave the record in a failed state. Queue instead of cancel — a live + # build must finish, and the newer run then lands on the idempotent path. + concurrency: + group: daytona-snapshot-${{ inputs.n8n_version }}-${{ matrix.environment }} + cancel-in-progress: false + # Checkout + monorepo build (~10 min) run before the snapshot script, whose + # own budgets are 30 min create/verify + 5 min prune — keep headroom beyond + # that sum so the script's in-process timeouts always fire before the + # runner's hard kill. + timeout-minutes: 50 # Dev is best-effort: a dev failure is reported but does not block the release. # Only the prod publish gates the release. continue-on-error: ${{ matrix.environment == 'dev' }} @@ -60,4 +71,11 @@ jobs: DAYTONA_API_KEY: ${{ matrix.environment == 'dev' && secrets.DAYTONA_API_KEY_DEV || matrix.environment == 'prod' && secrets.DAYTONA_API_KEY_PROD || '' }} # Fall back to the shared DAYTONA_API_URL when an env-specific URL isn't set. DAYTONA_API_URL: ${{ matrix.environment == 'dev' && secrets.DAYTONA_API_URL_DEV || matrix.environment == 'prod' && secrets.DAYTONA_API_URL_PROD || secrets.DAYTONA_API_URL }} + # Prune versioned snapshots not used in 20 days (lastUsedAt-based, so + # versions still in use survive regardless of release cadence). + DAYTONA_SNAPSHOT_MAX_AGE_DAYS: '20' + # Hard count cap as a quota backstop (LRU eviction). Dev has a + # 30-snapshot org quota shared with ad-hoc testing, so keep it well + # below that. + DAYTONA_SNAPSHOT_RETENTION: ${{ matrix.environment == 'dev' && '15' || '100' }} run: node packages/@n8n/instance-ai/scripts/build-snapshot.cjs --version "$N8N_VERSION" diff --git a/packages/@n8n/instance-ai/scripts/build-snapshot.cjs b/packages/@n8n/instance-ai/scripts/build-snapshot.cjs index 8511d5941cd..f327b1eb528 100755 --- a/packages/@n8n/instance-ai/scripts/build-snapshot.cjs +++ b/packages/@n8n/instance-ai/scripts/build-snapshot.cjs @@ -21,6 +21,15 @@ * Required env vars: * DAYTONA_API_KEY admin key with snapshot.create permissions * DAYTONA_API_URL Daytona API base URL (optional — SDK default used if absent) + * DAYTONA_SNAPSHOT_MAX_AGE_DAYS + * prune versioned snapshots not used within this many days + * (lastUsedAt, falling back to createdAt). Runs after a + * successful publish and on quota-exceeded errors. + * 0 disables age pruning. Default: 20. + * DAYTONA_SNAPSHOT_RETENTION + * hard cap on versioned snapshots per org (quota backstop); + * least-recently-used ones are evicted beyond this count. + * 0 disables the cap. Default: 10. * * Usage: * node packages/@n8n/instance-ai/scripts/build-snapshot.cjs --version 1.123.0 @@ -38,6 +47,24 @@ function parseVersion(argv) { return process.env.N8N_VERSION; } +const DEFAULT_SNAPSHOT_RETENTION = 10; +const DEFAULT_SNAPSHOT_MAX_AGE_DAYS = 20; + +/** + * Read a non-negative integer env var. These are tuning knobs — a malformed + * value warns and falls back to the default instead of failing the release. + */ +function readNonNegativeIntEnv(name, defaultValue) { + const rawValue = process.env[name]; + if (rawValue === undefined || rawValue === '') return defaultValue; + const parsed = Number.parseInt(rawValue, 10); + if (!Number.isInteger(parsed) || parsed < 0 || String(parsed) !== rawValue.trim()) { + console.warn(`Invalid ${name} "${rawValue}" — using default ${defaultValue}`); + return defaultValue; + } + return parsed; +} + const consoleLogger = { info: (msg, meta) => console.log(JSON.stringify({ level: 'info', msg, ...meta })), warn: (msg, meta) => console.warn(JSON.stringify({ level: 'warn', msg, ...meta })), @@ -59,6 +86,12 @@ async function main() { } const apiUrl = process.env.DAYTONA_API_URL || undefined; + const retention = readNonNegativeIntEnv('DAYTONA_SNAPSHOT_RETENTION', DEFAULT_SNAPSHOT_RETENTION); + const maxAgeDays = readNonNegativeIntEnv( + 'DAYTONA_SNAPSHOT_MAX_AGE_DAYS', + DEFAULT_SNAPSHOT_MAX_AGE_DAYS, + ); + const daytona = new Daytona({ apiKey, apiUrl }); const baseImage = process.env.SANDBOX_IMAGE || undefined; const manager = new SnapshotManager(baseImage, consoleLogger, version); @@ -66,14 +99,23 @@ async function main() { const name = await manager.createSnapshot(daytona, { timeout: 1800, onLogs: (chunk) => process.stdout.write(`${chunk}\n`), + retention: retention > 0 ? retention : undefined, + maxAgeDays: maxAgeDays > 0 ? maxAgeDays : undefined, }); consoleLogger.info('Snapshot ready', { name }); } -main().catch((error) => { - consoleLogger.error('Snapshot creation failed', { - error: error instanceof Error ? error.message : String(error), - }); - process.exit(1); -}); +main().then( + // Exit explicitly: a Daytona request abandoned by its deadline (e.g. a hung + // prune call) would otherwise keep the event loop alive until the CI runner + // kills the job despite a successful publish. The empty write drains any + // buffered stdout (e.g. streamed build logs) before the forced exit. + () => process.stdout.write('', () => process.exit(0)), + (error) => { + consoleLogger.error('Snapshot creation failed', { + error: error instanceof Error ? error.message : String(error), + }); + process.exit(1); + }, +); diff --git a/packages/@n8n/instance-ai/src/workspace/__tests__/snapshot-manager.test.ts b/packages/@n8n/instance-ai/src/workspace/__tests__/snapshot-manager.test.ts index 52833ea3c43..68d5cab2dd9 100644 --- a/packages/@n8n/instance-ai/src/workspace/__tests__/snapshot-manager.test.ts +++ b/packages/@n8n/instance-ai/src/workspace/__tests__/snapshot-manager.test.ts @@ -87,9 +87,27 @@ interface CreateSnapshotParams { image: { dockerfile: string }; } +interface FakeSnapshot { + name: string; + state: string; + errorReason?: string; + createdAt?: string; + lastUsedAt?: string; +} + +interface FakeSnapshotList { + items: FakeSnapshot[]; + total: number; + page: number; + totalPages: number; +} + interface FakeSnapshotApi { - get: Mock<(...args: [string]) => Promise<{ name: string; state: string; errorReason?: string }>>; + get: Mock<(...args: [string]) => Promise>; create: Mock<(...args: [CreateSnapshotParams, unknown?]) => Promise<{ name: string }>>; + list: Mock<(...args: [number?, number?]) => Promise>; + delete: Mock<(...args: [FakeSnapshot]) => Promise>; + activate: Mock<(...args: [FakeSnapshot]) => Promise>; } interface FakeDaytona { @@ -136,13 +154,36 @@ function makeFakeDaytona(): FakeDaytona { return { snapshot: { get: vi - .fn<(...args: [string]) => Promise<{ name: string; state: string; errorReason?: string }>>() + .fn<(...args: [string]) => Promise>() .mockResolvedValue({ name: SNAPSHOT_NAME, state: 'active' }), create: vi.fn<(...args: [CreateSnapshotParams, unknown?]) => Promise<{ name: string }>>(), + list: vi + .fn<(...args: [number?, number?]) => Promise>() + .mockResolvedValue({ items: [], total: 0, page: 1, totalPages: 1 }), + delete: vi.fn<(...args: [FakeSnapshot]) => Promise>().mockResolvedValue(undefined), + activate: vi + .fn<(...args: [FakeSnapshot]) => Promise>() + .mockImplementation(async (snapshot) => await Promise.resolve(snapshot)), }, }; } +function snapshotPage(items: FakeSnapshot[], page = 1, totalPages = 1): FakeSnapshotList { + return { items, total: items.length, page, totalPages }; +} + +function daysAgo(days: number): string { + return new Date(Date.now() - days * 24 * 60 * 60 * 1000).toISOString(); +} + +/** After a prune, lookups of deleted snapshots must 404 for the removal wait. */ +function mockGetActiveOnlyFor(daytona: FakeDaytona, name: string): void { + daytona.snapshot.get.mockImplementation(async (requested) => { + if (requested !== name) throw new DaytonaNotFoundError(`Snapshot ${requested} not found`); + return await Promise.resolve({ name, state: 'active' }); + }); +} + describe('SnapshotManager.ensureImage', () => { it('stages workspace files and builds a small COPY-based Daytona image descriptor', async () => { const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0'); @@ -290,19 +331,170 @@ describe('SnapshotManager.createSnapshot', () => { expect(result).toBe(SNAPSHOT_NAME); }); - it('throws when the created snapshot is in a failed state', async () => { + it('deletes an unusable snapshot, rebuilds once, and throws when the rebuild also fails', async () => { const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0'); const daytona = makeFakeDaytona(); daytona.snapshot.create.mockResolvedValue({ name: SNAPSHOT_NAME }); - daytona.snapshot.get.mockResolvedValue({ + // Every build lands in a failed state; the record 404s while deleted. + let record: FakeSnapshot | undefined = { name: SNAPSHOT_NAME, state: 'build_failed', errorReason: 'npm install exited 1', + }; + daytona.snapshot.create.mockImplementation(async () => { + record = { name: SNAPSHOT_NAME, state: 'build_failed', errorReason: 'npm install exited 1' }; + return await Promise.resolve({ name: SNAPSHOT_NAME }); + }); + daytona.snapshot.get.mockImplementation(async () => { + if (!record) throw new DaytonaNotFoundError('removed'); + return await Promise.resolve(record); + }); + daytona.snapshot.delete.mockImplementation(async () => { + record = undefined; + await Promise.resolve(); }); - await expect(manager.createSnapshot(daytona as never)).rejects.toThrow( - `Versioned Daytona snapshot "${SNAPSHOT_NAME}" exists but is unusable (state: build_failed, reason: npm install exited 1)`, - ); + await manager.ensureImage(); + vi.useFakeTimers(); + try { + const assertion = expect(manager.createSnapshot(daytona as never)).rejects.toThrow( + `Versioned Daytona snapshot "${SNAPSHOT_NAME}" exists but is unusable (state: build_failed, reason: npm install exited 1)`, + ); + await vi.runAllTimersAsync(); + await assertion; + expect(daytona.snapshot.delete).toHaveBeenCalledTimes(1); + expect(daytona.snapshot.create).toHaveBeenCalledTimes(2); + } finally { + vi.useRealTimers(); + } + }); + + it('deletes the failed record left by a concurrent operation and retries the create', async () => { + // The 2.36.3 incident: the SDK's create poll saw the record land in `error` + // with "An operation is already in progress for this resource" and threw a + // synthesized DaytonaError without a statusCode. The failed record blocks + // every retry until deleted (previously a manual step in the Daytona UI). + const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0'); + const daytona = makeFakeDaytona(); + let record: FakeSnapshot | undefined; + daytona.snapshot.create + .mockImplementationOnce(async () => { + await Promise.resolve(); + record = { + name: SNAPSHOT_NAME, + state: 'error', + errorReason: 'An operation is already in progress for this resource', + }; + throw new DaytonaError( + `Failed to create snapshot. Name: ${SNAPSHOT_NAME} Reason: An operation is already in progress for this resource`, + ); + }) + .mockImplementationOnce(async () => { + record = { name: SNAPSHOT_NAME, state: 'active' }; + return await Promise.resolve({ name: SNAPSHOT_NAME }); + }); + daytona.snapshot.get.mockImplementation(async () => { + if (!record) throw new DaytonaNotFoundError('removed'); + return await Promise.resolve(record); + }); + daytona.snapshot.delete.mockImplementation(async () => { + record = undefined; + await Promise.resolve(); + }); + + await manager.ensureImage(); + vi.useFakeTimers(); + try { + const promise = manager.createSnapshot(daytona as never); + await vi.runAllTimersAsync(); + + await expect(promise).resolves.toBe(SNAPSHOT_NAME); + expect(daytona.snapshot.delete).toHaveBeenCalledTimes(1); + expect(daytona.snapshot.create).toHaveBeenCalledTimes(2); + } finally { + vi.useRealTimers(); + } + }); + + it('recovers a re-run blocked by a leftover failed record', async () => { + const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0'); + const daytona = makeFakeDaytona(); + let record: FakeSnapshot | undefined = { + name: SNAPSHOT_NAME, + state: 'error', + errorReason: 'An operation is already in progress for this resource', + }; + daytona.snapshot.create + .mockImplementationOnce(async () => { + await Promise.resolve(); + throw new DaytonaError('already exists', 409); + }) + .mockImplementationOnce(async () => { + record = { name: SNAPSHOT_NAME, state: 'active' }; + return await Promise.resolve({ name: SNAPSHOT_NAME }); + }); + daytona.snapshot.get.mockImplementation(async () => { + if (!record) throw new DaytonaNotFoundError('removed'); + return await Promise.resolve(record); + }); + daytona.snapshot.delete.mockImplementation(async () => { + record = undefined; + await Promise.resolve(); + }); + + await manager.ensureImage(); + vi.useFakeTimers(); + try { + const promise = manager.createSnapshot(daytona as never); + await vi.runAllTimersAsync(); + + await expect(promise).resolves.toBe(SNAPSHOT_NAME); + expect(daytona.snapshot.delete).toHaveBeenCalledTimes(1); + expect(daytona.snapshot.create).toHaveBeenCalledTimes(2); + } finally { + vi.useRealTimers(); + } + }); + + it('bounds failed-record cleanups and surfaces the create failure', async () => { + const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0'); + const daytona = makeFakeDaytona(); + let record: FakeSnapshot | undefined; + // Every create attempt re-registers a record that lands in `error`. + daytona.snapshot.create.mockImplementation(async () => { + await Promise.resolve(); + record = { + name: SNAPSHOT_NAME, + state: 'error', + errorReason: 'An operation is already in progress for this resource', + }; + throw new DaytonaError( + `Failed to create snapshot. Name: ${SNAPSHOT_NAME} Reason: An operation is already in progress for this resource`, + ); + }); + daytona.snapshot.get.mockImplementation(async () => { + if (!record) throw new DaytonaNotFoundError('removed'); + return await Promise.resolve(record); + }); + daytona.snapshot.delete.mockImplementation(async () => { + record = undefined; + await Promise.resolve(); + }); + + await manager.ensureImage(); + vi.useFakeTimers(); + try { + const assertion = expect(manager.createSnapshot(daytona as never)).rejects.toThrow( + /Failed to create snapshot/, + ); + await vi.runAllTimersAsync(); + await assertion; + // 1 initial attempt + 2 cleanup retries, then give up. + expect(daytona.snapshot.create).toHaveBeenCalledTimes(3); + expect(daytona.snapshot.delete).toHaveBeenCalledTimes(2); + } finally { + vi.useRealTimers(); + } }); it('waits for an existing snapshot that is still building', async () => { @@ -327,12 +519,186 @@ describe('SnapshotManager.createSnapshot', () => { } }); - it('throws on transient errors', async () => { + it('retries transient errors and throws after exhausting retries', async () => { const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0'); const daytona = makeFakeDaytona(); daytona.snapshot.create.mockRejectedValue(new DaytonaError('upstream 500', 500)); - await expect(manager.createSnapshot(daytona as never)).rejects.toThrow('upstream 500'); + await manager.ensureImage(); + vi.useFakeTimers(); + try { + const assertion = expect(manager.createSnapshot(daytona as never)).rejects.toThrow( + 'upstream 500', + ); + await vi.runAllTimersAsync(); + await assertion; + // 1 initial attempt + 3 transient retries + expect(daytona.snapshot.create).toHaveBeenCalledTimes(4); + } finally { + vi.useRealTimers(); + } + }); + + it('recovers when a transient error is followed by already-exists on retry', async () => { + const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0'); + const daytona = makeFakeDaytona(); + daytona.snapshot.create + .mockRejectedValueOnce(new DaytonaError('502 Bad Gateway', 502)) + .mockRejectedValueOnce(new DaytonaError('already exists', 409)); + + await manager.ensureImage(); + vi.useFakeTimers(); + try { + const promise = manager.createSnapshot(daytona as never); + await vi.runAllTimersAsync(); + + await expect(promise).resolves.toBe(SNAPSHOT_NAME); + expect(daytona.snapshot.create).toHaveBeenCalledTimes(2); + } finally { + vi.useRealTimers(); + } + }); + + it('does not retry non-transient errors', async () => { + const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0'); + const daytona = makeFakeDaytona(); + daytona.snapshot.create.mockRejectedValue(new DaytonaError('invalid image', 400)); + + await expect(manager.createSnapshot(daytona as never)).rejects.toThrow('invalid image'); + expect(daytona.snapshot.create).toHaveBeenCalledTimes(1); + }); + + it('reactivates an existing inactive snapshot instead of failing', async () => { + const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0'); + const daytona = makeFakeDaytona(); + daytona.snapshot.create.mockRejectedValue(new DaytonaError('already exists', 409)); + daytona.snapshot.get + .mockResolvedValueOnce({ name: SNAPSHOT_NAME, state: 'inactive' }) + .mockResolvedValue({ name: SNAPSHOT_NAME, state: 'active' }); + + await manager.ensureImage(); + vi.useFakeTimers(); + try { + const promise = manager.createSnapshot(daytona as never); + await vi.runAllTimersAsync(); + + await expect(promise).resolves.toBe(SNAPSHOT_NAME); + expect(daytona.snapshot.activate).toHaveBeenCalledTimes(1); + expect(daytona.snapshot.activate).toHaveBeenCalledWith( + expect.objectContaining({ name: SNAPSHOT_NAME, state: 'inactive' }), + ); + } finally { + vi.useRealTimers(); + } + }); + + it('retries activation on a transient error at the next settle window', async () => { + const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0'); + const daytona = makeFakeDaytona(); + daytona.snapshot.create.mockRejectedValue(new DaytonaError('already exists', 409)); + let polls = 0; + daytona.snapshot.get.mockImplementation( + async () => + // 8 inactive polls: attempt 1 fails on poll 1, the settle window + // elapses over polls 2-7, attempt 2 succeeds on poll 8. + await Promise.resolve({ name: SNAPSHOT_NAME, state: ++polls <= 8 ? 'inactive' : 'active' }), + ); + daytona.snapshot.activate + .mockRejectedValueOnce(new DaytonaError('502 Bad Gateway', 502)) + .mockImplementation(async (snapshot) => await Promise.resolve(snapshot)); + + await manager.ensureImage(); + vi.useFakeTimers(); + try { + const promise = manager.createSnapshot(daytona as never); + await vi.runAllTimersAsync(); + + await expect(promise).resolves.toBe(SNAPSHOT_NAME); + expect(daytona.snapshot.activate).toHaveBeenCalledTimes(2); + } finally { + vi.useRealTimers(); + } + }); + + it('gives up after exhausting activation attempts on a stuck-inactive snapshot', async () => { + const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0'); + const daytona = makeFakeDaytona(); + daytona.snapshot.create.mockRejectedValue(new DaytonaError('already exists', 409)); + daytona.snapshot.get.mockResolvedValue({ name: SNAPSHOT_NAME, state: 'inactive' }); + + await manager.ensureImage(); + vi.useFakeTimers(); + try { + const assertion = expect(manager.createSnapshot(daytona as never)).rejects.toThrow( + 'remained inactive after 3 activation requests', + ); + await vi.runAllTimersAsync(); + await assertion; + expect(daytona.snapshot.activate).toHaveBeenCalledTimes(3); + } finally { + vi.useRealTimers(); + } + }); + + it('waits after requesting activation and times out if the snapshot stays inactive', async () => { + const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0'); + const daytona = makeFakeDaytona(); + daytona.snapshot.create.mockRejectedValue(new DaytonaError('already exists', 409)); + daytona.snapshot.get.mockResolvedValue({ name: SNAPSHOT_NAME, state: 'inactive' }); + + await manager.ensureImage(); + vi.useFakeTimers(); + try { + const assertion = expect( + manager.createSnapshot(daytona as never, { timeout: 1 }), + ).rejects.toThrow('Timed out waiting'); + await vi.runAllTimersAsync(); + await assertion; + expect(daytona.snapshot.activate).toHaveBeenCalledTimes(1); + } finally { + vi.useRealTimers(); + } + }); + + it('bounds a hung status poll with the overall deadline', async () => { + const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0'); + const daytona = makeFakeDaytona(); + daytona.snapshot.create.mockResolvedValue({ name: SNAPSHOT_NAME }); + // A request that never settles (stalled transport, no error). + daytona.snapshot.get.mockImplementation(async () => await new Promise(() => {})); + + await manager.ensureImage(); + vi.useFakeTimers(); + try { + const assertion = expect( + manager.createSnapshot(daytona as never, { timeout: 60 }), + ).rejects.toThrow('Timed out fetching state'); + await vi.runAllTimersAsync(); + await assertion; + } finally { + vi.useRealTimers(); + } + }); + + it('tolerates a transient error while polling snapshot state', async () => { + const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0'); + const daytona = makeFakeDaytona(); + daytona.snapshot.create.mockResolvedValue({ name: SNAPSHOT_NAME }); + daytona.snapshot.get + .mockRejectedValueOnce(new DaytonaError('502 Bad Gateway', 502)) + .mockResolvedValue({ name: SNAPSHOT_NAME, state: 'active' }); + + await manager.ensureImage(); + vi.useFakeTimers(); + try { + const promise = manager.createSnapshot(daytona as never); + await vi.runAllTimersAsync(); + + await expect(promise).resolves.toBe(SNAPSHOT_NAME); + expect(daytona.snapshot.get).toHaveBeenCalledTimes(2); + } finally { + vi.useRealTimers(); + } }); it('throws when no version is configured', async () => { @@ -357,6 +723,340 @@ describe('SnapshotManager.createSnapshot', () => { }); }); +describe('SnapshotManager snapshot pruning', () => { + it('prunes on quota exhaustion and retries the create once', async () => { + const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0'); + const daytona = makeFakeDaytona(); + daytona.snapshot.create + .mockRejectedValueOnce(new DaytonaError('Snapshot quota exceeded. Maximum allowed: 30')) + .mockResolvedValue({ name: SNAPSHOT_NAME }); + daytona.snapshot.list.mockResolvedValue( + snapshotPage([ + { name: 'n8n/instance-ai:1.122.0', state: 'active' }, + { name: 'n8n/instance-ai:1.121.0', state: 'active' }, + { name: 'n8n/instance-ai:1.120.0', state: 'active' }, + { name: 'n8n/instance-ai:1.119.0', state: 'active' }, + { name: 'n8n/instance-ai:1.118.0', state: 'inactive' }, + ]), + ); + mockGetActiveOnlyFor(daytona, SNAPSHOT_NAME); + + const result = await manager.createSnapshot(daytona as never, { retention: 2 }); + + expect(result).toBe(SNAPSHOT_NAME); + expect(daytona.snapshot.create).toHaveBeenCalledTimes(2); + // The count backstop evicts beyond the newest-3 floor. + const deletedNames = daytona.snapshot.delete.mock.calls.map(([snapshot]) => snapshot.name); + expect(deletedNames).toContain('n8n/instance-ai:1.119.0'); + expect(deletedNames).toContain('n8n/instance-ai:1.118.0'); + }); + + it('waits for pruned snapshots to finish removing before retrying after a quota error', async () => { + const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0'); + const daytona = makeFakeDaytona(); + daytona.snapshot.create + .mockRejectedValueOnce(new DaytonaError('Snapshot quota exceeded. Maximum allowed: 30')) + .mockResolvedValue({ name: SNAPSHOT_NAME }); + daytona.snapshot.list.mockResolvedValue( + snapshotPage([ + { name: 'n8n/instance-ai:1.122.0', state: 'active' }, + { name: 'n8n/instance-ai:1.121.0', state: 'active' }, + { name: 'n8n/instance-ai:1.120.0', state: 'active' }, + { name: 'n8n/instance-ai:1.119.0', state: 'active' }, + ]), + ); + // The deleted snapshot lingers in `removing` before disappearing. + daytona.snapshot.get.mockImplementation(async (requested) => { + if (requested === SNAPSHOT_NAME) + return await Promise.resolve({ name: SNAPSHOT_NAME, state: 'active' }); + if (daytona.snapshot.get.mock.calls.filter(([n]) => n === requested).length <= 1) + return await Promise.resolve({ name: requested, state: 'removing' }); + throw new DaytonaNotFoundError(`Snapshot ${requested} not found`); + }); + + await manager.ensureImage(); + vi.useFakeTimers(); + try { + const promise = manager.createSnapshot(daytona as never, { retention: 3 }); + await vi.runAllTimersAsync(); + + await expect(promise).resolves.toBe(SNAPSHOT_NAME); + // The retry happened only after the pruned snapshot was gone. + expect(daytona.snapshot.create).toHaveBeenCalledTimes(2); + const removalPolls = daytona.snapshot.get.mock.calls.filter( + ([requested]) => requested === 'n8n/instance-ai:1.119.0', + ); + expect(removalPolls.length).toBeGreaterThanOrEqual(2); + } finally { + vi.useRealTimers(); + } + }); + + it('throws the quota error when no retention is configured', async () => { + const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0'); + const daytona = makeFakeDaytona(); + daytona.snapshot.create.mockRejectedValue( + new DaytonaError('Snapshot quota exceeded. Maximum allowed: 30'), + ); + + await expect(manager.createSnapshot(daytona as never)).rejects.toThrow('quota exceeded'); + expect(daytona.snapshot.list).not.toHaveBeenCalled(); + expect(daytona.snapshot.create).toHaveBeenCalledTimes(1); + }); + + it('treats an explicit retention of 0 as pruning disabled on the quota path', async () => { + const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0'); + const daytona = makeFakeDaytona(); + daytona.snapshot.create.mockRejectedValue( + new DaytonaError('Snapshot quota exceeded. Maximum allowed: 30'), + ); + + await expect(manager.createSnapshot(daytona as never, { retention: 0 })).rejects.toThrow( + 'quota exceeded', + ); + expect(daytona.snapshot.list).not.toHaveBeenCalled(); + }); + + it('force-evicts the least-recently-used snapshot when quota is held below the retention window', async () => { + // Foreign snapshots can exhaust the org quota while our own count is + // within policy — the publish still needs one slot freed. + const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0'); + const daytona = makeFakeDaytona(); + daytona.snapshot.create + .mockRejectedValueOnce(new DaytonaError('Snapshot quota exceeded. Maximum allowed: 30')) + .mockResolvedValue({ name: SNAPSHOT_NAME }); + daytona.snapshot.list.mockResolvedValue( + snapshotPage([ + { name: 'n8n/instance-ai:1.122.0', state: 'active', lastUsedAt: daysAgo(1) }, + { name: 'n8n/instance-ai:1.121.0', state: 'active', lastUsedAt: daysAgo(2) }, + { name: 'n8n/instance-ai:1.120.0', state: 'active', lastUsedAt: daysAgo(3) }, + { name: 'n8n/instance-ai:1.119.0', state: 'active', lastUsedAt: daysAgo(10) }, + ]), + ); + mockGetActiveOnlyFor(daytona, SNAPSHOT_NAME); + + const result = await manager.createSnapshot(daytona as never, { retention: 15 }); + + expect(result).toBe(SNAPSHOT_NAME); + expect(daytona.snapshot.create).toHaveBeenCalledTimes(2); + const deletedNames = daytona.snapshot.delete.mock.calls.map(([snapshot]) => snapshot.name); + expect(deletedNames).toEqual(['n8n/instance-ai:1.119.0']); + }); + + it('throws the quota error when pruning frees nothing', async () => { + const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0'); + const daytona = makeFakeDaytona(); + daytona.snapshot.create.mockRejectedValue( + new DaytonaError('Snapshot quota exceeded. Maximum allowed: 30'), + ); + daytona.snapshot.list.mockResolvedValue( + snapshotPage([{ name: 'n8n/instance-ai:1.122.0', state: 'active' }]), + ); + + await expect(manager.createSnapshot(daytona as never, { retention: 5 })).rejects.toThrow( + 'quota exceeded', + ); + expect(daytona.snapshot.create).toHaveBeenCalledTimes(1); + expect(daytona.snapshot.delete).not.toHaveBeenCalled(); + }); + + it('enforces the count cap after a successful publish, sparing the newest versions and in-progress builds', async () => { + const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0'); + const daytona = makeFakeDaytona(); + daytona.snapshot.create.mockResolvedValue({ name: SNAPSHOT_NAME }); + daytona.snapshot.list.mockResolvedValue( + snapshotPage([ + { name: 'n8n/instance-ai:1.122.0', state: 'active' }, + { name: SNAPSHOT_NAME, state: 'active' }, + { name: 'n8n/instance-ai:1.121.0', state: 'inactive' }, + { name: 'n8n/instance-ai:1.121.0-abc123', state: 'active' }, + { name: 'n8n/instance-ai:1.120.0', state: 'building' }, + { name: 'someone-elses/snapshot:1.0.0', state: 'active' }, + ]), + ); + + await manager.createSnapshot(daytona as never, { retention: 2 }); + + // The newest 3 versions (1.123.0, 1.122.0, 1.121.0 — suffixed + // 1.121.0-abc123 ranks below plain 1.121.0) are floor-protected; the + // building and foreign snapshots are untouched. + const deletedNames = daytona.snapshot.delete.mock.calls.map(([snapshot]) => snapshot.name); + expect(deletedNames).toEqual(['n8n/instance-ai:1.121.0-abc123']); + }); + + it('age-prunes snapshots unused beyond maxAgeDays, keeping recently used and floor-protected ones', async () => { + const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0'); + const daytona = makeFakeDaytona(); + daytona.snapshot.create.mockResolvedValue({ name: SNAPSHOT_NAME }); + daytona.snapshot.list.mockResolvedValue( + snapshotPage([ + { name: SNAPSHOT_NAME, state: 'active', createdAt: daysAgo(0), lastUsedAt: daysAgo(0) }, + // Aged but within the newest-3 floor → kept. + { + name: 'n8n/instance-ai:1.122.0', + state: 'active', + createdAt: daysAgo(40), + lastUsedAt: daysAgo(25), + }, + // Created long ago but recently used → kept. + { + name: 'n8n/instance-ai:1.121.0', + state: 'active', + createdAt: daysAgo(30), + lastUsedAt: daysAgo(2), + }, + // Idle past the cutoff → pruned. + { + name: 'n8n/instance-ai:1.120.0', + state: 'inactive', + createdAt: daysAgo(40), + lastUsedAt: daysAgo(25), + }, + // No lastUsedAt → createdAt fallback → pruned. + { name: 'n8n/instance-ai:1.119.0', state: 'active', createdAt: daysAgo(25) }, + ]), + ); + + await manager.createSnapshot(daytona as never, { maxAgeDays: 20 }); + + const deletedNames = daytona.snapshot.delete.mock.calls.map(([snapshot]) => snapshot.name); + expect(deletedNames).toEqual(['n8n/instance-ai:1.120.0', 'n8n/instance-ai:1.119.0']); + }); + + it('does not let failed or suffixed snapshots consume rollback-floor slots', async () => { + const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0'); + const daytona = makeFakeDaytona(); + daytona.snapshot.create.mockResolvedValue({ name: SNAPSHOT_NAME }); + daytona.snapshot.list.mockResolvedValue( + snapshotPage([ + { name: SNAPSHOT_NAME, state: 'active', lastUsedAt: daysAgo(0) }, + // Failed build: deleted, and must not occupy a floor slot. + { name: 'n8n/instance-ai:1.122.0', state: 'build_failed', lastUsedAt: daysAgo(1) }, + // Suffixed build: not a rollback target; aged out → pruned. + { name: 'n8n/instance-ai:1.121.0-pr1', state: 'active', lastUsedAt: daysAgo(25) }, + // Idle plain releases: floor-protected because the failed and + // suffixed snapshots above don't count toward the newest-3 floor. + { name: 'n8n/instance-ai:1.120.0', state: 'active', lastUsedAt: daysAgo(25) }, + { name: 'n8n/instance-ai:1.119.0', state: 'active', lastUsedAt: daysAgo(25) }, + ]), + ); + + await manager.createSnapshot(daytona as never, { maxAgeDays: 20 }); + + const deletedNames = daytona.snapshot.delete.mock.calls.map(([snapshot]) => snapshot.name); + expect(deletedNames).toEqual(['n8n/instance-ai:1.122.0', 'n8n/instance-ai:1.121.0-pr1']); + }); + + it('count-cap eviction is LRU: an old version still in use outlives an idle newer one', async () => { + const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0'); + const daytona = makeFakeDaytona(); + daytona.snapshot.create.mockResolvedValue({ name: SNAPSHOT_NAME }); + daytona.snapshot.list.mockResolvedValue( + snapshotPage([ + { name: SNAPSHOT_NAME, state: 'active', lastUsedAt: daysAgo(0) }, + { name: 'n8n/instance-ai:1.122.0', state: 'active', lastUsedAt: daysAgo(1) }, + { name: 'n8n/instance-ai:1.121.0', state: 'active', lastUsedAt: daysAgo(1) }, + // Oldest version but used yesterday (a pinned instance) → kept. + { name: 'n8n/instance-ai:1.100.0', state: 'active', lastUsedAt: daysAgo(1) }, + // Idle for two weeks → evicted first. + { name: 'n8n/instance-ai:1.119.0', state: 'active', lastUsedAt: daysAgo(15) }, + { name: 'n8n/instance-ai:1.118.0', state: 'active', lastUsedAt: daysAgo(10) }, + ]), + ); + + await manager.createSnapshot(daytona as never, { retention: 5 }); + + const deletedNames = daytona.snapshot.delete.mock.calls.map(([snapshot]) => snapshot.name); + expect(deletedNames).toEqual(['n8n/instance-ai:1.119.0']); + }); + + it('deletes failed snapshots even within the retention window', async () => { + const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0'); + const daytona = makeFakeDaytona(); + daytona.snapshot.create.mockResolvedValue({ name: SNAPSHOT_NAME }); + daytona.snapshot.list.mockResolvedValue( + snapshotPage([ + { name: SNAPSHOT_NAME, state: 'active' }, + { name: 'n8n/instance-ai:1.122.0', state: 'build_failed', errorReason: 'npm exit 1' }, + { name: 'n8n/instance-ai:1.121.0', state: 'active' }, + ]), + ); + + await manager.createSnapshot(daytona as never, { retention: 3 }); + + const deletedNames = daytona.snapshot.delete.mock.calls.map(([snapshot]) => snapshot.name); + expect(deletedNames).toEqual(['n8n/instance-ai:1.122.0']); + }); + + it('never deletes the snapshot being published', async () => { + // Republish of an old version that ranks outside the retention window. + const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0'); + const daytona = makeFakeDaytona(); + daytona.snapshot.create.mockResolvedValue({ name: SNAPSHOT_NAME }); + daytona.snapshot.list.mockResolvedValue( + snapshotPage([ + { name: 'n8n/instance-ai:2.0.0', state: 'active' }, + { name: SNAPSHOT_NAME, state: 'active' }, + ]), + ); + + await manager.createSnapshot(daytona as never, { retention: 1 }); + + expect(daytona.snapshot.delete).not.toHaveBeenCalled(); + }); + + it('paginates through the snapshot list', async () => { + const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0'); + const daytona = makeFakeDaytona(); + daytona.snapshot.create.mockResolvedValue({ name: SNAPSHOT_NAME }); + daytona.snapshot.list + .mockResolvedValueOnce(snapshotPage([{ name: SNAPSHOT_NAME, state: 'active' }], 1, 2)) + .mockResolvedValueOnce( + snapshotPage([{ name: 'n8n/instance-ai:1.100.0', state: 'build_failed' }], 2, 2), + ); + + await manager.createSnapshot(daytona as never, { retention: 1 }); + + expect(daytona.snapshot.list).toHaveBeenCalledTimes(2); + const deletedNames = daytona.snapshot.delete.mock.calls.map(([snapshot]) => snapshot.name); + expect(deletedNames).toEqual(['n8n/instance-ai:1.100.0']); + }); + + it('does not fail the publish when pruning fails', async () => { + const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0'); + const daytona = makeFakeDaytona(); + daytona.snapshot.create.mockResolvedValue({ name: SNAPSHOT_NAME }); + daytona.snapshot.list.mockRejectedValue(new DaytonaError('boom', 500)); + + await expect(manager.createSnapshot(daytona as never, { retention: 2 })).resolves.toBe( + SNAPSHOT_NAME, + ); + }); + + it('does not fail the publish when a single delete fails', async () => { + const manager = new SnapshotManager(undefined, NOOP_LOGGER, '1.123.0'); + const daytona = makeFakeDaytona(); + daytona.snapshot.create.mockResolvedValue({ name: SNAPSHOT_NAME }); + daytona.snapshot.list.mockResolvedValue( + snapshotPage([ + { name: SNAPSHOT_NAME, state: 'active' }, + { name: 'n8n/instance-ai:1.122.0', state: 'active' }, + { name: 'n8n/instance-ai:1.121.0', state: 'active' }, + { name: 'n8n/instance-ai:1.120.0', state: 'active' }, + { name: 'n8n/instance-ai:1.119.0', state: 'active' }, + ]), + ); + daytona.snapshot.delete + .mockRejectedValueOnce(new DaytonaError('delete failed', 500)) + .mockResolvedValue(undefined); + + await expect(manager.createSnapshot(daytona as never, { retention: 1 })).resolves.toBe( + SNAPSHOT_NAME, + ); + expect(daytona.snapshot.delete).toHaveBeenCalledTimes(2); + }); +}); + describe('SnapshotManager.snapshotName', () => { it('returns null when no version is configured', () => { const manager = new SnapshotManager(undefined, NOOP_LOGGER, undefined); diff --git a/packages/@n8n/instance-ai/src/workspace/snapshot-manager.ts b/packages/@n8n/instance-ai/src/workspace/snapshot-manager.ts index e6ee599328a..d7bc6f0cabc 100644 --- a/packages/@n8n/instance-ai/src/workspace/snapshot-manager.ts +++ b/packages/@n8n/instance-ai/src/workspace/snapshot-manager.ts @@ -34,13 +34,56 @@ import { loadInstanceAiRuntimeSkillSource } from '../skills/runtime-skills'; export interface CreateSnapshotOptions { timeout?: number; onLogs?: (chunk: string) => void; + /** + * Hard cap on versioned snapshots per organization (quota backstop). When + * the count exceeds this after age pruning, the least-recently-used + * snapshots are deleted until the cap is met. Unset disables the cap. + */ + retention?: number; + /** + * Delete versioned snapshots not used within this many days + * (`lastUsedAt`, falling back to `createdAt`). Snapshots still in use by + * older n8n versions keep a fresh `lastUsedAt` and survive regardless of + * release cadence. Unset disables age pruning. + */ + maxAgeDays?: number; } +type DaytonaSnapshot = Awaited>; + const DAYTONA_WORKSPACE_BAKE_ROOT = '/tmp/n8n-workspace-bake'; const SNAPSHOT_WORKSPACE_LAYOUT_DIRS = ['src', 'chunks', 'node-types'] as const; const SNAPSHOT_BUILDING_STATES = new Set(['building', 'pending', 'pulling']); const SNAPSHOT_VERIFY_POLL_MS = 5_000; const DEFAULT_SNAPSHOT_VERIFY_TIMEOUT_S = 1_800; +const SNAPSHOT_NAME_PREFIX = 'n8n/instance-ai:'; +const MAX_TRANSIENT_CREATE_RETRIES = 3; +const TRANSIENT_CREATE_RETRY_BACKOFF_MS = 5_000; +/** + * Times a failed (`error`/`build_failed`) record for the target version is deleted and + * the create retried before giving up. Automates the manual "delete the broken snapshot + * in the Daytona UI and re-run" remediation. + */ +const MAX_FAILED_SNAPSHOT_CLEANUPS = 2; +const SNAPSHOT_LIST_PAGE_SIZE = 100; +const MAX_SNAPSHOT_LIST_PAGES = 20; +// Bound for the post-publish prune: it runs after the snapshot is verified, so +// a hung Daytona call must not stall a release (the SDK has no sane default +// HTTP timeout). +const SNAPSHOT_PRUNE_TIMEOUT_MS = 5 * 60_000; +// Deleted snapshots pass through `removing` before their quota slot frees up. +const SNAPSHOT_REMOVAL_WAIT_MS = 60_000; +const SNAPSHOT_REMOVAL_POLL_MS = 2_000; +const MAX_ACTIVATION_ATTEMPTS = 3; +// Polls (5s apart) to let an activation request settle before re-requesting. +const ACTIVATION_SETTLE_POLLS = 6; +// States a snapshot can be safely deleted from. Never delete in-progress +// builds (a concurrent release job may own them) or already-removing ones. +const SNAPSHOT_DELETABLE_STATES = new Set(['active', 'inactive', 'error', 'build_failed']); +const SNAPSHOT_FAILED_STATES = new Set(['error', 'build_failed']); +// Rollback insurance: the newest versions are never age- or count-pruned even +// when idle (e.g. a quiet dev org where nothing was used for weeks). +const MIN_KEEP_NEWEST_VERSIONS = 3; function isAlreadyExistsError(error: unknown): error is TDaytonaError { const { DaytonaError } = loadDaytona(); @@ -49,6 +92,91 @@ function isAlreadyExistsError(error: unknown): error is TDaytonaError { return /already exists/i.test(error.message); } +/** + * The SDK's `snapshot.create` polls the new record and synthesizes this error (no + * statusCode) when the record lands in `error`/`build_failed` — e.g. "Reason: An + * operation is already in progress for this resource" when concurrent publishes race. + * The failed record persists under the version's name and blocks every retry until + * it is deleted. + */ +function isCreateFailedStateError(error: unknown): boolean { + const { DaytonaError } = loadDaytona(); + return error instanceof DaytonaError && /^failed to create snapshot\b/i.test(error.message); +} + +/** + * Internal signal from {@link SnapshotManager.verifySnapshot}: the record exists in a + * state that can never become active. Carries the state so the publish loop can decide + * whether deleting and rebuilding is worthwhile (failed states only). + */ +class SnapshotUnusableError extends Error { + constructor( + message: string, + readonly snapshotState: string, + ) { + super(message); + } +} + +/** + * Gateway/availability errors worth retrying: 5xx/408/429 responses, plus the + * SDK's connection/timeout errors (matched by name — they carry no statusCode). + */ +function isTransientDaytonaError(error: unknown): boolean { + const { DaytonaError } = loadDaytona(); + if (!(error instanceof DaytonaError)) return false; + if (error.name === 'DaytonaConnectionError' || error.name === 'DaytonaTimeoutError') return true; + const status = error.statusCode; + return status !== undefined && (status >= 500 || status === 408 || status === 429); +} + +// The SDK has no dedicated error class or status mapping for quota rejections; +// the message (e.g. "Snapshot quota exceeded. Maximum allowed: 30") is the only +// signal. Deliberately narrow: a rewording degrades to the pre-existing +// hard-fail rather than triggering pruning on unrelated quota-flavored errors. +function isQuotaExceededError(error: unknown): boolean { + const { DaytonaError } = loadDaytona(); + return error instanceof DaytonaError && /snapshot quota exceeded/i.test(error.message); +} + +/** + * Order snapshots newest-version-first by the version in their name. + * A suffixed version (`2.23.0-`) sorts older than its plain release; + * names whose version segment is unparseable sort oldest. + */ +function compareSnapshotVersionsDesc(a: DaytonaSnapshot, b: DaytonaSnapshot): number { + return parseSnapshotVersionRank(b.name) - parseSnapshotVersionRank(a.name); +} + +function parseSnapshotVersionRank(name: string): number { + const version = name.slice(SNAPSHOT_NAME_PREFIX.length); + const match = /^(\d+)\.(\d+)\.(\d+)(-.+)?$/.exec(version); + if (!match) return Number.NEGATIVE_INFINITY; + const [, major, minor, patch, suffix] = match; + // Scale leaves room for four-digit minor/patch; -0.5 ranks suffixed builds + // below their plain release. + return ( + Number(major) * 1e8 + Number(minor) * 1e4 + Number(patch) + (suffix !== undefined ? -0.5 : 0) + ); +} + +// Plain release versions only — suffixed builds (`2.23.0-`) and failed +// snapshots must not consume rollback-floor slots. +function isPlainVersionName(name: string): boolean { + return /^\d+\.\d+\.\d+$/.test(name.slice(SNAPSHOT_NAME_PREFIX.length)); +} + +/** + * When a snapshot was last used, for LRU pruning. `lastUsedAt` is bumped by + * sandbox creation (verified against real org data); `createdAt` covers + * never-used snapshots. Unparseable timestamps count as just-used so bad data + * never causes a deletion. + */ +function lastUsedTime(snapshot: DaytonaSnapshot): number { + const time = new Date(snapshot.lastUsedAt ?? snapshot.createdAt).getTime(); + return Number.isNaN(time) ? Number.POSITIVE_INFINITY : time; +} + export class SnapshotManager { private cachedImage: Promise | null = null; @@ -118,8 +246,9 @@ export class SnapshotManager { /** * Create the versioned Daytona snapshot for the configured n8n version. * Treats 409 / "already exists" as success — re-runs against the same - * version are idempotent. Throws on transient or unexpected errors so - * callers can decide whether to retry, fall back, or fail loudly. + * version are idempotent. Retries transient gateway errors, prunes old + * versioned snapshots on quota exhaustion (when `retention` is set), and + * reactivates an existing-but-inactive snapshot instead of failing. * * Single source of truth for snapshot creation in the CI release pipeline * (`scripts/build-snapshot.cjs`). Runtime never calls this. @@ -130,45 +259,314 @@ export class SnapshotManager { throw new Error('SnapshotManager: n8nVersion is required to derive a snapshot name'); } - try { - await daytona.snapshot.create({ name, image: await this.ensureImage() }, options); - this.logger.info('Created versioned Daytona snapshot; verifying it is usable', { name }); - } catch (error) { - if (isAlreadyExistsError(error)) { - this.logger.info('Versioned Daytona snapshot already exists; verifying it is usable', { + // The SDK treats `timeout: 0` as "no timeout"; here it means the default + // total budget rather than an instantly-expired deadline. + const timeoutS = + options?.timeout !== undefined && options.timeout > 0 + ? options.timeout + : DEFAULT_SNAPSHOT_VERIFY_TIMEOUT_S; + const deadline = Date.now() + timeoutS * 1000; + + // One rebuild when the published record turns out unusable: a build that + // finished in a failed state (e.g. broken by a concurrent operation) blocks + // this version until its record is deleted, so delete it and publish again. + for (let rebuilds = 0; ; rebuilds++) { + await this.createWithRecovery(daytona, name, deadline, options); + try { + await this.verifySnapshot(daytona, name, deadline); + break; + } catch (error) { + const rebuildable = + error instanceof SnapshotUnusableError && + SNAPSHOT_FAILED_STATES.has(error.snapshotState) && + rebuilds < 1 && + Date.now() < deadline; + if (!rebuildable) throw error; + this.logger.warn('Published Daytona snapshot is unusable; deleting it and rebuilding', { name, + state: error.snapshotState, }); - } else { + await this.reconcileFailedSnapshotRecord(daytona, name, deadline); + } + } + try { + // Best-effort and time-boxed: neither a prune failure nor a hung + // Daytona call may fail a release that just published a healthy + // snapshot. + await this.withDeadline( + this.pruneSnapshots(daytona, name, options), + Date.now() + SNAPSHOT_PRUNE_TIMEOUT_MS, + 'Timed out pruning Daytona snapshots', + ); + } catch (error) { + this.logger.warn('Snapshot pruning did not complete', { + error: error instanceof Error ? error.message : String(error), + }); + } + return name; + } + + /** + * Run `snapshot.create` with recovery for the two failure modes CI has hit: + * quota exhaustion (prune old versions once, then retry) and transient + * gateway errors (bounded retries; a re-create after the first request + * registered lands on the idempotent 409 path). + * + * The SDK's `timeout` option only bounds the initial POST — its status + * polling loop is unbounded — so each attempt is raced against `deadline`. + */ + private async createWithRecovery( + daytona: Daytona, + name: string, + deadline: number, + options?: CreateSnapshotOptions, + ): Promise { + const createOptions = { timeout: options?.timeout, onLogs: options?.onLogs }; + let transientRetries = 0; + let failedCleanups = 0; + let prunedForQuota = false; + for (;;) { + try { + const image = await this.withDeadline( + this.ensureImage(), + deadline, + `Timed out preparing the image for Daytona snapshot "${name}"`, + ); + await this.withDeadline( + daytona.snapshot.create({ name, image }, createOptions), + deadline, + `Timed out creating Daytona snapshot "${name}"`, + ); + this.logger.info('Created versioned Daytona snapshot; verifying it is usable', { name }); + return; + } catch (error) { + if (isCreateFailedStateError(error)) { + // The build landed in `error`/`build_failed` (e.g. broken by a concurrent + // operation on the same name). The failed record blocks every retry of + // this version, so delete it and retry instead of requiring a manual + // deletion in the Daytona UI. + if (failedCleanups < MAX_FAILED_SNAPSHOT_CLEANUPS) { + const record = await this.reconcileFailedSnapshotRecord(daytona, name, deadline); + if (record !== 'usable') { + failedCleanups++; + await sleep( + Math.min(TRANSIENT_CREATE_RETRY_BACKOFF_MS * failedCleanups, deadline - Date.now()), + ); + if (Date.now() >= deadline) throw error; + continue; + } + } + throw error; + } + if (isAlreadyExistsError(error)) { + // A pre-existing failed record is handled by the verify + rebuild loop. + this.logger.info('Versioned Daytona snapshot already exists; verifying it is usable', { + name, + }); + return; + } + const canPrune = (options?.retention ?? 0) > 0 || (options?.maxAgeDays ?? 0) > 0; + if (isQuotaExceededError(error) && !prunedForQuota && canPrune) { + prunedForQuota = true; + this.logger.warn('Snapshot quota exceeded; pruning old versioned snapshots', { name }); + // Quota may be held below the retention window (e.g. by foreign + // snapshots in the org), so force at least one LRU eviction. + const deleted = await this.withDeadline( + this.pruneSnapshots(daytona, name, options, { ensureAtLeastOne: true }), + deadline, + 'Timed out pruning Daytona snapshots', + ).catch((): DaytonaSnapshot[] => []); + if (deleted.length === 0) throw error; + // Deletion is asynchronous server-side (snapshots pass through + // `removing`); the quota slot only frees once they are gone. + await this.waitForSnapshotRemoval(daytona, deleted, deadline); + continue; + } + if ( + isTransientDaytonaError(error) && + transientRetries < MAX_TRANSIENT_CREATE_RETRIES && + Date.now() < deadline + ) { + transientRetries++; + this.logger.warn('Transient Daytona error during snapshot create; retrying', { + name, + attempt: transientRetries, + error: error instanceof Error ? error.message : String(error), + }); + await sleep( + Math.min(TRANSIENT_CREATE_RETRY_BACKOFF_MS * transientRetries, deadline - Date.now()), + ); + // The backoff may have consumed the remaining budget; surface the + // real error instead of a confusing deadline timeout. + if (Date.now() >= deadline) throw error; + continue; + } throw error; } } + } - await this.verifySnapshot(daytona, name, options?.timeout); - return name; + /** + * Look up the record for `name` and, when it sits in a failed state, delete it and + * wait for the removal — a failed record permanently blocks republishing the version. + * Returns 'cleaned' when a failed record was deleted, 'absent' when no record exists + * (create can be retried directly), and 'usable' when the record is in any live state. + */ + private async reconcileFailedSnapshotRecord( + daytona: Daytona, + name: string, + deadline: number, + ): Promise<'cleaned' | 'absent' | 'usable'> { + const { DaytonaNotFoundError } = loadDaytona(); + let snapshot: DaytonaSnapshot; + try { + snapshot = await this.withDeadline( + daytona.snapshot.get(name), + deadline, + `Timed out fetching state of Daytona snapshot "${name}"`, + ); + } catch (error) { + if (error instanceof DaytonaNotFoundError) return 'absent'; + throw error; + } + if (!SNAPSHOT_FAILED_STATES.has(snapshot.state)) return 'usable'; + this.logger.warn('Versioned Daytona snapshot is in a failed state; deleting it to retry', { + name, + state: snapshot.state, + ...(snapshot.errorReason ? { reason: snapshot.errorReason } : {}), + }); + await this.withDeadline( + daytona.snapshot.delete(snapshot), + deadline, + `Timed out deleting failed Daytona snapshot "${name}"`, + ); + await this.waitForSnapshotRemoval(daytona, [snapshot], deadline); + return 'cleaned'; + } + + /** + * Wait until pruned snapshots are actually gone (a 404 on lookup). Bounded + * and best-effort: on timeout or an unexpected lookup result the create is + * retried anyway and surfaces whatever is still wrong. + */ + private async waitForSnapshotRemoval( + daytona: Daytona, + snapshots: DaytonaSnapshot[], + deadline: number, + ): Promise { + const { DaytonaNotFoundError } = loadDaytona(); + const waitDeadline = Math.min(deadline, Date.now() + SNAPSHOT_REMOVAL_WAIT_MS); + for (const snapshot of snapshots) { + for (;;) { + try { + // Race the lookup itself against the wait budget — the SDK's transport + // timeout is effectively unbounded, so a stalled request would + // otherwise hang past the deadline. + await this.withDeadline( + daytona.snapshot.get(snapshot.name), + waitDeadline, + `Timed out waiting for Daytona snapshot "${snapshot.name}" to be removed`, + ); + } catch (error) { + if (!(error instanceof DaytonaNotFoundError)) { + this.logger.warn('Unexpected error while waiting for snapshot removal', { + name: snapshot.name, + error: error instanceof Error ? error.message : String(error), + }); + } + break; + } + if (Date.now() >= waitDeadline) { + this.logger.warn('Timed out waiting for pruned snapshots to be removed', { + name: snapshot.name, + }); + return; + } + await sleep(SNAPSHOT_REMOVAL_POLL_MS); + } + } } /** * Verify that the snapshot is actually usable. A snapshot build can finish in * `error`/`build_failed` state; returning before it is active would let a release - * ship without a working snapshot. Waits out in-progress builds, throws on any - * unusable state. + * ship without a working snapshot. Waits out in-progress builds, reactivates an + * `inactive` snapshot (Daytona deactivates idle ones), tolerates transient poll + * errors, and throws on any unusable state. */ - private async verifySnapshot( - daytona: Daytona, - name: string, - timeoutS = DEFAULT_SNAPSHOT_VERIFY_TIMEOUT_S, - ): Promise { - const deadline = Date.now() + timeoutS * 1000; + private async verifySnapshot(daytona: Daytona, name: string, deadline: number): Promise { + let activationAttempts = 0; + // Start at the threshold so the first `inactive` poll requests activation + // immediately. + let pollsSinceActivation = ACTIVATION_SETTLE_POLLS; for (;;) { - const snapshot = await daytona.snapshot.get(name); + let snapshot: DaytonaSnapshot; + try { + // Race every request against the deadline — the SDK's transport + // timeout is effectively unbounded, so a stalled request would + // otherwise hang the job past the budget. + snapshot = await this.withDeadline( + daytona.snapshot.get(name), + deadline, + `Timed out fetching state of Daytona snapshot "${name}"`, + ); + } catch (error) { + if (!isTransientDaytonaError(error) || Date.now() >= deadline) throw error; + this.logger.warn('Transient Daytona error while polling snapshot state; retrying', { + name, + error: error instanceof Error ? error.message : String(error), + }); + await sleep(SNAPSHOT_VERIFY_POLL_MS); + continue; + } if (snapshot.state === 'active') { this.logger.info('Versioned Daytona snapshot is active', { name }); return; } + if (snapshot.state === 'inactive') { + // Re-request activation every settle window; transient failures + // retry on the next window instead of failing the release. + if (pollsSinceActivation >= ACTIVATION_SETTLE_POLLS) { + if (activationAttempts >= MAX_ACTIVATION_ATTEMPTS) { + throw new Error( + `Versioned Daytona snapshot "${name}" remained inactive after ${MAX_ACTIVATION_ATTEMPTS} activation requests`, + ); + } + activationAttempts++; + pollsSinceActivation = 0; + this.logger.info('Versioned Daytona snapshot is inactive; requesting activation', { + name, + attempt: activationAttempts, + }); + try { + await this.withDeadline( + daytona.snapshot.activate(snapshot), + deadline, + `Timed out requesting activation of Daytona snapshot "${name}"`, + ); + } catch (error) { + if (!isTransientDaytonaError(error)) throw error; + this.logger.warn('Transient Daytona error during snapshot activation; will retry', { + name, + error: error instanceof Error ? error.message : String(error), + }); + } + } else { + pollsSinceActivation++; + } + if (Date.now() >= deadline) { + throw new Error( + `Timed out waiting for existing Daytona snapshot "${name}" to become active (state: ${snapshot.state})`, + ); + } + await sleep(SNAPSHOT_VERIFY_POLL_MS); + continue; + } if (!SNAPSHOT_BUILDING_STATES.has(snapshot.state)) { const reason = snapshot.errorReason ? `, reason: ${snapshot.errorReason}` : ''; - throw new Error( + throw new SnapshotUnusableError( `Versioned Daytona snapshot "${name}" exists but is unusable (state: ${snapshot.state}${reason})`, + snapshot.state, ); } if (Date.now() >= deadline) { @@ -176,7 +574,7 @@ export class SnapshotManager { `Timed out waiting for existing Daytona snapshot "${name}" to become active (state: ${snapshot.state})`, ); } - this.logger.info('Waiting for existing Daytona snapshot to finish building', { + this.logger.info('Waiting for existing Daytona snapshot to become active', { name, state: snapshot.state, }); @@ -184,6 +582,158 @@ export class SnapshotManager { } } + /** + * Prune versioned snapshots. Age-based pruning (`maxAgeDays`, keyed on + * `lastUsedAt`) is the primary policy; the `retention` count is a quota + * backstop that evicts least-recently-used snapshots when the total still + * exceeds it. Failed snapshots are always deleted. Only touches + * `n8n/instance-ai:*` names, never the snapshot being published, never + * in-progress states, and never the newest `MIN_KEEP_NEWEST_VERSIONS` + * versions. Never throws: returns the snapshots actually deleted. + */ + private async pruneSnapshots( + daytona: Daytona, + protectedName: string, + options?: Pick, + { ensureAtLeastOne = false }: { ensureAtLeastOne?: boolean } = {}, + ): Promise { + const retention = options?.retention; + const maxAgeDays = options?.maxAgeDays; + if (!retention && !maxAgeDays) return []; + + let matching: DaytonaSnapshot[]; + try { + matching = await this.listVersionedSnapshots(daytona); + } catch (error) { + this.logger.warn('Failed to list Daytona snapshots for pruning; skipping', { + error: error instanceof Error ? error.message : String(error), + }); + return []; + } + + // Floor slots must go to usable release versions — failed and suffixed + // snapshots don't count as rollback targets. + const newestVersionNames = new Set( + matching + .filter( + (snapshot) => + !SNAPSHOT_FAILED_STATES.has(snapshot.state) && isPlainVersionName(snapshot.name), + ) + .sort(compareSnapshotVersionsDesc) + .slice(0, MIN_KEEP_NEWEST_VERSIONS) + .map((snapshot) => snapshot.name), + ); + const isDeletable = (snapshot: DaytonaSnapshot) => + snapshot.name !== protectedName && SNAPSHOT_DELETABLE_STATES.has(snapshot.state); + const isPrunable = (snapshot: DaytonaSnapshot) => + isDeletable(snapshot) && !newestVersionNames.has(snapshot.name); + + const toDelete: DaytonaSnapshot[] = []; + const selected = new Set(); + const select = (snapshot: DaytonaSnapshot) => { + if (selected.has(snapshot.name)) return; + selected.add(snapshot.name); + toDelete.push(snapshot); + }; + + // Failed snapshots are quota dead weight; the newest-versions floor + // does not protect them (they are unusable anyway). + for (const snapshot of matching) { + if (SNAPSHOT_FAILED_STATES.has(snapshot.state) && isDeletable(snapshot)) select(snapshot); + } + + if (maxAgeDays) { + const cutoff = Date.now() - maxAgeDays * 24 * 60 * 60 * 1000; + for (const snapshot of matching) { + if (!selected.has(snapshot.name) && isPrunable(snapshot) && lastUsedTime(snapshot) < cutoff) + select(snapshot); + } + } + + if (retention) { + const remaining = matching.filter((snapshot) => !selected.has(snapshot.name)); + let excess = remaining.length - retention; + const evictable = remaining + .filter(isPrunable) + .sort((a, b) => lastUsedTime(a) - lastUsedTime(b)); + for (const snapshot of evictable) { + if (excess <= 0) break; + select(snapshot); + excess--; + } + } + + // Under quota pressure the publish needs a free slot even when our own + // snapshots are within policy (e.g. foreign snapshots hold the quota): + // evict the least-recently-used prunable one. + if (ensureAtLeastOne && toDelete.length === 0) { + const lruCandidate = matching + .filter(isPrunable) + .sort((a, b) => lastUsedTime(a) - lastUsedTime(b))[0]; + if (lruCandidate) { + this.logger.warn( + 'Quota pressure: evicting least-recently-used snapshot despite retention policy', + { name: lruCandidate.name }, + ); + select(lruCandidate); + } + } + + const deleted: DaytonaSnapshot[] = []; + for (const snapshot of toDelete) { + try { + await daytona.snapshot.delete(snapshot); + deleted.push(snapshot); + this.logger.info('Pruned versioned Daytona snapshot', { + name: snapshot.name, + state: snapshot.state, + lastUsedAt: snapshot.lastUsedAt ?? null, + }); + } catch (error) { + this.logger.warn('Failed to delete Daytona snapshot during pruning', { + name: snapshot.name, + error: error instanceof Error ? error.message : String(error), + }); + } + } + return deleted; + } + + private async listVersionedSnapshots(daytona: Daytona): Promise { + const matching: DaytonaSnapshot[] = []; + let page = 1; + for (let fetched = 0; fetched < MAX_SNAPSHOT_LIST_PAGES; fetched++) { + const result = await daytona.snapshot.list(page, SNAPSHOT_LIST_PAGE_SIZE); + matching.push(...result.items.filter((item) => item.name.startsWith(SNAPSHOT_NAME_PREFIX))); + if (result.items.length === 0 || result.page >= result.totalPages) break; + page = result.page + 1; + } + return matching; + } + + private async withDeadline( + promise: Promise, + deadline: number, + timeoutMessage: string, + ): Promise { + let timer: NodeJS.Timeout | undefined; + try { + return await Promise.race([ + promise, + new Promise((_, reject) => { + timer = setTimeout( + () => reject(new Error(timeoutMessage)), + Math.max(deadline - Date.now(), 0), + ); + }), + ]); + } finally { + clearTimeout(timer); + // The losing promise keeps running; swallow its eventual rejection. + promise.catch(() => {}); + } + } + /** * Derive the versioned snapshot name for the running n8n version, or null * when no version is configured. Existence is validated implicitly by the