mirror of
https://github.com/n8n-io/n8n.git
synced 2026-08-29 01:39:24 +08:00
fix(core): Harden Instance AI sandbox acquisition (no-changelog) (#35314)
This commit is contained in:
@@ -12,6 +12,7 @@ interface MockSandbox {
|
||||
process: { executeCommand: Mock };
|
||||
fs: Record<string, Mock>;
|
||||
start: Mock;
|
||||
waitUntilStarted: Mock;
|
||||
stop: Mock;
|
||||
delete: Mock;
|
||||
getWorkDir: Mock;
|
||||
@@ -36,8 +37,10 @@ const {
|
||||
queuedCreateResults,
|
||||
makeMockSandbox,
|
||||
Daytona,
|
||||
DaytonaConnectionError,
|
||||
DaytonaError,
|
||||
DaytonaNotFoundError,
|
||||
DaytonaTimeoutError,
|
||||
resetDaytonaMockState,
|
||||
} = vi.hoisted(() => {
|
||||
const clientLog: DaytonaClientLog[] = [];
|
||||
@@ -71,6 +74,7 @@ const {
|
||||
moveFiles: vi.fn(),
|
||||
},
|
||||
start: vi.fn().mockResolvedValue(undefined),
|
||||
waitUntilStarted: vi.fn().mockResolvedValue(undefined),
|
||||
stop: vi.fn().mockResolvedValue(undefined),
|
||||
delete: vi.fn().mockResolvedValue(undefined),
|
||||
getWorkDir: vi.fn().mockResolvedValue('/home/daytona/workspace'),
|
||||
@@ -137,6 +141,8 @@ const {
|
||||
super(message, 404);
|
||||
}
|
||||
}
|
||||
class DaytonaConnectionError extends DaytonaError {}
|
||||
class DaytonaTimeoutError extends DaytonaError {}
|
||||
|
||||
function resetDaytonaMockState(): void {
|
||||
clientLog.length = 0;
|
||||
@@ -154,14 +160,22 @@ const {
|
||||
queuedCreateResults,
|
||||
makeMockSandbox,
|
||||
Daytona,
|
||||
DaytonaConnectionError,
|
||||
DaytonaError,
|
||||
DaytonaNotFoundError,
|
||||
DaytonaTimeoutError,
|
||||
resetDaytonaMockState,
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock('../../../workspace/sandbox/lazy-daytona', () => ({
|
||||
loadDaytona: () => ({ Daytona, DaytonaError, DaytonaNotFoundError }),
|
||||
loadDaytona: () => ({
|
||||
Daytona,
|
||||
DaytonaConnectionError,
|
||||
DaytonaError,
|
||||
DaytonaNotFoundError,
|
||||
DaytonaTimeoutError,
|
||||
}),
|
||||
}));
|
||||
|
||||
import { DaytonaFilesystem } from '../../../workspace/filesystem/daytona-filesystem';
|
||||
@@ -297,6 +311,313 @@ describe('DaytonaSandbox (creation strategies)', () => {
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('reattaches to the existing sandbox when creation reports a name conflict', async () => {
|
||||
const logger = makeLogger();
|
||||
const errorReporter: ErrorReporter = { error: vi.fn() };
|
||||
queueNotFound('not found');
|
||||
queuedCreateResults.push(
|
||||
new DaytonaError('Sandbox with name sandbox-name already exists', 409),
|
||||
);
|
||||
queuedGetResults.push(makeMockSandbox('remote-existing'));
|
||||
|
||||
const sandbox = new DaytonaSandbox({
|
||||
id: 'sandbox-id',
|
||||
name: 'sandbox-name',
|
||||
apiKey: 'api-key',
|
||||
snapshot: 'n8n/instance-ai:1.123.0',
|
||||
logger,
|
||||
errorReporter,
|
||||
});
|
||||
|
||||
await sandbox.start();
|
||||
|
||||
expect(clientLog[0].create).toHaveBeenCalledTimes(1);
|
||||
expect(sandbox.getInfo().metadata?.remoteSandboxId).toBe('remote-existing');
|
||||
expect(errorReporter.error).not.toHaveBeenCalled();
|
||||
expect(logger.info).toHaveBeenCalledWith(
|
||||
'Sandbox name already exists; reattached to existing sandbox',
|
||||
expect.objectContaining({ sandboxName: 'sandbox-name', remoteSandboxId: 'remote-existing' }),
|
||||
);
|
||||
});
|
||||
|
||||
it.each(['stopped', 'archived'])('resumes a %s sandbox after a name conflict', async (state) => {
|
||||
queueNotFound('not found');
|
||||
queuedCreateResults.push(new DaytonaError('Sandbox with name sandbox-name already exists'));
|
||||
const existing = makeMockSandbox(`remote-${state}`, state);
|
||||
queuedGetResults.push(existing);
|
||||
|
||||
const sandbox = new DaytonaSandbox({
|
||||
id: 'sandbox-id',
|
||||
name: 'sandbox-name',
|
||||
apiKey: 'api-key',
|
||||
snapshot: 'n8n/instance-ai:1.123.0',
|
||||
});
|
||||
|
||||
await sandbox.start();
|
||||
|
||||
expect(existing.start).toHaveBeenCalled();
|
||||
expect(existing.waitUntilStarted).not.toHaveBeenCalled();
|
||||
expect(sandbox.getInfo().metadata?.remoteSandboxId).toBe(`remote-${state}`);
|
||||
});
|
||||
|
||||
it.each(['creating', 'restoring', 'starting', 'pending_build', 'pulling_snapshot'])(
|
||||
'waits for a %s sandbox after a concurrent create conflict',
|
||||
async (state) => {
|
||||
queueNotFound('not found');
|
||||
queuedCreateResults.push(
|
||||
new DaytonaError('Sandbox with name sandbox-name already exists', 409),
|
||||
);
|
||||
const existing = makeMockSandbox(`remote-${state}`, state);
|
||||
queuedGetResults.push(existing);
|
||||
|
||||
const sandbox = new DaytonaSandbox({
|
||||
id: 'sandbox-id',
|
||||
name: 'sandbox-name',
|
||||
apiKey: 'api-key',
|
||||
snapshot: 'n8n/instance-ai:1.123.0',
|
||||
});
|
||||
|
||||
await sandbox.start();
|
||||
|
||||
expect(existing.waitUntilStarted).toHaveBeenCalled();
|
||||
expect(existing.start).not.toHaveBeenCalled();
|
||||
expect(sandbox.getInfo().metadata?.remoteSandboxId).toBe(`remote-${state}`);
|
||||
},
|
||||
);
|
||||
|
||||
it('keeps polling until a long-running stop can be resumed after a name conflict', async () => {
|
||||
queueNotFound('not found');
|
||||
queuedCreateResults.push(
|
||||
new DaytonaError('Sandbox with name sandbox-name already exists', 409),
|
||||
);
|
||||
const stopping = makeMockSandbox('remote-transitioning', 'stopping');
|
||||
const stopped = makeMockSandbox('remote-transitioning', 'stopped');
|
||||
queuedGetResults.push(stopping, stopping, stopping, stopping, stopping, stopped);
|
||||
|
||||
const sandbox = new DaytonaSandbox({
|
||||
id: 'sandbox-id',
|
||||
name: 'sandbox-name',
|
||||
apiKey: 'api-key',
|
||||
snapshot: 'n8n/instance-ai:1.123.0',
|
||||
createRetryBackoffBaseMs: 1,
|
||||
});
|
||||
|
||||
await sandbox.start();
|
||||
|
||||
expect(stopping.start).not.toHaveBeenCalled();
|
||||
expect(stopping.waitUntilStarted).not.toHaveBeenCalled();
|
||||
expect(stopped.start).toHaveBeenCalled();
|
||||
expect(clientLog[0].get).toHaveBeenCalledTimes(7);
|
||||
});
|
||||
|
||||
it('retries the lookup when a conflicted sandbox is not immediately visible', async () => {
|
||||
queueNotFound('not found');
|
||||
queuedCreateResults.push(
|
||||
new DaytonaError('Sandbox with name sandbox-name already exists', 409),
|
||||
new DaytonaError('Sandbox with name sandbox-name already exists', 409),
|
||||
);
|
||||
queueNotFound('not visible yet');
|
||||
queuedGetResults.push(makeMockSandbox('remote-eventually-visible'));
|
||||
|
||||
const sandbox = new DaytonaSandbox({
|
||||
id: 'sandbox-id',
|
||||
name: 'sandbox-name',
|
||||
apiKey: 'api-key',
|
||||
snapshot: 'n8n/instance-ai:1.123.0',
|
||||
createRetryBackoffBaseMs: 1,
|
||||
});
|
||||
|
||||
await sandbox.start();
|
||||
|
||||
expect(clientLog[0].get).toHaveBeenCalledTimes(3);
|
||||
expect(sandbox.getInfo().metadata?.remoteSandboxId).toBe('remote-eventually-visible');
|
||||
});
|
||||
|
||||
it.each([
|
||||
['server error', new DaytonaError('Bad Gateway', 502)],
|
||||
['rate limit', new DaytonaError('Too Many Requests', 429)],
|
||||
])('fails without creating when the initial lookup returns a %s', async (_kind, error) => {
|
||||
queuedGetErrors.push(error);
|
||||
|
||||
const sandbox = new DaytonaSandbox({
|
||||
id: 'sandbox-id',
|
||||
name: 'sandbox-name',
|
||||
apiKey: 'api-key',
|
||||
snapshot: 'n8n/instance-ai:1.123.0',
|
||||
});
|
||||
|
||||
await expect(sandbox.start()).rejects.toThrow(error.message);
|
||||
expect(clientLog[0].get).toHaveBeenCalledTimes(1);
|
||||
expect(clientLog[0].create).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('fails without polling when the lookup after a name conflict errors', async () => {
|
||||
queueNotFound('not found');
|
||||
queuedCreateResults.push(
|
||||
new DaytonaError('Sandbox with name sandbox-name already exists', 409),
|
||||
);
|
||||
queuedGetErrors.push(new DaytonaError('Bad Gateway', 502));
|
||||
|
||||
const sandbox = new DaytonaSandbox({
|
||||
id: 'sandbox-id',
|
||||
name: 'sandbox-name',
|
||||
apiKey: 'api-key',
|
||||
snapshot: 'n8n/instance-ai:1.123.0',
|
||||
});
|
||||
|
||||
await expect(sandbox.start()).rejects.toThrow('Bad Gateway');
|
||||
expect(clientLog[0].get).toHaveBeenCalledTimes(2);
|
||||
expect(clientLog[0].create).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('deletes a dead sandbox before creating a replacement', async () => {
|
||||
const dead = makeMockSandbox('remote-dead', 'error');
|
||||
queuedGetResults.push(dead);
|
||||
queuedCreateResults.push(makeMockSandbox('remote-replacement'));
|
||||
|
||||
const sandbox = new DaytonaSandbox({
|
||||
id: 'sandbox-id',
|
||||
name: 'sandbox-name',
|
||||
apiKey: 'api-key',
|
||||
snapshot: 'n8n/instance-ai:1.123.0',
|
||||
});
|
||||
|
||||
await sandbox.start();
|
||||
|
||||
expect(dead.delete).toHaveBeenCalled();
|
||||
expect(sandbox.getInfo().metadata?.remoteSandboxId).toBe('remote-replacement');
|
||||
});
|
||||
|
||||
it('replaces a failed sandbox discovered after a name conflict', async () => {
|
||||
queueNotFound('not found');
|
||||
queuedCreateResults.push(
|
||||
new DaytonaError('Sandbox with name sandbox-name already exists', 409),
|
||||
makeMockSandbox('remote-replacement'),
|
||||
);
|
||||
const failed = makeMockSandbox('remote-failed', 'build_failed');
|
||||
failed.delete.mockImplementation(async () => {
|
||||
await Promise.resolve();
|
||||
queuedGetErrors.push(new DaytonaNotFoundError('deleted'));
|
||||
});
|
||||
queuedGetResults.push(failed);
|
||||
|
||||
const sandbox = new DaytonaSandbox({
|
||||
id: 'sandbox-id',
|
||||
name: 'sandbox-name',
|
||||
apiKey: 'api-key',
|
||||
snapshot: 'n8n/instance-ai:1.123.0',
|
||||
createRetryBackoffBaseMs: 1,
|
||||
});
|
||||
|
||||
await sandbox.start();
|
||||
|
||||
expect(failed.delete).toHaveBeenCalled();
|
||||
expect(clientLog[0].create).toHaveBeenCalledTimes(2);
|
||||
expect(sandbox.getInfo().metadata?.remoteSandboxId).toBe('remote-replacement');
|
||||
});
|
||||
|
||||
it.each([
|
||||
['server', new DaytonaError('Bad Gateway', 502)],
|
||||
['rate-limit', new DaytonaError('Too Many Requests', 429)],
|
||||
])('does not retry a %s create failure', async (_kind, error) => {
|
||||
queueNotFound('not found');
|
||||
queuedCreateResults.push(error);
|
||||
|
||||
const sandbox = new DaytonaSandbox({
|
||||
id: 'sandbox-id',
|
||||
name: 'sandbox-name',
|
||||
apiKey: 'api-key',
|
||||
snapshot: 'n8n/instance-ai:1.123.0',
|
||||
});
|
||||
|
||||
await expect(sandbox.start()).rejects.toThrow(error.message);
|
||||
expect(clientLog[0].create).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it.each([
|
||||
['connection', DaytonaConnectionError],
|
||||
['timeout', DaytonaTimeoutError],
|
||||
])('does not retry a transient %s create failure', async (_kind, ErrorType) => {
|
||||
queueNotFound('not found');
|
||||
queuedCreateResults.push(new ErrorType('transient create failure'));
|
||||
|
||||
const sandbox = new DaytonaSandbox({
|
||||
id: 'sandbox-id',
|
||||
name: 'sandbox-name',
|
||||
apiKey: 'api-key',
|
||||
snapshot: 'n8n/instance-ai:1.123.0',
|
||||
});
|
||||
|
||||
await expect(sandbox.start()).rejects.toThrow('transient create failure');
|
||||
expect(clientLog[0].create).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('does not retry non-transient create failures', async () => {
|
||||
queueNotFound('not found');
|
||||
queuedCreateResults.push(new DaytonaError('Forbidden', 403));
|
||||
|
||||
const sandbox = new DaytonaSandbox({
|
||||
id: 'sandbox-id',
|
||||
name: 'sandbox-name',
|
||||
apiKey: 'api-key',
|
||||
snapshot: 'n8n/instance-ai:1.123.0',
|
||||
createRetryBackoffBaseMs: 1,
|
||||
});
|
||||
|
||||
await expect(sandbox.start()).rejects.toThrow('Forbidden');
|
||||
expect(clientLog[0].create).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('does not treat an unrelated already-exists message as a sandbox name conflict', async () => {
|
||||
const errorReporter: ErrorReporter = { error: vi.fn() };
|
||||
const snapshotError = new DaytonaError('Snapshot build failed: file already exists');
|
||||
queueNotFound('not found');
|
||||
queuedCreateResults.push(snapshotError, makeMockSandbox('remote-from-image'));
|
||||
|
||||
const sandbox = new DaytonaSandbox({
|
||||
id: 'sandbox-id',
|
||||
name: 'sandbox-name',
|
||||
apiKey: 'api-key',
|
||||
snapshot: 'n8n/instance-ai:1.123.0',
|
||||
image: 'node:20',
|
||||
errorReporter,
|
||||
});
|
||||
|
||||
await sandbox.start();
|
||||
|
||||
expect(clientLog[0].create).toHaveBeenCalledTimes(2);
|
||||
expect(sandbox.getInfo().metadata?.remoteSandboxId).toBe('remote-from-image');
|
||||
expect(errorReporter.error).toHaveBeenCalledWith(
|
||||
snapshotError,
|
||||
expect.objectContaining({
|
||||
tags: expect.objectContaining({ strategy: 'snapshot' }),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('retries creation when a conflicted sandbox disappears before it becomes visible', async () => {
|
||||
queueNotFound('not found');
|
||||
queuedCreateResults.push(
|
||||
new DaytonaError('Sandbox with name sandbox-name already exists', 409),
|
||||
makeMockSandbox('remote-replacement'),
|
||||
);
|
||||
queueNotFound('gone before reattach');
|
||||
|
||||
const sandbox = new DaytonaSandbox({
|
||||
id: 'sandbox-id',
|
||||
name: 'sandbox-name',
|
||||
apiKey: 'api-key',
|
||||
snapshot: 'n8n/instance-ai:1.123.0',
|
||||
image: 'node:20',
|
||||
createRetryBackoffBaseMs: 1,
|
||||
});
|
||||
|
||||
await sandbox.start();
|
||||
|
||||
expect(clientLog[0].create).toHaveBeenCalledTimes(2);
|
||||
expect(sandbox.getInfo().metadata?.remoteSandboxId).toBe('remote-replacement');
|
||||
});
|
||||
});
|
||||
|
||||
describe('DaytonaSandbox (direct mode)', () => {
|
||||
|
||||
@@ -26,10 +26,22 @@ import type { ErrorReporter, Logger } from './logger';
|
||||
const SANDBOX_STATE_STARTED = 'started';
|
||||
const SANDBOX_STATE_STOPPED = 'stopped';
|
||||
const SANDBOX_STATE_ARCHIVED = 'archived';
|
||||
const SANDBOX_STATE_CREATING = 'creating';
|
||||
const SANDBOX_STATE_RESTORING = 'restoring';
|
||||
const SANDBOX_STATE_STARTING = 'starting';
|
||||
const SANDBOX_STATE_PENDING_BUILD = 'pending_build';
|
||||
const SANDBOX_STATE_PULLING_SNAPSHOT = 'pulling_snapshot';
|
||||
const SANDBOX_STATE_FORKING = 'forking';
|
||||
const SANDBOX_STATE_RESIZING = 'resizing';
|
||||
const SANDBOX_STATE_SNAPSHOTTING = 'snapshotting';
|
||||
const SANDBOX_STATE_BUILDING_SNAPSHOT = 'building_snapshot';
|
||||
const SANDBOX_STATE_STOPPING = 'stopping';
|
||||
const SANDBOX_STATE_ARCHIVING = 'archiving';
|
||||
const SANDBOX_STATE_DESTROYED = 'destroyed';
|
||||
const SANDBOX_STATE_DESTROYING = 'destroying';
|
||||
const SANDBOX_STATE_ERROR = 'error';
|
||||
const SANDBOX_STATE_BUILD_FAILED = 'build_failed';
|
||||
const MAX_ACQUISITION_RETRY_BACKOFF_MS = 5_000;
|
||||
|
||||
/**
|
||||
* States a failed operation may recover from by resuming the sandbox: an idle sandbox that
|
||||
@@ -41,11 +53,42 @@ const SANDBOX_STATE_BUILD_FAILED = 'build_failed';
|
||||
* we don't want to trigger off an unrelated operation failure.
|
||||
* Deletion is handled separately as a `DaytonaNotFoundError` fast-path.
|
||||
*/
|
||||
const RECOVERABLE_SANDBOX_STATES: ReadonlySet<string> = new Set([
|
||||
const RECOVERABLE_SANDBOX_STATES = new Set<SandboxState>([
|
||||
SANDBOX_STATE_STOPPED,
|
||||
SANDBOX_STATE_ARCHIVED,
|
||||
]);
|
||||
|
||||
const WAIT_FOR_STARTED_SANDBOX_STATES = new Set<SandboxState>([
|
||||
SANDBOX_STATE_CREATING,
|
||||
SANDBOX_STATE_RESTORING,
|
||||
SANDBOX_STATE_STARTING,
|
||||
SANDBOX_STATE_PENDING_BUILD,
|
||||
SANDBOX_STATE_PULLING_SNAPSHOT,
|
||||
SANDBOX_STATE_FORKING,
|
||||
SANDBOX_STATE_RESIZING,
|
||||
SANDBOX_STATE_SNAPSHOTTING,
|
||||
SANDBOX_STATE_BUILDING_SNAPSHOT,
|
||||
]);
|
||||
|
||||
const WAIT_FOR_RECOVERABLE_SANDBOX_STATES = new Set<SandboxState>([
|
||||
SANDBOX_STATE_STOPPING,
|
||||
SANDBOX_STATE_ARCHIVING,
|
||||
]);
|
||||
|
||||
const FAILED_SANDBOX_STATES = new Set<SandboxState>([
|
||||
SANDBOX_STATE_ERROR,
|
||||
SANDBOX_STATE_BUILD_FAILED,
|
||||
]);
|
||||
|
||||
const REMOVING_SANDBOX_STATES = new Set<SandboxState>([
|
||||
SANDBOX_STATE_DESTROYED,
|
||||
SANDBOX_STATE_DESTROYING,
|
||||
]);
|
||||
|
||||
type ExistingSandboxLookup =
|
||||
| { status: 'ready'; sandbox: Sandbox }
|
||||
| { status: 'absent' | 'pending' };
|
||||
|
||||
export interface DaytonaSandboxOptions {
|
||||
id?: string;
|
||||
/** Static Daytona API key (direct mode). Mutually exclusive with `getAuthToken`. */
|
||||
@@ -67,6 +110,8 @@ export interface DaytonaSandboxOptions {
|
||||
target?: string;
|
||||
timeout?: number;
|
||||
createTimeoutSeconds?: number;
|
||||
/** Base backoff for sandbox acquisition retries. Defaults to 1s. */
|
||||
createRetryBackoffBaseMs?: number;
|
||||
language?: 'typescript' | 'javascript' | 'python';
|
||||
resources?: Resources;
|
||||
env?: Record<string, string>;
|
||||
@@ -96,24 +141,18 @@ function toShellCommand(command: string, args: string[]): string {
|
||||
return [command, ...args.map((arg) => shellEscape(arg))].join(' ');
|
||||
}
|
||||
|
||||
function isDaytonaAuthError(error: unknown): boolean {
|
||||
const { DaytonaError } = loadDaytona();
|
||||
return error instanceof DaytonaError && (error.statusCode === 401 || error.statusCode === 403);
|
||||
}
|
||||
|
||||
function isSandboxGone(error: unknown): boolean {
|
||||
const { DaytonaNotFoundError } = loadDaytona();
|
||||
return error instanceof DaytonaNotFoundError;
|
||||
}
|
||||
|
||||
function isSandboxNameConflictError(error: unknown): boolean {
|
||||
if (!(error instanceof Error)) return false;
|
||||
const { DaytonaError } = loadDaytona();
|
||||
if (error instanceof DaytonaError && error.statusCode === 409) return true;
|
||||
return /sandbox with name .+ already exists/i.test(error.message);
|
||||
}
|
||||
export class DaytonaSandbox extends BaseSandbox {
|
||||
private static readonly DEAD_STATES: ReadonlySet<SandboxState> = new Set([
|
||||
SANDBOX_STATE_DESTROYED,
|
||||
SANDBOX_STATE_DESTROYING,
|
||||
SANDBOX_STATE_ERROR,
|
||||
SANDBOX_STATE_BUILD_FAILED,
|
||||
]) as ReadonlySet<SandboxState>;
|
||||
|
||||
readonly id: string;
|
||||
readonly name = 'DaytonaSandbox';
|
||||
readonly provider = 'daytona';
|
||||
@@ -164,10 +203,47 @@ export class DaytonaSandbox extends BaseSandbox {
|
||||
return;
|
||||
}
|
||||
|
||||
this.sandbox = await this.createSandbox(client);
|
||||
this.sandbox = await this.createSandboxOrReattach(client);
|
||||
await this.detectWorkingDirectory();
|
||||
}
|
||||
|
||||
/**
|
||||
* Create the remote sandbox, reattaching by name on a name conflict — the sandbox
|
||||
* exists even though the initial lookup missed it due to a concurrent create from
|
||||
* another main. Deterministic names make reattach safe.
|
||||
*/
|
||||
private async createSandboxOrReattach(client: Daytona): Promise<Sandbox> {
|
||||
let conflictDeadline: number | undefined;
|
||||
let conflictError: unknown;
|
||||
let attempt = 0;
|
||||
|
||||
while (conflictDeadline === undefined || Date.now() < conflictDeadline) {
|
||||
try {
|
||||
return await this.createSandbox(client);
|
||||
} catch (error) {
|
||||
if (!isSandboxNameConflictError(error)) throw error;
|
||||
conflictError = error;
|
||||
conflictDeadline ??= Date.now() + this.timeout;
|
||||
|
||||
const existing = await this.findExistingSandboxAfterConflict(client, conflictDeadline);
|
||||
if (existing) {
|
||||
this.options.logger?.info('Sandbox name already exists; reattached to existing sandbox', {
|
||||
sandboxName: this.sandboxName,
|
||||
remoteSandboxId: existing.id,
|
||||
});
|
||||
return existing;
|
||||
}
|
||||
|
||||
if (Date.now() >= conflictDeadline) break;
|
||||
await this.waitBeforeAcquisitionRetry(attempt++, conflictDeadline);
|
||||
}
|
||||
}
|
||||
|
||||
throw conflictError instanceof Error
|
||||
? conflictError
|
||||
: new Error('Failed to reconcile Daytona sandbox name conflict');
|
||||
}
|
||||
|
||||
override async stop(): Promise<void> {
|
||||
if (!this.sandbox) return;
|
||||
try {
|
||||
@@ -386,24 +462,54 @@ export class DaytonaSandbox extends BaseSandbox {
|
||||
}
|
||||
|
||||
private async findExistingSandbox(client: Daytona): Promise<Sandbox | null> {
|
||||
const result = await this.lookupExistingSandbox(client);
|
||||
return result.status === 'ready' ? result.sandbox : null;
|
||||
}
|
||||
|
||||
private async lookupExistingSandbox(
|
||||
client: Daytona,
|
||||
deadline?: number,
|
||||
): Promise<ExistingSandboxLookup> {
|
||||
try {
|
||||
const sandbox = await client.get(this.sandboxName);
|
||||
if (sandbox.state && this.isDeadState(sandbox.state)) {
|
||||
await sandbox.delete(Math.ceil(this.timeout / 1000));
|
||||
return null;
|
||||
const state = sandbox.state;
|
||||
if (state === undefined) return { status: 'pending' };
|
||||
if (FAILED_SANDBOX_STATES.has(state)) {
|
||||
await sandbox.delete(this.operationTimeoutSeconds(deadline));
|
||||
return { status: 'pending' };
|
||||
}
|
||||
if (sandbox.state !== SANDBOX_STATE_STARTED) {
|
||||
await sandbox.start(Math.ceil(this.timeout / 1000));
|
||||
if (REMOVING_SANDBOX_STATES.has(state)) return { status: 'pending' };
|
||||
if (RECOVERABLE_SANDBOX_STATES.has(state)) {
|
||||
await sandbox.start(this.operationTimeoutSeconds(deadline));
|
||||
} else if (WAIT_FOR_STARTED_SANDBOX_STATES.has(state)) {
|
||||
await sandbox.waitUntilStarted(this.operationTimeoutSeconds(deadline));
|
||||
} else if (
|
||||
WAIT_FOR_RECOVERABLE_SANDBOX_STATES.has(state) ||
|
||||
state !== SANDBOX_STATE_STARTED
|
||||
) {
|
||||
return { status: 'pending' };
|
||||
}
|
||||
return sandbox;
|
||||
return { status: 'ready', sandbox };
|
||||
} catch (error) {
|
||||
const { DaytonaNotFoundError } = loadDaytona();
|
||||
if (error instanceof DaytonaNotFoundError) return null;
|
||||
if (isDaytonaAuthError(error)) throw error;
|
||||
return null;
|
||||
if (error instanceof DaytonaNotFoundError) return { status: 'absent' };
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private async findExistingSandboxAfterConflict(
|
||||
client: Daytona,
|
||||
deadline: number,
|
||||
): Promise<Sandbox | null> {
|
||||
for (let attempt = 0; Date.now() < deadline; attempt++) {
|
||||
const result = await this.lookupExistingSandbox(client, deadline);
|
||||
if (result.status === 'ready') return result.sandbox;
|
||||
if (result.status === 'absent') return null;
|
||||
await this.waitBeforeAcquisitionRetry(attempt, deadline);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private async createSandbox(client: Daytona): Promise<Sandbox> {
|
||||
const candidates = this.createSandboxParams();
|
||||
let lastError: unknown;
|
||||
@@ -414,6 +520,8 @@ export class DaytonaSandbox extends BaseSandbox {
|
||||
? await client.create(candidate.params, { timeout: this.options.createTimeoutSeconds })
|
||||
: await client.create(candidate.params);
|
||||
} catch (error) {
|
||||
// A name conflict is strategy-independent; let the caller reattach by name.
|
||||
if (isSandboxNameConflictError(error)) throw error;
|
||||
lastError = error;
|
||||
this.reportCreateError(error, candidate.strategy);
|
||||
if (
|
||||
@@ -434,6 +542,21 @@ export class DaytonaSandbox extends BaseSandbox {
|
||||
throw lastError instanceof Error ? lastError : new Error('Failed to create Daytona sandbox');
|
||||
}
|
||||
|
||||
private async waitBeforeAcquisitionRetry(attempt: number, deadline: number): Promise<void> {
|
||||
const baseDelayMs = this.options.createRetryBackoffBaseMs ?? 1_000;
|
||||
const delayMs = Math.min(
|
||||
baseDelayMs * 2 ** attempt,
|
||||
MAX_ACQUISITION_RETRY_BACKOFF_MS,
|
||||
Math.max(0, deadline - Date.now()),
|
||||
);
|
||||
if (delayMs > 0) await new Promise((resolve) => setTimeout(resolve, delayMs));
|
||||
}
|
||||
|
||||
private operationTimeoutSeconds(deadline?: number): number {
|
||||
if (deadline === undefined) return Math.ceil(this.timeout / 1000);
|
||||
return Math.max(0.001, (deadline - Date.now()) / 1000);
|
||||
}
|
||||
|
||||
private createSandboxParams(): Array<{
|
||||
strategy: 'snapshot' | 'image';
|
||||
params: CreateSandboxFromImageParams | CreateSandboxFromSnapshotParams;
|
||||
@@ -516,10 +639,6 @@ export class DaytonaSandbox extends BaseSandbox {
|
||||
}
|
||||
}
|
||||
|
||||
private isDeadState(state: SandboxState): boolean {
|
||||
return DaytonaSandbox.DEAD_STATES.has(state);
|
||||
}
|
||||
|
||||
private compactEnv(env: NodeJS.ProcessEnv | undefined): Record<string, string> | undefined {
|
||||
const merged = {
|
||||
...this.options.env,
|
||||
|
||||
@@ -2,6 +2,7 @@ import type { Mock } from 'vitest';
|
||||
import type { InstanceAiConfig } from '@n8n/config';
|
||||
import type { User } from '@n8n/db';
|
||||
import type { ErrorReporter } from 'n8n-core';
|
||||
import { OperationalError } from 'n8n-workflow';
|
||||
|
||||
vi.mock('@n8n/instance-ai', () => ({
|
||||
createSandbox: vi.fn(),
|
||||
@@ -49,7 +50,7 @@ function createSandboxService(overrides: Overrides = {}) {
|
||||
...overrides.backgroundTasks,
|
||||
};
|
||||
const settingsService: InstanceAiSandboxSettings = {
|
||||
resolveDaytonaConfig: vi.fn(async () => ({})),
|
||||
resolveDaytonaConfig: vi.fn(async () => ({ apiKey: 'test-daytona-key' })),
|
||||
resolveN8nSandboxConfig: vi.fn(async () => ({})),
|
||||
...overrides.settingsService,
|
||||
};
|
||||
@@ -112,6 +113,22 @@ describe('InstanceAiSandboxService', () => {
|
||||
});
|
||||
});
|
||||
|
||||
it('fails with a setup error in direct mode when no Daytona API key is configured', async () => {
|
||||
const resolveDaytonaConfig = vi.fn(async () => ({}));
|
||||
const { service } = createSandboxService({
|
||||
config: { sandboxEnabled: true, sandboxProvider: 'daytona' },
|
||||
settingsService: { resolveDaytonaConfig },
|
||||
aiService: { isProxyEnabled: vi.fn(() => false) },
|
||||
});
|
||||
|
||||
const resolution = service.resolveSandboxConfig(fakeUser);
|
||||
await expect(resolution).rejects.toBeInstanceOf(OperationalError);
|
||||
await expect(resolution).rejects.toMatchObject({
|
||||
message: expect.stringContaining('no API key is configured'),
|
||||
shouldReport: false,
|
||||
});
|
||||
});
|
||||
|
||||
it('routes Daytona traffic through the assistant proxy when enabled', async () => {
|
||||
const getInstanceAiApiProxyToken = vi.fn(async () => ({ accessToken: 'token-1' }));
|
||||
const client = {
|
||||
|
||||
@@ -985,7 +985,7 @@ describe('InstanceAiService — runtime workspace setup', () => {
|
||||
},
|
||||
backgroundTasks: { getRunningTasks: vi.fn(() => []) },
|
||||
settingsService: {
|
||||
resolveDaytonaConfig: vi.fn(async () => ({})),
|
||||
resolveDaytonaConfig: vi.fn(async () => ({ apiKey: 'test-daytona-key' })),
|
||||
resolveN8nSandboxConfig: vi.fn(async () => ({})),
|
||||
},
|
||||
aiService: { isProxyEnabled: vi.fn(() => false), getClient: vi.fn() },
|
||||
|
||||
@@ -12,7 +12,7 @@ import {
|
||||
type SandboxConfig,
|
||||
} from '@n8n/instance-ai';
|
||||
import type { ErrorReporter } from 'n8n-core';
|
||||
import { UnexpectedError } from 'n8n-workflow';
|
||||
import { OperationalError, UnexpectedError } from 'n8n-workflow';
|
||||
import { nanoid } from 'nanoid';
|
||||
|
||||
import { N8N_VERSION } from '@/constants';
|
||||
@@ -274,10 +274,16 @@ export class InstanceAiSandboxService {
|
||||
|
||||
// Direct mode: Daytona credentials from env vars or admin credential
|
||||
const daytona = await this.options.settingsService.resolveDaytonaConfig();
|
||||
const daytonaApiKey = daytona.apiKey ?? base.daytonaApiKey;
|
||||
if (!daytonaApiKey) {
|
||||
throw new OperationalError(
|
||||
'The Daytona sandbox is enabled in direct mode but no API key is configured. Set the Daytona API key environment variable or connect the Daytona credential.',
|
||||
);
|
||||
}
|
||||
return {
|
||||
...base,
|
||||
daytonaApiUrl: daytona.apiUrl ?? base.daytonaApiUrl,
|
||||
daytonaApiKey: daytona.apiKey ?? base.daytonaApiKey,
|
||||
daytonaApiKey,
|
||||
};
|
||||
}
|
||||
const sandbox = await this.options.settingsService.resolveN8nSandboxConfig();
|
||||
|
||||
Reference in New Issue
Block a user