mirror of
https://github.com/simstudioai/sim.git
synced 2026-08-31 01:11:53 +08:00
fix(billing): close sandbox admission review gaps
This commit is contained in:
@@ -21,7 +21,7 @@ vi.mock('@/lib/billing/threshold-billing', () => ({
|
||||
}))
|
||||
vi.mock('@/lib/core/config/env-flags', () => ({
|
||||
getCostMultiplier: vi.fn(() => 1),
|
||||
isBillingEnabled: false,
|
||||
isBillingEnabled: true,
|
||||
}))
|
||||
vi.mock('@/lib/execution/remote-sandbox/provider', () => ({
|
||||
getSandboxProvider: vi.fn(() => ({ terminateById: mockTerminateById })),
|
||||
@@ -141,6 +141,16 @@ describe('sandbox usage outbox finalizer', () => {
|
||||
text: 'GREATEST(COALESCE(, 0) + ::numeric, ::numeric)',
|
||||
params: ['workflowExecutionLogs.costTotal', '0.00046', '0.02'],
|
||||
})
|
||||
expect(mockThresholdBilling).toHaveBeenCalledWith(
|
||||
{ type: 'user', id: 'user-1' },
|
||||
{
|
||||
onError: 'throw',
|
||||
expectedBillingPeriod: {
|
||||
start: new Date('2026-08-01T00:00:00.000Z'),
|
||||
end: new Date('2026-09-01T00:00:00.000Z'),
|
||||
},
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
it('terminates and checkpoints a sandbox whose terminal timestamp is missing', async () => {
|
||||
|
||||
@@ -333,7 +333,10 @@ async function finalizeSandboxUsage(
|
||||
})
|
||||
|
||||
if (isBillingEnabled) {
|
||||
await checkAndBillPayerOverageThreshold(billingContext.billingEntity, { onError: 'throw' })
|
||||
await checkAndBillPayerOverageThreshold(billingContext.billingEntity, {
|
||||
onError: 'throw',
|
||||
expectedBillingPeriod: billingContext.billingPeriod,
|
||||
})
|
||||
}
|
||||
|
||||
logger.info('Recorded Function sandbox usage', {
|
||||
|
||||
@@ -163,6 +163,24 @@ function deferred<T>() {
|
||||
return { promise, resolve, reject }
|
||||
}
|
||||
|
||||
async function deferProviderCreate(provider: Provider): Promise<{ resolve: () => void }> {
|
||||
if (provider === 'e2b') {
|
||||
const implementation = mockE2BCreate.getMockImplementation()
|
||||
if (!implementation) throw new Error('E2B create mock is not configured')
|
||||
const created = await implementation()
|
||||
const pending = deferred<typeof created>()
|
||||
mockE2BCreate.mockReturnValueOnce(pending.promise)
|
||||
return { resolve: () => pending.resolve(created) }
|
||||
}
|
||||
|
||||
const implementation = mockDaytonaCreate.getMockImplementation()
|
||||
if (!implementation) throw new Error('Daytona create mock is not configured')
|
||||
const created = await implementation()
|
||||
const pending = deferred<typeof created>()
|
||||
mockDaytonaCreate.mockReturnValueOnce(pending.promise)
|
||||
return { resolve: () => pending.resolve(created) }
|
||||
}
|
||||
|
||||
const usageContext: SandboxUsageContext = {
|
||||
workspaceId: 'ws-1',
|
||||
workflowId: 'wf-1',
|
||||
@@ -471,6 +489,87 @@ describe.each(PROVIDERS)('sandbox conformance [%s]', (provider) => {
|
||||
).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('preserves usage persistence failure when cancellation races provider creation', async () => {
|
||||
const controller = new AbortController()
|
||||
const providerCreate = await deferProviderCreate(provider)
|
||||
mockBeginSandboxUsage.mockRejectedValueOnce(new Error('database unavailable'))
|
||||
|
||||
const execution = executeInSandbox({
|
||||
code: 'x',
|
||||
language: CodeLanguage.Python,
|
||||
timeoutMs: 1000,
|
||||
signal: controller.signal,
|
||||
usageContext,
|
||||
})
|
||||
const rejection = expect(execution).rejects.toMatchObject({
|
||||
name: 'SandboxUsagePersistenceError',
|
||||
retryable: false,
|
||||
})
|
||||
|
||||
await vi.waitFor(() =>
|
||||
expect(provider === 'e2b' ? mockE2BCreate : mockDaytonaCreate).toHaveBeenCalledOnce()
|
||||
)
|
||||
controller.abort(new DOMException('cancelled', 'AbortError'))
|
||||
expect(mockBeginSandboxUsage).not.toHaveBeenCalled()
|
||||
providerCreate.resolve()
|
||||
|
||||
await rejection
|
||||
expect(mockBeginSandboxUsage).toHaveBeenCalledTimes(2)
|
||||
expect(
|
||||
provider === 'e2b' ? mockE2BCommandsRun : mockExecuteSessionCommand
|
||||
).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('returns cancellation after provider creation, usage admission, and cleanup settle', async () => {
|
||||
const controller = new AbortController()
|
||||
const providerCreate = await deferProviderCreate(provider)
|
||||
|
||||
const execution = executeInSandbox({
|
||||
code: 'x',
|
||||
language: CodeLanguage.Python,
|
||||
timeoutMs: 1000,
|
||||
signal: controller.signal,
|
||||
usageContext,
|
||||
})
|
||||
const rejection = expect(execution).rejects.toMatchObject({ name: 'AbortError' })
|
||||
|
||||
await vi.waitFor(() =>
|
||||
expect(provider === 'e2b' ? mockE2BCreate : mockDaytonaCreate).toHaveBeenCalledOnce()
|
||||
)
|
||||
controller.abort(new DOMException('cancelled', 'AbortError'))
|
||||
expect(mockBeginSandboxUsage).not.toHaveBeenCalled()
|
||||
providerCreate.resolve()
|
||||
|
||||
await rejection
|
||||
expect(mockBeginSandboxUsage).toHaveBeenCalledOnce()
|
||||
expect(mockReleaseAndProcessSandboxUsage).toHaveBeenCalledWith(
|
||||
'sandbox-usage-event',
|
||||
expect.objectContaining({ outcome: 'cancelled', cleanupStatus: 'terminated' })
|
||||
)
|
||||
expect(
|
||||
provider === 'e2b' ? mockE2BCommandsRun : mockExecuteSessionCommand
|
||||
).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it.each([1000, 120_000])(
|
||||
'bounds provider creation by the remaining execution budget (%sms)',
|
||||
async (timeoutMs) => {
|
||||
await executeInSandbox({
|
||||
code: 'x',
|
||||
language: CodeLanguage.Python,
|
||||
timeoutMs,
|
||||
usageContext,
|
||||
})
|
||||
|
||||
const requestTimeoutMs =
|
||||
provider === 'e2b'
|
||||
? mockE2BCreate.mock.calls[0]?.[1]?.requestTimeoutMs
|
||||
: mockDaytonaCreate.mock.calls[0]?.[1]?.timeout * 1000
|
||||
expect(requestTimeoutMs).toBeGreaterThan(0)
|
||||
expect(requestTimeoutMs).toBeLessThanOrEqual(Math.min(timeoutMs, 60_000))
|
||||
}
|
||||
)
|
||||
|
||||
it('durably releases cleanup when persistence and live teardown initially fail', async () => {
|
||||
mockBeginSandboxUsage.mockRejectedValueOnce(new Error('database unavailable'))
|
||||
const teardown = provider === 'e2b' ? mockE2BKill : mockDelete
|
||||
|
||||
@@ -800,7 +800,12 @@ export const daytonaProvider: SandboxProvider = {
|
||||
ephemeral: true,
|
||||
ttlMinutes: resolveDaytonaLifetimeMs(options?.lifetimeMs) / 60_000,
|
||||
}
|
||||
const sandbox = await daytona.create(createOptions)
|
||||
const sandbox =
|
||||
options?.requestTimeoutMs === undefined
|
||||
? await daytona.create(createOptions)
|
||||
: await daytona.create(createOptions, {
|
||||
timeout: Math.max(0.001, options.requestTimeoutMs / 1000),
|
||||
})
|
||||
|
||||
return new DaytonaSandboxHandle(sandbox, language)
|
||||
},
|
||||
|
||||
@@ -867,6 +867,9 @@ export const e2bProvider: SandboxProvider = {
|
||||
const createOptions = {
|
||||
apiKey,
|
||||
timeoutMs: effectiveLifetimeMs,
|
||||
...(options?.requestTimeoutMs !== undefined
|
||||
? { requestTimeoutMs: options.requestTimeoutMs }
|
||||
: {}),
|
||||
}
|
||||
|
||||
const { Sandbox } = await import('@e2b/code-interpreter')
|
||||
|
||||
@@ -55,6 +55,7 @@ export type {
|
||||
} from '@/lib/execution/remote-sandbox/types'
|
||||
|
||||
const logger = createLogger('RemoteSandbox')
|
||||
const MAX_SANDBOX_CREATE_REQUEST_TIMEOUT_MS = 60_000
|
||||
|
||||
interface CreatedSandbox {
|
||||
handle: SandboxHandle
|
||||
@@ -71,7 +72,7 @@ interface SandboxCleanupResult {
|
||||
}
|
||||
|
||||
interface SandboxAbortRaceState {
|
||||
usageAdmission?: Promise<string | undefined>
|
||||
usageAdmission?: Promise<unknown>
|
||||
}
|
||||
|
||||
async function createSandbox(
|
||||
@@ -156,21 +157,18 @@ function raceSandboxAbort<T>(
|
||||
return new Promise<T>((resolve, reject) => {
|
||||
const onAbort = () => {
|
||||
signal.removeEventListener('abort', onAbort)
|
||||
const usageAdmission = state.usageAdmission
|
||||
if (!usageAdmission) {
|
||||
if (!state.usageAdmission) {
|
||||
reject(abortReason(signal))
|
||||
return
|
||||
}
|
||||
|
||||
/**
|
||||
* A created metered sandbox must finish its durable usage admission before
|
||||
* cancellation can decide the response. Success preserves the abort;
|
||||
* failure surfaces the non-retryable persistence error after cleanup.
|
||||
* A metered provider create must finish usage admission and cleanup before
|
||||
* cancellation can decide the response. The operation observer below
|
||||
* preserves the abort after successful admission or propagates a
|
||||
* non-retryable persistence failure.
|
||||
*/
|
||||
void usageAdmission.then(
|
||||
() => reject(abortReason(signal)),
|
||||
(error) => reject(error)
|
||||
)
|
||||
return
|
||||
}
|
||||
signal.addEventListener('abort', onAbort, { once: true })
|
||||
operation.then(
|
||||
@@ -339,14 +337,25 @@ async function beginUsageOrStop(
|
||||
}
|
||||
}
|
||||
|
||||
async function admitSandboxUsage(
|
||||
created: CreatedSandbox,
|
||||
async function createAndAdmitSandbox(
|
||||
kind: SandboxKind,
|
||||
createOptions: CreateSandboxOptions,
|
||||
selected: ResolvedSandbox | null,
|
||||
signal: AbortSignal,
|
||||
sandboxKind: 'code' | 'shell',
|
||||
usageContext: SandboxUsageContext | undefined,
|
||||
abortBinding: ReturnType<typeof bindSandboxAbort>,
|
||||
state: SandboxAbortRaceState
|
||||
): Promise<string | undefined> {
|
||||
const admission = beginUsageOrStop(created, sandboxKind, usageContext, abortBinding)
|
||||
): Promise<{
|
||||
created: CreatedSandbox
|
||||
abortBinding: ReturnType<typeof bindSandboxAbort>
|
||||
usageEventId: string | undefined
|
||||
}> {
|
||||
const admission = (async () => {
|
||||
const created = await createSelectedSandbox(kind, createOptions, selected, signal)
|
||||
const abortBinding = bindSandboxAbort(created.handle, created.provider, signal)
|
||||
const usageEventId = await beginUsageOrStop(created, sandboxKind, usageContext, abortBinding)
|
||||
return { created, abortBinding, usageEventId }
|
||||
})()
|
||||
if (!usageContext) return admission
|
||||
|
||||
state.usageAdmission = admission
|
||||
@@ -726,26 +735,27 @@ async function executeInSandboxWithinBudget(
|
||||
})
|
||||
throwIfAborted(signal)
|
||||
|
||||
const created = await createSelectedSandbox(
|
||||
const createBudgetMs = remainingSandboxBudgetMs(signal)
|
||||
const { created, abortBinding, usageEventId } = await createAndAdmitSandbox(
|
||||
kind,
|
||||
{
|
||||
language,
|
||||
imageRef: selected?.imageRef,
|
||||
lifetimeMs: remainingSandboxBudgetMs(signal),
|
||||
lifetimeMs: createBudgetMs,
|
||||
...(req.usageContext
|
||||
? {
|
||||
requestTimeoutMs: Math.min(createBudgetMs, MAX_SANDBOX_CREATE_REQUEST_TIMEOUT_MS),
|
||||
}
|
||||
: {}),
|
||||
},
|
||||
selected,
|
||||
signal
|
||||
signal,
|
||||
'code',
|
||||
req.usageContext,
|
||||
state
|
||||
)
|
||||
const sandbox = created.handle
|
||||
const sandboxId = sandbox.sandboxId
|
||||
const abortBinding = bindSandboxAbort(sandbox, created.provider, signal)
|
||||
const usageEventId = await admitSandboxUsage(
|
||||
created,
|
||||
'code',
|
||||
req.usageContext,
|
||||
abortBinding,
|
||||
state
|
||||
)
|
||||
let outcome: SandboxUsageOutcome = 'infrastructure_error'
|
||||
|
||||
try {
|
||||
@@ -872,22 +882,26 @@ async function executeShellInSandboxWithinBudget(
|
||||
})
|
||||
throwIfAborted(signal)
|
||||
|
||||
const created = await createSelectedSandbox(
|
||||
const createBudgetMs = remainingSandboxBudgetMs(signal)
|
||||
const { created, abortBinding, usageEventId } = await createAndAdmitSandbox(
|
||||
kind,
|
||||
{ imageRef: selected?.imageRef, lifetimeMs: remainingSandboxBudgetMs(signal) },
|
||||
{
|
||||
imageRef: selected?.imageRef,
|
||||
lifetimeMs: createBudgetMs,
|
||||
...(req.usageContext
|
||||
? {
|
||||
requestTimeoutMs: Math.min(createBudgetMs, MAX_SANDBOX_CREATE_REQUEST_TIMEOUT_MS),
|
||||
}
|
||||
: {}),
|
||||
},
|
||||
selected,
|
||||
signal
|
||||
signal,
|
||||
'shell',
|
||||
req.usageContext,
|
||||
state
|
||||
)
|
||||
const sandbox = created.handle
|
||||
const sandboxId = sandbox.sandboxId
|
||||
const abortBinding = bindSandboxAbort(sandbox, created.provider, signal)
|
||||
const usageEventId = await admitSandboxUsage(
|
||||
created,
|
||||
'shell',
|
||||
req.usageContext,
|
||||
abortBinding,
|
||||
state
|
||||
)
|
||||
let outcome: SandboxUsageOutcome = 'infrastructure_error'
|
||||
|
||||
try {
|
||||
|
||||
@@ -218,6 +218,11 @@ export interface CreateSandboxOptions {
|
||||
* and creates the sandbox as ephemeral.
|
||||
*/
|
||||
lifetimeMs?: number
|
||||
/**
|
||||
* Maximum time to wait for the provider create request itself. This is
|
||||
* independent of {@link lifetimeMs}, which controls the created sandbox's TTL.
|
||||
*/
|
||||
requestTimeoutMs?: number
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user