fix(billing): close sandbox admission review gaps

This commit is contained in:
Bill Leoutsakos
2026-08-27 20:00:39 -07:00
parent b7cc411148
commit 90b2238bde
7 changed files with 179 additions and 40 deletions
@@ -21,7 +21,7 @@ vi.mock('@/lib/billing/threshold-billing', () => ({
}))
vi.mock('@/lib/core/config/env-flags', () => ({
getCostMultiplier: vi.fn(() => 1),
isBillingEnabled: false,
isBillingEnabled: true,
}))
vi.mock('@/lib/execution/remote-sandbox/provider', () => ({
getSandboxProvider: vi.fn(() => ({ terminateById: mockTerminateById })),
@@ -141,6 +141,16 @@ describe('sandbox usage outbox finalizer', () => {
text: 'GREATEST(COALESCE(, 0) + ::numeric, ::numeric)',
params: ['workflowExecutionLogs.costTotal', '0.00046', '0.02'],
})
expect(mockThresholdBilling).toHaveBeenCalledWith(
{ type: 'user', id: 'user-1' },
{
onError: 'throw',
expectedBillingPeriod: {
start: new Date('2026-08-01T00:00:00.000Z'),
end: new Date('2026-09-01T00:00:00.000Z'),
},
}
)
})
it('terminates and checkpoints a sandbox whose terminal timestamp is missing', async () => {
+4 -1
View File
@@ -333,7 +333,10 @@ async function finalizeSandboxUsage(
})
if (isBillingEnabled) {
await checkAndBillPayerOverageThreshold(billingContext.billingEntity, { onError: 'throw' })
await checkAndBillPayerOverageThreshold(billingContext.billingEntity, {
onError: 'throw',
expectedBillingPeriod: billingContext.billingPeriod,
})
}
logger.info('Recorded Function sandbox usage', {
@@ -163,6 +163,24 @@ function deferred<T>() {
return { promise, resolve, reject }
}
async function deferProviderCreate(provider: Provider): Promise<{ resolve: () => void }> {
if (provider === 'e2b') {
const implementation = mockE2BCreate.getMockImplementation()
if (!implementation) throw new Error('E2B create mock is not configured')
const created = await implementation()
const pending = deferred<typeof created>()
mockE2BCreate.mockReturnValueOnce(pending.promise)
return { resolve: () => pending.resolve(created) }
}
const implementation = mockDaytonaCreate.getMockImplementation()
if (!implementation) throw new Error('Daytona create mock is not configured')
const created = await implementation()
const pending = deferred<typeof created>()
mockDaytonaCreate.mockReturnValueOnce(pending.promise)
return { resolve: () => pending.resolve(created) }
}
const usageContext: SandboxUsageContext = {
workspaceId: 'ws-1',
workflowId: 'wf-1',
@@ -471,6 +489,87 @@ describe.each(PROVIDERS)('sandbox conformance [%s]', (provider) => {
).not.toHaveBeenCalled()
})
it('preserves usage persistence failure when cancellation races provider creation', async () => {
const controller = new AbortController()
const providerCreate = await deferProviderCreate(provider)
mockBeginSandboxUsage.mockRejectedValueOnce(new Error('database unavailable'))
const execution = executeInSandbox({
code: 'x',
language: CodeLanguage.Python,
timeoutMs: 1000,
signal: controller.signal,
usageContext,
})
const rejection = expect(execution).rejects.toMatchObject({
name: 'SandboxUsagePersistenceError',
retryable: false,
})
await vi.waitFor(() =>
expect(provider === 'e2b' ? mockE2BCreate : mockDaytonaCreate).toHaveBeenCalledOnce()
)
controller.abort(new DOMException('cancelled', 'AbortError'))
expect(mockBeginSandboxUsage).not.toHaveBeenCalled()
providerCreate.resolve()
await rejection
expect(mockBeginSandboxUsage).toHaveBeenCalledTimes(2)
expect(
provider === 'e2b' ? mockE2BCommandsRun : mockExecuteSessionCommand
).not.toHaveBeenCalled()
})
it('returns cancellation after provider creation, usage admission, and cleanup settle', async () => {
const controller = new AbortController()
const providerCreate = await deferProviderCreate(provider)
const execution = executeInSandbox({
code: 'x',
language: CodeLanguage.Python,
timeoutMs: 1000,
signal: controller.signal,
usageContext,
})
const rejection = expect(execution).rejects.toMatchObject({ name: 'AbortError' })
await vi.waitFor(() =>
expect(provider === 'e2b' ? mockE2BCreate : mockDaytonaCreate).toHaveBeenCalledOnce()
)
controller.abort(new DOMException('cancelled', 'AbortError'))
expect(mockBeginSandboxUsage).not.toHaveBeenCalled()
providerCreate.resolve()
await rejection
expect(mockBeginSandboxUsage).toHaveBeenCalledOnce()
expect(mockReleaseAndProcessSandboxUsage).toHaveBeenCalledWith(
'sandbox-usage-event',
expect.objectContaining({ outcome: 'cancelled', cleanupStatus: 'terminated' })
)
expect(
provider === 'e2b' ? mockE2BCommandsRun : mockExecuteSessionCommand
).not.toHaveBeenCalled()
})
it.each([1000, 120_000])(
'bounds provider creation by the remaining execution budget (%sms)',
async (timeoutMs) => {
await executeInSandbox({
code: 'x',
language: CodeLanguage.Python,
timeoutMs,
usageContext,
})
const requestTimeoutMs =
provider === 'e2b'
? mockE2BCreate.mock.calls[0]?.[1]?.requestTimeoutMs
: mockDaytonaCreate.mock.calls[0]?.[1]?.timeout * 1000
expect(requestTimeoutMs).toBeGreaterThan(0)
expect(requestTimeoutMs).toBeLessThanOrEqual(Math.min(timeoutMs, 60_000))
}
)
it('durably releases cleanup when persistence and live teardown initially fail', async () => {
mockBeginSandboxUsage.mockRejectedValueOnce(new Error('database unavailable'))
const teardown = provider === 'e2b' ? mockE2BKill : mockDelete
@@ -800,7 +800,12 @@ export const daytonaProvider: SandboxProvider = {
ephemeral: true,
ttlMinutes: resolveDaytonaLifetimeMs(options?.lifetimeMs) / 60_000,
}
const sandbox = await daytona.create(createOptions)
const sandbox =
options?.requestTimeoutMs === undefined
? await daytona.create(createOptions)
: await daytona.create(createOptions, {
timeout: Math.max(0.001, options.requestTimeoutMs / 1000),
})
return new DaytonaSandboxHandle(sandbox, language)
},
@@ -867,6 +867,9 @@ export const e2bProvider: SandboxProvider = {
const createOptions = {
apiKey,
timeoutMs: effectiveLifetimeMs,
...(options?.requestTimeoutMs !== undefined
? { requestTimeoutMs: options.requestTimeoutMs }
: {}),
}
const { Sandbox } = await import('@e2b/code-interpreter')
+51 -37
View File
@@ -55,6 +55,7 @@ export type {
} from '@/lib/execution/remote-sandbox/types'
const logger = createLogger('RemoteSandbox')
const MAX_SANDBOX_CREATE_REQUEST_TIMEOUT_MS = 60_000
interface CreatedSandbox {
handle: SandboxHandle
@@ -71,7 +72,7 @@ interface SandboxCleanupResult {
}
interface SandboxAbortRaceState {
usageAdmission?: Promise<string | undefined>
usageAdmission?: Promise<unknown>
}
async function createSandbox(
@@ -156,21 +157,18 @@ function raceSandboxAbort<T>(
return new Promise<T>((resolve, reject) => {
const onAbort = () => {
signal.removeEventListener('abort', onAbort)
const usageAdmission = state.usageAdmission
if (!usageAdmission) {
if (!state.usageAdmission) {
reject(abortReason(signal))
return
}
/**
* A created metered sandbox must finish its durable usage admission before
* cancellation can decide the response. Success preserves the abort;
* failure surfaces the non-retryable persistence error after cleanup.
* A metered provider create must finish usage admission and cleanup before
* cancellation can decide the response. The operation observer below
* preserves the abort after successful admission or propagates a
* non-retryable persistence failure.
*/
void usageAdmission.then(
() => reject(abortReason(signal)),
(error) => reject(error)
)
return
}
signal.addEventListener('abort', onAbort, { once: true })
operation.then(
@@ -339,14 +337,25 @@ async function beginUsageOrStop(
}
}
async function admitSandboxUsage(
created: CreatedSandbox,
async function createAndAdmitSandbox(
kind: SandboxKind,
createOptions: CreateSandboxOptions,
selected: ResolvedSandbox | null,
signal: AbortSignal,
sandboxKind: 'code' | 'shell',
usageContext: SandboxUsageContext | undefined,
abortBinding: ReturnType<typeof bindSandboxAbort>,
state: SandboxAbortRaceState
): Promise<string | undefined> {
const admission = beginUsageOrStop(created, sandboxKind, usageContext, abortBinding)
): Promise<{
created: CreatedSandbox
abortBinding: ReturnType<typeof bindSandboxAbort>
usageEventId: string | undefined
}> {
const admission = (async () => {
const created = await createSelectedSandbox(kind, createOptions, selected, signal)
const abortBinding = bindSandboxAbort(created.handle, created.provider, signal)
const usageEventId = await beginUsageOrStop(created, sandboxKind, usageContext, abortBinding)
return { created, abortBinding, usageEventId }
})()
if (!usageContext) return admission
state.usageAdmission = admission
@@ -726,26 +735,27 @@ async function executeInSandboxWithinBudget(
})
throwIfAborted(signal)
const created = await createSelectedSandbox(
const createBudgetMs = remainingSandboxBudgetMs(signal)
const { created, abortBinding, usageEventId } = await createAndAdmitSandbox(
kind,
{
language,
imageRef: selected?.imageRef,
lifetimeMs: remainingSandboxBudgetMs(signal),
lifetimeMs: createBudgetMs,
...(req.usageContext
? {
requestTimeoutMs: Math.min(createBudgetMs, MAX_SANDBOX_CREATE_REQUEST_TIMEOUT_MS),
}
: {}),
},
selected,
signal
signal,
'code',
req.usageContext,
state
)
const sandbox = created.handle
const sandboxId = sandbox.sandboxId
const abortBinding = bindSandboxAbort(sandbox, created.provider, signal)
const usageEventId = await admitSandboxUsage(
created,
'code',
req.usageContext,
abortBinding,
state
)
let outcome: SandboxUsageOutcome = 'infrastructure_error'
try {
@@ -872,22 +882,26 @@ async function executeShellInSandboxWithinBudget(
})
throwIfAborted(signal)
const created = await createSelectedSandbox(
const createBudgetMs = remainingSandboxBudgetMs(signal)
const { created, abortBinding, usageEventId } = await createAndAdmitSandbox(
kind,
{ imageRef: selected?.imageRef, lifetimeMs: remainingSandboxBudgetMs(signal) },
{
imageRef: selected?.imageRef,
lifetimeMs: createBudgetMs,
...(req.usageContext
? {
requestTimeoutMs: Math.min(createBudgetMs, MAX_SANDBOX_CREATE_REQUEST_TIMEOUT_MS),
}
: {}),
},
selected,
signal
signal,
'shell',
req.usageContext,
state
)
const sandbox = created.handle
const sandboxId = sandbox.sandboxId
const abortBinding = bindSandboxAbort(sandbox, created.provider, signal)
const usageEventId = await admitSandboxUsage(
created,
'shell',
req.usageContext,
abortBinding,
state
)
let outcome: SandboxUsageOutcome = 'infrastructure_error'
try {
@@ -218,6 +218,11 @@ export interface CreateSandboxOptions {
* and creates the sandbox as ephemeral.
*/
lifetimeMs?: number
/**
* Maximum time to wait for the provider create request itself. This is
* independent of {@link lifetimeMs}, which controls the created sandbox's TTL.
*/
requestTimeoutMs?: number
}
/**