fix(execution): make function runs rollout-safe (#7143)

apps/sim/app/api/workflows/[id]/execute/route.async.test.ts

@@ -53,7 +53,8 @@ const {
   mockHandlePostExecutionPauseState,
   mockHasDurableExecutionOwner,
   mockInitializeExecutionStreamMeta,
-  mockSetExecutionMeta,
+  mockMarkExecutionStreamTerminal,
+  mockSetExecutionActiveBlockStarts,
   mockReleaseExecutionIdClaim,
   mockReleaseExecutionSlot,
   mockReleaseWorkflowToolExecutionClaim,
@@ -82,7 +83,8 @@ const {
   mockHandlePostExecutionPauseState: vi.fn(),
   mockHasDurableExecutionOwner: vi.fn(),
   mockInitializeExecutionStreamMeta: vi.fn(),
-  mockSetExecutionMeta: vi.fn(),
+  mockMarkExecutionStreamTerminal: vi.fn(),
+  mockSetExecutionActiveBlockStarts: vi.fn(),
   mockReleaseExecutionIdClaim: vi.fn(),
   mockReleaseExecutionSlot: vi.fn(),
   mockReleaseWorkflowToolExecutionClaim: vi.fn(),
@@ -152,7 +154,8 @@ vi.mock('@/lib/execution/event-buffer', () => ({
   createExecutionEventWriter: mockCreateExecutionEventWriter,
   flushExecutionStreamReplayBuffer: mockFlushExecutionStreamReplayBuffer,
   initializeExecutionStreamMeta: mockInitializeExecutionStreamMeta,
-  setExecutionMeta: mockSetExecutionMeta,
+  markExecutionStreamTerminal: mockMarkExecutionStreamTerminal,
+  setExecutionActiveBlockStarts: mockSetExecutionActiveBlockStarts,
   LIVE_ONLY_EXECUTION_EVENT_TYPES: new Set(),
 }))
 
@@ -491,11 +494,13 @@ describe('workflow execute async route', () => {
     })
     mockHandlePostExecutionPauseState.mockResolvedValue(undefined)
     mockInitializeExecutionStreamMeta.mockReset().mockResolvedValue(true)
-    mockSetExecutionMeta.mockReset().mockResolvedValue(true)
+    mockMarkExecutionStreamTerminal.mockReset().mockResolvedValue(true)
+    mockSetExecutionActiveBlockStarts.mockReset().mockResolvedValue(true)
     mockFlushExecutionStreamReplayBuffer.mockReset().mockResolvedValue(true)
     mockCreateExecutionEventWriter.mockReset().mockReturnValue({
       write: vi.fn(async (event: unknown) => ({ event, eventId: '1' })),
       writeTerminal: vi.fn(async (event: unknown) => ({ event, eventId: '2' })),
+      flush: vi.fn().mockResolvedValue(undefined),
       close: vi.fn().mockResolvedValue(undefined),
     })
     loggingSessionMockFns.mockWaitForPostExecution.mockReset().mockResolvedValue(undefined)
@@ -723,10 +728,169 @@ describe('workflow execute async route', () => {
     })
   })
 
+  it('keeps a manual execution alive when its browser SSE observer detaches', async () => {
+    let capturedSignal: AbortSignal | undefined
+    let resolveStarted: (() => void) | undefined
+    const started = new Promise<void>((resolve) => {
+      resolveStarted = resolve
+    })
+    let resolveExecution: (() => void) | undefined
+    mockExecuteWorkflowCore.mockImplementationOnce(
+      ({ abortSignal }: { abortSignal: AbortSignal }) =>
+        new Promise((resolve) => {
+          capturedSignal = abortSignal
+          resolveStarted?.()
+          resolveExecution = () =>
+            resolve({
+              success: true,
+              status: 'completed',
+              output: { ok: true },
+              metadata: {
+                duration: 100,
+                startTime: '2026-01-01T00:00:00Z',
+                endTime: '2026-01-01T00:00:01Z',
+              },
+            })
+        })
+    )
+
+    const response = await POST(
+      createMockRequest(
+        'POST',
+        {
+          stream: true,
+          input: { message: 'hello' },
+          triggerType: 'manual',
+          isClientSession: true,
+        },
+        {
+          'Content-Type': 'application/json',
+          Cookie: 'session=value',
+        }
+      ),
+      { params: Promise.resolve({ id: 'workflow-1' }) }
+    )
+    await started
+
+    const detach = response.body?.cancel()
+    await Promise.resolve()
+    expect(capturedSignal?.aborted).toBe(false)
+
+    resolveExecution?.()
+    await detach
+    expect(capturedSignal?.aborted).toBe(false)
+  })
+
+  it('fails the observer closed when an active-block snapshot cannot be persisted', async () => {
+    mockSetExecutionActiveBlockStarts.mockResolvedValueOnce(false)
+    mockExecuteWorkflowCore.mockImplementationOnce(async ({ callbacks, abortSignal }) => {
+      try {
+        await callbacks.onBlockStart('function-1', 'Function', 'function', 1)
+      } finally {
+        expect(abortSignal.aborted).toBe(true)
+      }
+      throw new Error('workflow should stop after the lifecycle persistence failure')
+    })
+
+    const response = await POST(
+      createMockRequest(
+        'POST',
+        {
+          stream: true,
+          input: { message: 'hello' },
+          triggerType: 'manual',
+          isClientSession: true,
+        },
+        {
+          'Content-Type': 'application/json',
+          Cookie: 'session=value',
+        }
+      ),
+      { params: Promise.resolve({ id: 'workflow-1' }) }
+    )
+
+    await expect(response.text()).rejects.toThrow('Failed to persist active execution snapshot')
+    await vi.waitFor(() => {
+      expect(mockMarkExecutionStreamTerminal).toHaveBeenCalledWith('execution-123', 'error')
+    })
+  })
+
+  it('serializes parallel lifecycle events before sending them to the observer', async () => {
+    let releaseFirstSnapshot!: () => void
+    const firstSnapshotReleased = new Promise<void>((resolve) => {
+      releaseFirstSnapshot = resolve
+    })
+    let firstSnapshotStarted!: () => void
+    const firstSnapshotPending = new Promise<void>((resolve) => {
+      firstSnapshotStarted = resolve
+    })
+    let snapshotCalls = 0
+    mockSetExecutionActiveBlockStarts.mockImplementation(async () => {
+      snapshotCalls += 1
+      if (snapshotCalls === 1) {
+        firstSnapshotStarted()
+        await firstSnapshotReleased
+      }
+      return true
+    })
+    let eventId = 0
+    mockCreateExecutionEventWriter.mockReturnValue({
+      write: vi.fn(async (event: unknown) => ({ event, eventId: String(++eventId) })),
+      writeTerminal: vi.fn(async (event: unknown) => ({ event, eventId: String(++eventId) })),
+      flush: vi.fn().mockResolvedValue(undefined),
+      close: vi.fn().mockResolvedValue(undefined),
+    })
+    mockExecuteWorkflowCore.mockImplementationOnce(async ({ callbacks }) => {
+      const first = callbacks.onBlockStart('function-a', 'Function A', 'function', 1)
+      const second = callbacks.onBlockStart('function-b', 'Function B', 'function', 2)
+      await Promise.all([first, second])
+      return {
+        success: true,
+        status: 'completed',
+        output: { ok: true },
+        metadata: {
+          duration: 100,
+          startTime: '2026-01-01T00:00:00Z',
+          endTime: '2026-01-01T00:00:01Z',
+        },
+      }
+    })
+
+    const response = await POST(
+      createMockRequest(
+        'POST',
+        {
+          stream: true,
+          input: { message: 'hello' },
+          triggerType: 'manual',
+          isClientSession: true,
+        },
+        {
+          'Content-Type': 'application/json',
+          Cookie: 'session=value',
+        }
+      ),
+      { params: Promise.resolve({ id: 'workflow-1' }) }
+    )
+    await firstSnapshotPending
+    releaseFirstSnapshot()
+    const body = await response.text()
+
+    expect(body.indexOf('function-a')).toBeGreaterThan(-1)
+    expect(body.indexOf('function-b')).toBeGreaterThan(body.indexOf('function-a'))
+    expect(mockSetExecutionActiveBlockStarts).toHaveBeenNthCalledWith(
+      2,
+      'execution-123',
+      expect.arrayContaining([
+        expect.objectContaining({ data: expect.objectContaining({ blockId: 'function-a' }) }),
+        expect.objectContaining({ data: expect.objectContaining({ blockId: 'function-b' }) }),
+      ])
+    )
+  })
+
   /**
-   * A terminal event the replay buffer rejected leaves the stream meta on
-   * `active`, so a reconnecting reader polls until its deadline and then errors.
-   * Recording the status directly is the only signal it gets.
+   * A terminal event the replay buffer rejected leaves no terminal event for a reconnecting reader.
+   * Recording terminal metadata lets the pushed wake close the observer instead of leaving it active.
    */
   it('records terminal stream meta when the replay buffer rejects the terminal event', async () => {
     mockCreateExecutionEventWriter.mockReturnValue({
@@ -734,18 +898,19 @@ describe('workflow execute async route', () => {
       writeTerminal: vi.fn(async () => {
         throw new Error('Execution memory limit exceeded. Reduce payload size and try again.')
       }),
+      flush: vi.fn().mockResolvedValue(undefined),
       close: vi.fn().mockResolvedValue(undefined),
     })
 
     const response = await POST(createBoundCopilotExecutionRequest(), {
       params: Promise.resolve({ id: 'workflow-1' }),
     })
-    const body = await response.text()
 
     expect(response.status).toBe(200)
-    // The live client still receives the terminal event over SSE.
-    expect(body).toContain('execution:completed')
-    expect(mockSetExecutionMeta).toHaveBeenCalledWith('execution-123', { status: 'complete' })
+    await expect(response.text()).rejects.toThrow(
+      'Execution memory limit exceeded. Reduce payload size and try again.'
+    )
+    expect(mockMarkExecutionStreamTerminal).toHaveBeenCalledWith('execution-123', 'error')
   })
 
   it('rejects a competing Copilot workflow execution before logging starts', async () => {

apps/sim/app/api/workflows/[id]/execute/route.ts

@@ -63,7 +63,8 @@ import {
   createExecutionEventWriter,
   flushExecutionStreamReplayBuffer,
   initializeExecutionStreamMeta,
-  setExecutionMeta,
+  markExecutionStreamTerminal,
+  setExecutionActiveBlockStarts,
   type TerminalExecutionStreamStatus,
 } from '@/lib/execution/event-buffer'
 import {
@@ -105,8 +106,10 @@ import { enqueueWorkflowExecution } from '@/lib/workflows/executor/enqueue-execu
 import { executeWorkflow } from '@/lib/workflows/executor/execute-workflow'
 import { executeWorkflowCore } from '@/lib/workflows/executor/execution-core'
 import {
+  type BlockStartedData,
   type ExecutionEvent,
   encodeSSEEvent,
+  getBlockInvocationKey,
   LIVE_ONLY_EXECUTION_EVENT_TYPES,
 } from '@/lib/workflows/executor/execution-events'
 import {
@@ -1752,6 +1755,7 @@ async function handleExecutePost(
       isTimeoutError(error)
     let isStreamClosed = false
     let isManualAbortRegistered = false
+    const activeBlockStarts = new Map<string, { eventId: number; data: BlockStartedData }>()
 
     const eventWriter = createExecutionEventWriter(executionId, {
       workspaceId,
@@ -1788,63 +1792,73 @@ async function handleExecutePost(
         isManualAbortRegistered = true
 
         let terminalEventPublished = false
-        const sendEvent = async (
+        let sendFailure: unknown
+        let sendQueue: Promise<void> = Promise.resolve()
+        const sendEvent = (
           event: ExecutionEvent,
           terminalStatus?: TerminalExecutionStreamStatus
         ) => {
-          const isBuffered = !LIVE_ONLY_EXECUTION_EVENT_TYPES.has(event.type)
-          let eventToSend = event
-          let terminalBufferWriteFailed = false
-          if (isBuffered) {
-            try {
+          const task = sendQueue.then(async () => {
+            if (sendFailure) throw sendFailure
+            const isBuffered = !LIVE_ONLY_EXECUTION_EVENT_TYPES.has(event.type)
+            let eventToSend = event
+            if (isBuffered) {
               const entry = terminalStatus
                 ? await eventWriter.writeTerminal(event, terminalStatus)
                 : await eventWriter.write(event)
+              if (!terminalStatus) {
+                await eventWriter.flush()
+              }
               eventToSend = entry.event
               eventToSend.eventId = entry.eventId
               terminalEventPublished ||= Boolean(terminalStatus)
-            } catch (e) {
-              // The event buffer (Redis replay store) rejected this event — e.g. the flush
-              // batch exceeds the per-write byte cap for large block outputs. The buffer only
-              // backs reconnect/replay; the live SSE stream is the primary delivery. Fall
-              // through to enqueue the event live (below) instead of throwing, so terminal
-              // events still reach the active client and the UI doesn't hang on "running".
-              // Marking a terminal event delivered-live as published lets finalization close
-              // the stream cleanly instead of aborting it with controller.error().
-              reqLogger.warn(
-                'Event buffer write failed; delivering event over live stream only',
-                loggingSession.projectDiagnosticError(e, {
-                  eventType: event.type,
-                  terminal: Boolean(terminalStatus),
+
+              if (eventToSend.type === 'block:started') {
+                activeBlockStarts.set(getBlockInvocationKey(eventToSend.data), {
+                  eventId: entry.eventId,
+                  data: eventToSend.data,
                 })
-              )
-              terminalBufferWriteFailed = Boolean(terminalStatus)
-              terminalEventPublished ||= Boolean(terminalStatus)
+              } else if (
+                eventToSend.type === 'block:completed' ||
+                eventToSend.type === 'block:error'
+              ) {
+                activeBlockStarts.delete(getBlockInvocationKey(eventToSend.data))
+              } else if (terminalStatus) {
+                activeBlockStarts.clear()
+              }
+
+              if (
+                eventToSend.type === 'block:started' ||
+                eventToSend.type === 'block:completed' ||
+                eventToSend.type === 'block:error'
+              ) {
+                const activeSnapshotPersisted = await setExecutionActiveBlockStarts(executionId, [
+                  ...activeBlockStarts.values(),
+                ])
+                if (!activeSnapshotPersisted) {
+                  throw new Error('Failed to persist active execution snapshot')
+                }
+              }
             }
-          }
-          if (!isStreamClosed) {
-            try {
-              controller.enqueue(encodeSSEEvent(eventToSend))
-            } catch {
+            if (!isStreamClosed) {
+              try {
+                controller.enqueue(encodeSSEEvent(eventToSend))
+              } catch {
+                isStreamClosed = true
+              }
+            }
+          })
+          sendQueue = task.catch((error) => {
+            sendFailure ??= error
+            timeoutController.abort()
+            if (!isStreamClosed) {
+              try {
+                controller.error(error)
+              } catch {}
               isStreamClosed = true
             }
-          }
-          if (terminalBufferWriteFailed && terminalStatus) {
-            // Without this the reconnect route polls an `active` stream until its
-            // deadline. The meta write is a plain HSET, so it bypasses the byte budget
-            // that rejected the event. Runs after the live enqueue because Redis is the
-            // likely reason we are here at all, and a slow best-effort durability write
-            // must not delay the primary delivery path.
-            const metaPersisted = await setExecutionMeta(executionId, {
-              status: terminalStatus,
-            })
-            if (!metaPersisted) {
-              reqLogger.error(
-                'Failed to record terminal execution meta after buffer write failure',
-                { executionId, status: terminalStatus }
-              )
-            }
-          }
+          })
+          return task
         }
 
         try {
@@ -2057,13 +2071,13 @@ async function handleExecutePost(
             }
 
             try {
-              if (timeoutController.signal.aborted || isStreamClosed) return
+              if (timeoutController.signal.aborted) return
               timeoutController.signal.addEventListener('abort', cancelReader, { once: true })
 
               while (true) {
-                if (timeoutController.signal.aborted || isStreamClosed) break
+                if (timeoutController.signal.aborted) break
                 const { done, value } = await reader.read()
-                if (timeoutController.signal.aborted || isStreamClosed) break
+                if (timeoutController.signal.aborted) break
                 if (done) break
 
                 if (answerTextFromSink) continue
@@ -2083,7 +2097,7 @@ async function handleExecutePost(
                 })
               }
 
-              if (!timeoutController.signal.aborted && !isStreamClosed) {
+              if (!timeoutController.signal.aborted) {
                 await sendEvent({
                   type: 'stream:done',
                   timestamp: new Date().toISOString(),
@@ -2093,7 +2107,7 @@ async function handleExecutePost(
                 })
               }
             } catch (error) {
-              if (!timeoutController.signal.aborted && !isStreamClosed) {
+              if (!timeoutController.signal.aborted) {
                 reqLogger.error(
                   'Error streaming block content',
                   loggingSession.projectDiagnosticError(error, { blockId })
@@ -2400,10 +2414,15 @@ async function handleExecutePost(
               executionId,
               eventWriter
             )
+            const terminalMetaPersisted = await markExecutionStreamTerminal(
+              executionId,
+              finalMetaStatus
+            )
             reqLogger.error('Failed to publish terminal execution event durably', {
               executionId,
               status: finalMetaStatus,
               replayBufferFlushed,
+              terminalMetaPersisted,
             })
             if (!isStreamClosed) {
               controller.error(new Error('Run buffer terminal event publish failed'))
@@ -2440,8 +2459,7 @@ async function handleExecutePost(
       },
       cancel() {
         isStreamClosed = true
-        timeoutController.abort()
-        reqLogger.info('Client disconnected from SSE stream')
+        reqLogger.info('Client detached from SSE stream; workflow execution remains active')
       },
     })
 

apps/sim/app/api/workflows/[id]/executions/[executionId]/stream/route.test.ts

@@ -5,19 +5,27 @@ import { authMockFns, createMockRequest, workflowAuthzMockFns } from '@sim/testi
 import { beforeEach, describe, expect, it, vi } from 'vitest'
 import type { ExecutionEventEntry } from '@/lib/execution/event-buffer'
 
-const { mockReadExecutionEventsState, mockReadExecutionMetaState } = vi.hoisted(() => ({
-  mockReadExecutionEventsState: vi.fn(),
-  mockReadExecutionMetaState: vi.fn(),
-}))
+const { mockReadExecutionEventsState, mockReadExecutionMetaState, mockSubscribe } = vi.hoisted(
+  () => ({
+    mockReadExecutionEventsState: vi.fn(),
+    mockReadExecutionMetaState: vi.fn(),
+    mockSubscribe: vi.fn(),
+  })
+)
 
 const mockAuthorizeWorkflowByWorkspacePermission =
   workflowAuthzMockFns.mockAuthorizeWorkflowByWorkspacePermission
 
 vi.mock('@/lib/execution/event-buffer', () => ({
+  EXECUTION_STREAM_PROTOCOL_VERSION: 1,
   readExecutionEventsState: mockReadExecutionEventsState,
   readExecutionMetaState: mockReadExecutionMetaState,
 }))
 
+vi.mock('@/lib/execution/execution-signal', () => ({
+  getExecutionSignalHub: () => ({ subscribe: mockSubscribe }),
+}))
+
 import { GET } from './route'
 
 const mockGetSession = authMockFns.mockGetSession
@@ -43,6 +51,29 @@ function completedEntry(eventId: number): ExecutionEventEntry {
   }
 }
 
+function blockCompletedEntry(eventId: number): ExecutionEventEntry {
+  return {
+    eventId,
+    executionId: 'exec-1',
+    event: {
+      type: 'block:completed',
+      timestamp: new Date().toISOString(),
+      executionId: 'exec-1',
+      workflowId: 'wf-1',
+      data: {
+        blockId: 'function-1',
+        blockName: 'Qualify',
+        blockType: 'function',
+        output: { ok: true },
+        durationMs: 10,
+        startedAt: new Date().toISOString(),
+        executionOrder: 2,
+        endedAt: new Date().toISOString(),
+      },
+    },
+  }
+}
+
 describe('execution stream reconnect route', () => {
   beforeEach(() => {
     vi.clearAllMocks()
@@ -50,9 +81,10 @@ describe('execution stream reconnect route', () => {
     mockAuthorizeWorkflowByWorkspacePermission.mockResolvedValue({ allowed: true })
     mockReadExecutionMetaState.mockResolvedValue({
       status: 'found',
-      meta: { status: 'active', workflowId: 'wf-1' },
+      meta: { status: 'active', workflowId: 'wf-1', protocolVersion: 1 },
     })
     mockReadExecutionEventsState.mockResolvedValue({ status: 'ok', events: [] })
+    mockSubscribe.mockResolvedValue(vi.fn())
   })
 
   it('drains final events after terminal meta before sending DONE', async () => {
@@ -90,7 +122,7 @@ describe('execution stream reconnect route', () => {
     expect(mockReadExecutionEventsState).toHaveBeenNthCalledWith(2, 'exec-1', 3)
   })
 
-  it('ends the stream cleanly when terminal metadata has no terminal event to replay', async () => {
+  it('fails closed when terminal metadata has no terminal event to replay', async () => {
     mockReadExecutionMetaState
       .mockResolvedValueOnce({
         status: 'found',
@@ -115,7 +147,11 @@ describe('execution stream reconnect route', () => {
     })
 
     expect(response.status).toBe(200)
-    await expect(response.text()).resolves.toContain('data: [DONE]')
+    const body = await response.text()
+
+    expect(body).toContain('"type":"execution:error"')
+    expect(body).toContain('its final event could not be recovered')
+    expect(body).toContain('data: [DONE]')
   })
 
   it('allows replay event id gaps from reserved but unused writer ids', async () => {
@@ -141,6 +177,253 @@ describe('execution stream reconnect route', () => {
     expect(body).toContain('data: [DONE]')
   })
 
+  it('hydrates active block starts that were already acknowledged before refresh', async () => {
+    const activeBlockStarts = [
+      {
+        eventId: 4,
+        data: {
+          blockId: 'function-1',
+          blockName: 'Qualify',
+          blockType: 'function',
+          executionOrder: 2,
+        },
+      },
+    ]
+    mockReadExecutionMetaState.mockResolvedValue({
+      status: 'found',
+      meta: { status: 'active', workflowId: 'wf-1', activeBlockStarts },
+    })
+    mockReadExecutionEventsState.mockResolvedValueOnce({
+      status: 'ok',
+      events: [completedEntry(6)],
+    })
+
+    const req = createMockRequest(
+      'GET',
+      undefined,
+      undefined,
+      'http://localhost/api/workflows/wf-1/executions/exec-1/stream?from=5'
+    )
+    const response = await GET(req, {
+      params: Promise.resolve({ id: 'wf-1', executionId: 'exec-1' }),
+    })
+
+    expect(response.status).toBe(200)
+    const body = await response.text()
+    expect(body.indexOf('"type":"block:started"')).toBeLessThan(
+      body.indexOf('"type":"execution:completed"')
+    )
+    expect(body).toContain('"blockId":"function-1"')
+  })
+
+  it('hydrates an active block snapshot committed after the reconnect subscription starts', async () => {
+    let signalHandler: ((reason: 'event') => void) | undefined
+    mockSubscribe.mockImplementationOnce(async (_executionId, handler) => {
+      signalHandler = handler
+      return vi.fn()
+    })
+    mockReadExecutionMetaState
+      .mockResolvedValueOnce({
+        status: 'found',
+        meta: { status: 'active', workflowId: 'wf-1', protocolVersion: 1 },
+      })
+      .mockResolvedValueOnce({
+        status: 'found',
+        meta: { status: 'active', workflowId: 'wf-1', protocolVersion: 1 },
+      })
+      .mockResolvedValueOnce({
+        status: 'found',
+        meta: {
+          status: 'active',
+          workflowId: 'wf-1',
+          protocolVersion: 1,
+          activeBlockStarts: [
+            {
+              eventId: 4,
+              data: {
+                blockId: 'function-1',
+                blockName: 'Qualify',
+                blockType: 'function',
+                executionOrder: 2,
+              },
+            },
+          ],
+        },
+      })
+    mockReadExecutionEventsState
+      .mockResolvedValueOnce({ status: 'ok', events: [] })
+      .mockResolvedValueOnce({ status: 'ok', events: [] })
+      .mockResolvedValueOnce({ status: 'ok', events: [completedEntry(6)] })
+
+    const req = createMockRequest(
+      'GET',
+      undefined,
+      undefined,
+      'http://localhost/api/workflows/wf-1/executions/exec-1/stream?from=5'
+    )
+    const response = await GET(req, {
+      params: Promise.resolve({ id: 'wf-1', executionId: 'exec-1' }),
+    })
+    const body = response.text()
+    await vi.waitFor(() => expect(mockReadExecutionEventsState).toHaveBeenCalledTimes(1))
+
+    signalHandler?.('event')
+    await vi.waitFor(() => expect(mockReadExecutionMetaState).toHaveBeenCalledTimes(3))
+    signalHandler?.('event')
+
+    const text = await body
+    expect(text.indexOf('"type":"block:started"')).toBeLessThan(
+      text.indexOf('"type":"execution:completed"')
+    )
+    expect(text.match(/"type":"block:started"/g)).toHaveLength(1)
+  })
+
+  it('does not rehydrate a stale active snapshot after its completion event was replayed', async () => {
+    let signalHandler: ((reason: 'event') => void) | undefined
+    mockSubscribe.mockImplementationOnce(async (_executionId, handler) => {
+      signalHandler = handler
+      return vi.fn()
+    })
+    const staleActiveMeta = {
+      status: 'found' as const,
+      meta: {
+        status: 'active' as const,
+        workflowId: 'wf-1',
+        protocolVersion: 1,
+        activeBlockStarts: [
+          {
+            eventId: 4,
+            data: {
+              blockId: 'function-1',
+              blockName: 'Qualify',
+              blockType: 'function',
+              executionOrder: 2,
+            },
+          },
+        ],
+      },
+    }
+    mockReadExecutionMetaState
+      .mockResolvedValueOnce({
+        status: 'found',
+        meta: { status: 'active', workflowId: 'wf-1', protocolVersion: 1 },
+      })
+      .mockResolvedValueOnce({
+        status: 'found',
+        meta: { status: 'active', workflowId: 'wf-1', protocolVersion: 1 },
+      })
+      .mockResolvedValueOnce(staleActiveMeta)
+    mockReadExecutionEventsState
+      .mockResolvedValueOnce({ status: 'ok', events: [] })
+      .mockResolvedValueOnce({ status: 'ok', events: [blockCompletedEntry(6)] })
+      .mockResolvedValueOnce({ status: 'ok', events: [completedEntry(7)] })
+
+    const req = createMockRequest(
+      'GET',
+      undefined,
+      undefined,
+      'http://localhost/api/workflows/wf-1/executions/exec-1/stream?from=5'
+    )
+    const response = await GET(req, {
+      params: Promise.resolve({ id: 'wf-1', executionId: 'exec-1' }),
+    })
+    const body = response.text()
+    await vi.waitFor(() => expect(mockReadExecutionEventsState).toHaveBeenCalledTimes(1))
+
+    signalHandler?.('event')
+    await vi.waitFor(() => expect(mockReadExecutionMetaState).toHaveBeenCalledTimes(3))
+    signalHandler?.('event')
+
+    const text = await body
+    expect(text).toContain('"type":"block:completed"')
+    expect(text).not.toContain('"type":"block:started"')
+  })
+
+  it('fails the observer closed when its Redis signal subscription becomes unavailable', async () => {
+    mockSubscribe.mockImplementationOnce(async (_executionId, handler) => {
+      queueMicrotask(() => handler('unavailable'))
+      return vi.fn()
+    })
+    const req = createMockRequest(
+      'GET',
+      undefined,
+      undefined,
+      'http://localhost/api/workflows/wf-1/executions/exec-1/stream?from=0'
+    )
+    const response = await GET(req, {
+      params: Promise.resolve({ id: 'wf-1', executionId: 'exec-1' }),
+    })
+
+    await expect(response.text()).rejects.toThrow(
+      'Execution signal subscription became unavailable'
+    )
+  })
+
+  it('unsubscribes immediately when the client closes before subscription acknowledgement', async () => {
+    let acknowledge!: (unsubscribe: () => void) => void
+    const cleanup = vi.fn()
+    mockSubscribe.mockReturnValueOnce(
+      new Promise<() => void>((resolve) => {
+        acknowledge = resolve
+      })
+    )
+    const req = createMockRequest(
+      'GET',
+      undefined,
+      undefined,
+      'http://localhost/api/workflows/wf-1/executions/exec-1/stream?from=0'
+    )
+    const response = await GET(req, {
+      params: Promise.resolve({ id: 'wf-1', executionId: 'exec-1' }),
+    })
+    const reader = response.body!.getReader()
+    const cancelled = reader.cancel()
+
+    acknowledge(cleanup)
+    await cancelled
+
+    expect(cleanup).toHaveBeenCalledOnce()
+    expect(mockReadExecutionEventsState).not.toHaveBeenCalled()
+  })
+
+  it('keeps reading events from a legacy producer that cannot publish wake signals', async () => {
+    vi.useFakeTimers()
+    try {
+      mockReadExecutionMetaState
+        .mockResolvedValueOnce({
+          status: 'found',
+          meta: { status: 'active', workflowId: 'wf-1' },
+        })
+        .mockResolvedValueOnce({
+          status: 'found',
+          meta: { status: 'active', workflowId: 'wf-1' },
+        })
+        .mockResolvedValueOnce({
+          status: 'found',
+          meta: { status: 'complete', workflowId: 'wf-1' },
+        })
+      mockReadExecutionEventsState
+        .mockResolvedValueOnce({ status: 'ok', events: [] })
+        .mockResolvedValueOnce({ status: 'ok', events: [completedEntry(1)] })
+
+      const req = createMockRequest(
+        'GET',
+        undefined,
+        undefined,
+        'http://localhost/api/workflows/wf-1/executions/exec-1/stream?from=0'
+      )
+      const response = await GET(req, {
+        params: Promise.resolve({ id: 'wf-1', executionId: 'exec-1' }),
+      })
+      const body = response.text()
+      await vi.advanceTimersByTimeAsync(500)
+
+      await expect(body).resolves.toContain('"type":"execution:completed"')
+    } finally {
+      vi.useRealTimers()
+    }
+  })
+
   it('errors when replay events are not strictly increasing', async () => {
     mockReadExecutionEventsState.mockResolvedValueOnce({
       status: 'ok',
@@ -207,7 +490,7 @@ describe('execution stream reconnect route', () => {
     expect(body).toContain('"type":"execution:completed"')
     expect(body).toContain('data: [DONE]')
     expect(mockReadExecutionEventsState).toHaveBeenCalledTimes(1)
-    expect(mockReadExecutionMetaState).toHaveBeenCalledTimes(1)
+    expect(mockReadExecutionMetaState).toHaveBeenCalledTimes(2)
   })
 
   it('errors the stream when replay events cannot be read', async () => {

apps/sim/app/api/workflows/[id]/executions/[executionId]/stream/route.ts

@@ -1,7 +1,6 @@
 import { createLogger } from '@sim/logger'
 import { authorizeWorkflowByWorkspacePermission } from '@sim/platform-authz/workflow'
 import { toError } from '@sim/utils/errors'
-import { sleep } from '@sim/utils/helpers'
 import { type NextRequest, NextResponse } from 'next/server'
 import { streamWorkflowExecutionContract } from '@/lib/api/contracts/workflows'
 import { parseRequest } from '@/lib/api/server'
@@ -9,18 +8,23 @@ import { getSession } from '@/lib/auth'
 import { SSE_HEADERS } from '@/lib/core/utils/sse'
 import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
 import {
+  EXECUTION_STREAM_PROTOCOL_VERSION,
   type ExecutionEventEntry,
   type ExecutionStreamStatus,
   readExecutionEventsState,
   readExecutionMetaState,
 } from '@/lib/execution/event-buffer'
-import type { ExecutionEvent } from '@/lib/workflows/executor/execution-events'
-import { formatSSEEvent } from '@/lib/workflows/executor/execution-events'
+import { type ExecutionSignalReason, getExecutionSignalHub } from '@/lib/execution/execution-signal'
+import {
+  type ExecutionEvent,
+  formatSSEEvent,
+  getBlockInvocationKey,
+} from '@/lib/workflows/executor/execution-events'
 
 const logger = createLogger('ExecutionStreamReconnectAPI')
 
-const POLL_INTERVAL_MS = 500
-const MAX_POLL_DURATION_MS = 55 * 60 * 1000 // 55 minutes (just under Redis 1hr TTL)
+const HEARTBEAT_INTERVAL_MS = 15_000
+const LEGACY_REPLAY_INTERVAL_MS = 500
 
 function isTerminalStatus(status: ExecutionStreamStatus): boolean {
   return status === 'complete' || status === 'error' || status === 'cancelled'
@@ -71,6 +75,7 @@ export const GET = withRouteHandler(
         return NextResponse.json({ error: 'Run buffer not found or expired' }, { status: 404 })
       }
       const { meta } = metaResult
+      const legacyProducer = meta.protocolVersion !== EXECUTION_STREAM_PROTOCOL_VERSION
 
       if (meta.workflowId && meta.workflowId !== workflowId) {
         return NextResponse.json({ error: 'Run does not belong to this workflow' }, { status: 403 })
@@ -86,11 +91,17 @@ export const GET = withRouteHandler(
       const encoder = new TextEncoder()
 
       let closed = false
+      let unsubscribe: (() => void) | undefined
+      let wakeStream: (() => void) | undefined
 
       const stream = new ReadableStream<Uint8Array>({
         async start(controller) {
           let lastEventId = fromEventId
-          const pollDeadline = Date.now() + MAX_POLL_DURATION_MS
+          let wakePending = false
+          let signalError: Error | undefined
+          let lastHeartbeatAt = Date.now()
+          const deliveredActiveInvocationKeys = new Set<string>()
+          const settledBlockInvocationKeys = new Set<string>()
 
           const enqueue = (text: string) => {
             if (closed) return
@@ -131,55 +142,150 @@ export const GET = withRouteHandler(
               if (closed) break
               entry.event.eventId = entry.eventId
               enqueue(formatSSEEvent(entry.event))
+              if (entry.event.type === 'block:started') {
+                const invocationKey = getBlockInvocationKey(entry.event.data)
+                deliveredActiveInvocationKeys.add(invocationKey)
+                settledBlockInvocationKeys.delete(invocationKey)
+              } else if (
+                entry.event.type === 'block:completed' ||
+                entry.event.type === 'block:error'
+              ) {
+                const invocationKey = getBlockInvocationKey(entry.event.data)
+                deliveredActiveInvocationKeys.delete(invocationKey)
+                settledBlockInvocationKeys.add(invocationKey)
+              }
               lastEventId = entry.eventId
               sawTerminalEvent ||= isTerminalEvent(entry.event)
             }
             return sawTerminalEvent
           }
 
+          const enqueueActiveBlockStarts = (activeBlockStarts = meta.activeBlockStarts ?? []) => {
+            const snapshotInvocationKeys = new Set<string>()
+            for (const active of activeBlockStarts) {
+              const invocationKey = getBlockInvocationKey(active.data)
+              snapshotInvocationKeys.add(invocationKey)
+              if (
+                active.eventId > lastEventId ||
+                deliveredActiveInvocationKeys.has(invocationKey) ||
+                settledBlockInvocationKeys.has(invocationKey)
+              ) {
+                continue
+              }
+              enqueue(
+                formatSSEEvent({
+                  type: 'block:started',
+                  timestamp: new Date().toISOString(),
+                  executionId,
+                  workflowId,
+                  data: active.data,
+                })
+              )
+              deliveredActiveInvocationKeys.add(invocationKey)
+            }
+            for (const invocationKey of settledBlockInvocationKeys) {
+              if (!snapshotInvocationKeys.has(invocationKey)) {
+                settledBlockInvocationKeys.delete(invocationKey)
+              }
+            }
+          }
+
           const closeWithDone = () => {
             enqueue('data: [DONE]\n\n')
             if (!closed) controller.close()
           }
 
-          /**
-           * Terminal metadata is the authoritative end-of-run signal. The
-           * happy path writes it atomically with the terminal event, and a run
-           * whose terminal event could not be buffered records the status on
-           * its own — so metadata without a matching event means the buffer
-           * degraded, not that the run is still going. End the stream cleanly:
-           * the reader has already received every event that was buffered, and
-           * failing here would turn a degraded replay into a broken one.
-           */
+          const waitForWake = (): Promise<'signal' | 'heartbeat' | 'legacy-poll'> => {
+            if (wakePending) {
+              wakePending = false
+              return Promise.resolve('signal')
+            }
+            return new Promise<'signal' | 'heartbeat' | 'legacy-poll'>((resolve) => {
+              const heartbeatTimer = setTimeout(
+                () => {
+                  wakeStream = undefined
+                  resolve(legacyProducer ? 'legacy-poll' : 'heartbeat')
+                },
+                legacyProducer ? LEGACY_REPLAY_INTERVAL_MS : HEARTBEAT_INTERVAL_MS
+              )
+              wakeStream = () => {
+                clearTimeout(heartbeatTimer)
+                resolve('signal')
+              }
+            })
+          }
+
+          const signal = (reason: ExecutionSignalReason) => {
+            if (reason === 'unavailable') {
+              signalError = new Error('Execution signal subscription became unavailable')
+            }
+            if (wakeStream) {
+              const resolve = wakeStream
+              wakeStream = undefined
+              resolve()
+              return
+            }
+            wakePending = true
+          }
+
           const closeAfterTerminalEvent = (events: ExecutionEventEntry[]) => {
             if (!enqueueEvents(events)) {
               logger.warn('Execution reached terminal metadata without a terminal event', {
                 executionId,
               })
+              enqueue(
+                formatSSEEvent({
+                  type: 'execution:error',
+                  timestamp: new Date().toISOString(),
+                  executionId,
+                  workflowId,
+                  data: {
+                    error:
+                      'Execution reached a terminal state, but its final event could not be recovered',
+                    duration: 0,
+                  },
+                })
+              )
             }
             closeWithDone()
           }
 
           try {
+            const subscribed = await getExecutionSignalHub().subscribe(executionId, signal)
+            if (closed) {
+              subscribed()
+              return
+            }
+            unsubscribe = subscribed
+            const initialMeta = await readExecutionMetaState(executionId)
+            if (initialMeta.status === 'unavailable') {
+              throw new Error(`Execution metadata unavailable: ${initialMeta.error}`)
+            }
+            if (initialMeta.status === 'found' && fromEventId > 0) {
+              enqueueActiveBlockStarts(initialMeta.meta.activeBlockStarts)
+            }
             const events = await readEventsOrThrow(lastEventId)
             if (enqueueEvents(events)) {
               closeWithDone()
               return
             }
 
-            const currentMeta = await readExecutionMetaState(executionId)
-            if (currentMeta.status === 'unavailable') {
-              throw new Error(`Execution metadata unavailable: ${currentMeta.error}`)
-            }
+            const currentMeta = initialMeta
             if (currentMeta.status === 'missing' || isTerminalStatus(currentMeta.meta.status)) {
               const finalEvents = await readEventsOrThrow(lastEventId)
               closeAfterTerminalEvent(finalEvents)
               return
             }
 
-            while (!closed && Date.now() < pollDeadline) {
-              await sleep(POLL_INTERVAL_MS)
+            while (!closed) {
+              const wakeReason = await waitForWake()
               if (closed) return
+              if (signalError) throw signalError
+              if (wakeReason === 'heartbeat') {
+                enqueue(`: ping ${Date.now()}\n\n`)
+                lastHeartbeatAt = Date.now()
+                continue
+              }
 
               const newEvents = await readEventsOrThrow(lastEventId)
               if (enqueueEvents(newEvents)) {
@@ -187,20 +293,28 @@ export const GET = withRouteHandler(
                 return
               }
 
-              const polledMeta = await readExecutionMetaState(executionId)
-              if (polledMeta.status === 'unavailable') {
-                throw new Error(`Execution metadata unavailable: ${polledMeta.error}`)
+              const signalledMeta = await readExecutionMetaState(executionId)
+              if (signalledMeta.status === 'unavailable') {
+                throw new Error(`Execution metadata unavailable: ${signalledMeta.error}`)
               }
-              if (polledMeta.status === 'missing' || isTerminalStatus(polledMeta.meta.status)) {
+              if (signalledMeta.status === 'found') {
+                enqueueActiveBlockStarts(signalledMeta.meta.activeBlockStarts)
+              }
+              if (
+                signalledMeta.status === 'missing' ||
+                isTerminalStatus(signalledMeta.meta.status)
+              ) {
                 const finalEvents = await readEventsOrThrow(lastEventId)
                 closeAfterTerminalEvent(finalEvents)
                 return
               }
-            }
-
-            if (!closed) {
-              logger.warn('Reconnection stream poll deadline reached', { executionId })
-              throw new Error('Execution stream ended before a terminal event was available')
+              if (
+                wakeReason === 'legacy-poll' &&
+                Date.now() - lastHeartbeatAt >= HEARTBEAT_INTERVAL_MS
+              ) {
+                enqueue(`: ping ${Date.now()}\n\n`)
+                lastHeartbeatAt = Date.now()
+              }
             }
           } catch (error) {
             logger.error('Error in reconnection stream', {
@@ -212,10 +326,17 @@ export const GET = withRouteHandler(
                 controller.error(error)
               } catch {}
             }
+          } finally {
+            unsubscribe?.()
+            unsubscribe = undefined
           }
         },
         cancel() {
           closed = true
+          wakeStream?.()
+          wakeStream = undefined
+          unsubscribe?.()
+          unsubscribe = undefined
           logger.info('Client disconnected from reconnection stream', { executionId })
         },
       })

apps/sim/app/workspace/[workspaceId]/w/[workflowId]/hooks/use-workflow-execution.ts

@@ -2403,7 +2403,12 @@ export function useWorkflowExecution() {
           .getWorkflowExecution(reconnectWorkflowId)
         const currentId = executionState?.currentExecutionId ?? null
         if (currentId && currentId !== capturedExecutionId) return
-        finishRunningEntries(reconnectWorkflowId, capturedExecutionId)
+        handleExecutionErrorConsole({
+          workflowId: reconnectWorkflowId,
+          executionId: capturedExecutionId,
+          error: 'Execution state is no longer available after reconnect',
+          blockLogs: [],
+        })
         setCurrentExecutionId(reconnectWorkflowId, null)
         setIsExecuting(reconnectWorkflowId, false)
         setActiveBlocks(reconnectWorkflowId, new Set())

apps/sim/executor/execution/block-retry.ts

@@ -1,5 +1,6 @@
 import { findCause } from '@sim/utils/errors'
 import { type BlockRetryConfig, resolveBlockRetryConfig } from '@sim/workflow-types/workflow'
+import { isNonRetryableExecutionError } from '@/lib/execution/non-retryable-error'
 import { isRetryEligibleBlock } from '@/lib/workflows/blocks/retry-eligibility'
 import { ChildWorkflowError } from '@/executor/errors/child-workflow-error'
 import type { SerializedBlock } from '@/serializer/types'
@@ -35,6 +36,7 @@ const isAbortError = (value: unknown): value is Error =>
  * `name`, leaving the original classification only on `cause`.
  */
 export function isRetryableBlockError(error: unknown): boolean {
+  if (isNonRetryableExecutionError(error)) return false
   if (findCause(error, ChildWorkflowError.isChildWorkflowError)) return false
   if (findCause(error, isAbortError)) return false
   return true

apps/sim/executor/execution/engine.test.ts

@@ -6,30 +6,24 @@ import { sleep } from '@sim/utils/helpers'
 import { afterEach, beforeEach, describe, expect, it, type Mock, vi } from 'vitest'
 import { createTimeoutAbortController, getRemainingExecutionMs } from '@/lib/core/execution-limits'
 
-const { mockCancellationSubscribers } = vi.hoisted(() => ({
+const { mockCancellationSubscribers, mockIsExecutionCancelled } = vi.hoisted(() => ({
   mockCancellationSubscribers: new Set<(event: { executionId: string }) => void>(),
+  mockIsExecutionCancelled: vi.fn(),
 }))
 
 vi.mock('@/lib/execution/cancellation', () => ({
-  isExecutionCancelled: vi.fn(),
-  isRedisCancellationEnabled: vi.fn(),
-  getCancellationChannel: () => ({
-    publish: (event: { executionId: string }) => {
-      for (const handler of mockCancellationSubscribers) handler(event)
-    },
-    subscribe: (handler: (event: { executionId: string }) => void) => {
-      mockCancellationSubscribers.add(handler)
-      return () => {
-        mockCancellationSubscribers.delete(handler)
-      }
-    },
-    dispose: () => {
-      mockCancellationSubscribers.clear()
-    },
-  }),
+  subscribeToExecutionCancellation: async (executionId: string, onCancelled: () => void) => {
+    const handler = (event: { executionId: string }) => {
+      if (event.executionId === executionId) onCancelled()
+    }
+    mockCancellationSubscribers.add(handler)
+    if (await mockIsExecutionCancelled(executionId)) onCancelled()
+    return () => {
+      mockCancellationSubscribers.delete(handler)
+    }
+  },
 }))
 
-import { isExecutionCancelled, isRedisCancellationEnabled } from '@/lib/execution/cancellation'
 import { EDGE } from '@/executor/constants'
 import type { DAG, DAGNode } from '@/executor/dag/builder'
 import type { EdgeManager } from '@/executor/execution/edge-manager'
@@ -150,8 +144,7 @@ describe('ExecutionEngine', () => {
   beforeEach(() => {
     vi.clearAllMocks()
     mockCancellationSubscribers.clear()
-    ;(isExecutionCancelled as Mock).mockResolvedValue(false)
-    ;(isRedisCancellationEnabled as Mock).mockReturnValue(false)
+    mockIsExecutionCancelled.mockResolvedValue(false)
   })
 
   afterEach(() => {
@@ -608,8 +601,7 @@ describe('ExecutionEngine', () => {
 
   describe('Cancellation via Redis', () => {
     it('aborts the active execution signal without dropping its deadline', async () => {
-      ;(isRedisCancellationEnabled as Mock).mockReturnValue(true)
-      ;(isExecutionCancelled as Mock).mockResolvedValue(false)
+      mockIsExecutionCancelled.mockResolvedValue(true)
       const timeoutController = createTimeoutAbortController(60_000)
       const startNode = createMockNode('start', 'starter')
       const context = createMockContext({
@@ -627,20 +619,15 @@ describe('ExecutionEngine', () => {
         expect(context.abortSignal).not.toBe(timeoutController.signal)
         expect(getRemainingExecutionMs(context.abortSignal)).toBeGreaterThan(0)
 
-        for (const handler of mockCancellationSubscribers) {
-          handler({ executionId: 'pubsub-signal-execution' })
-        }
-
-        expect(context.abortSignal?.aborted).toBe(true)
         await expect(engine.run('start')).resolves.toMatchObject({ status: 'cancelled' })
+        expect(context.abortSignal?.aborted).toBe(true)
       } finally {
         timeoutController.cleanup()
       }
     })
 
-    it('should check Redis for cancellation when enabled', async () => {
-      ;(isRedisCancellationEnabled as Mock).mockReturnValue(true)
-      ;(isExecutionCancelled as Mock).mockResolvedValue(false)
+    it('checks the durable cancellation state when subscribing', async () => {
+      mockIsExecutionCancelled.mockResolvedValue(false)
 
       const startNode = createMockNode('start', 'starter')
       const dag = createMockDAG([startNode])
@@ -651,13 +638,11 @@ describe('ExecutionEngine', () => {
       const engine = new ExecutionEngine(context, dag, edgeManager, nodeOrchestrator)
       await engine.run('start')
 
-      expect(isExecutionCancelled as Mock).toHaveBeenCalled()
+      expect(mockIsExecutionCancelled).toHaveBeenCalled()
     })
 
     it('should stop execution when Redis reports cancellation', async () => {
-      ;(isRedisCancellationEnabled as Mock).mockReturnValue(true)
-
-      ;(isExecutionCancelled as Mock).mockResolvedValue(true)
+      mockIsExecutionCancelled.mockResolvedValue(true)
 
       const nodes = Array.from({ length: 5 }, (_, i) => createMockNode(`node${i}`, 'function'))
       for (let i = 0; i < nodes.length - 1; i++) {
@@ -681,8 +666,7 @@ describe('ExecutionEngine', () => {
     })
 
     it('wakes from a slow in-flight node when a pub/sub cancellation arrives', async () => {
-      ;(isRedisCancellationEnabled as Mock).mockReturnValue(true)
-      ;(isExecutionCancelled as Mock).mockResolvedValue(false)
+      mockIsExecutionCancelled.mockResolvedValue(false)
 
       const startNode = createMockNode('start', 'starter')
       const slowNode = createMockNode('slow', 'function')
@@ -711,8 +695,7 @@ describe('ExecutionEngine', () => {
     })
 
     it('ignores pub/sub events targeting other executions', async () => {
-      ;(isRedisCancellationEnabled as Mock).mockReturnValue(true)
-      ;(isExecutionCancelled as Mock).mockResolvedValue(false)
+      mockIsExecutionCancelled.mockResolvedValue(false)
 
       const startNode = createMockNode('start', 'starter')
       const dag = createMockDAG([startNode])
@@ -732,26 +715,25 @@ describe('ExecutionEngine', () => {
     })
 
     it('unsubscribes from the cancellation channel after run completes', async () => {
-      ;(isRedisCancellationEnabled as Mock).mockReturnValue(true)
-      ;(isExecutionCancelled as Mock).mockResolvedValue(false)
+      mockIsExecutionCancelled.mockResolvedValue(false)
 
       const startNode = createMockNode('start', 'starter')
       const dag = createMockDAG([startNode])
       const context = createMockContext({ executionId: 'cleanup-execution' })
       const edgeManager = createMockEdgeManager()
-      const nodeOrchestrator = createMockNodeOrchestrator()
+      const nodeOrchestrator = createMockNodeOrchestrator(20)
 
       const engine = new ExecutionEngine(context, dag, edgeManager, nodeOrchestrator)
-      expect(mockCancellationSubscribers.size).toBe(1)
+      const runPromise = engine.run('start')
+      await vi.waitFor(() => expect(mockCancellationSubscribers.size).toBe(1))
 
-      await engine.run('start')
+      await runPromise
 
       expect(mockCancellationSubscribers.size).toBe(0)
     })
 
     it('honours the durable backstop when cancelled before subscribing', async () => {
-      ;(isRedisCancellationEnabled as Mock).mockReturnValue(true)
-      ;(isExecutionCancelled as Mock).mockResolvedValue(true)
+      mockIsExecutionCancelled.mockResolvedValue(true)
 
       const startNode = createMockNode('start', 'starter')
       const dag = createMockDAG([startNode])
@@ -767,9 +749,8 @@ describe('ExecutionEngine', () => {
       expect(context.abortSignal?.aborted).toBe(true)
     })
 
-    it('calls isExecutionCancelled once for a run that finishes before the first poll', async () => {
-      ;(isRedisCancellationEnabled as Mock).mockReturnValue(true)
-      ;(isExecutionCancelled as Mock).mockResolvedValue(false)
+    it('reads the durable cancellation flag once when subscribing', async () => {
+      mockIsExecutionCancelled.mockResolvedValue(false)
 
       const startNode = createMockNode('start', 'starter')
       const dag = createMockDAG([startNode])
@@ -780,12 +761,11 @@ describe('ExecutionEngine', () => {
       const engine = new ExecutionEngine(context, dag, edgeManager, nodeOrchestrator)
       await engine.run('start')
 
-      expect((isExecutionCancelled as Mock).mock.calls.length).toBe(1)
+      expect(mockIsExecutionCancelled).toHaveBeenCalledOnce()
     })
 
-    it('cancels a long-running node when only the durable flag reports it', async () => {
-      ;(isRedisCancellationEnabled as Mock).mockReturnValue(true)
-      ;(isExecutionCancelled as Mock).mockResolvedValue(false)
+    it('does not poll the durable flag while a node is running', async () => {
+      mockIsExecutionCancelled.mockResolvedValue(false)
 
       let releaseNode = () => {}
       const nodeReleased = new Promise<void>((resolve) => {
@@ -803,9 +783,7 @@ describe('ExecutionEngine', () => {
       ;(nodeOrchestrator.executeNode as Mock).mockImplementation(
         async (_ctx: ExecutionContext, nodeId: string) => {
           if (nodeId === 'slow') {
-            // Cancel durably with no pub/sub event, mirroring a cancel served by another replica
-            // whose published event never reaches this engine.
-            ;(isExecutionCancelled as Mock).mockResolvedValue(true)
+            mockIsExecutionCancelled.mockResolvedValue(true)
             await nodeReleased
           }
           return { nodeId, output: {}, isFinalOutput: false }
@@ -815,15 +793,20 @@ describe('ExecutionEngine', () => {
       const engine = new ExecutionEngine(context, dag, edgeManager, nodeOrchestrator)
       const runPromise = engine.run('start')
 
-      await vi.waitFor(() => expect(context.abortSignal?.aborted).toBe(true), { timeout: 3000 })
+      await vi.waitFor(() => expect(mockCancellationSubscribers.size).toBe(1))
+      await sleep(25)
+      expect(context.abortSignal?.aborted).toBe(false)
+      expect(mockIsExecutionCancelled).toHaveBeenCalledOnce()
+      for (const handler of mockCancellationSubscribers) {
+        handler({ executionId: 'redis-poll-execution' })
+      }
       releaseNode()
 
       await expect(runPromise).resolves.toMatchObject({ success: false, status: 'cancelled' })
     })
 
-    it('leaves no polling timer behind once the run settles', async () => {
-      ;(isRedisCancellationEnabled as Mock).mockReturnValue(true)
-      ;(isExecutionCancelled as Mock).mockResolvedValue(false)
+    it('leaves no cancellation timer behind once the run settles', async () => {
+      mockIsExecutionCancelled.mockResolvedValue(false)
       vi.useFakeTimers()
 
       const startNode = createMockNode('start', 'starter')
@@ -835,12 +818,12 @@ describe('ExecutionEngine', () => {
       const engine = new ExecutionEngine(context, dag, edgeManager, nodeOrchestrator)
       await engine.run('start')
 
-      const callsAtCompletion = (isExecutionCancelled as Mock).mock.calls.length
+      const callsAtCompletion = mockIsExecutionCancelled.mock.calls.length
       // Well past several poll intervals: a surviving timer would add calls here.
       await vi.advanceTimersByTimeAsync(5_000)
 
       expect(vi.getTimerCount()).toBe(0)
-      expect((isExecutionCancelled as Mock).mock.calls.length).toBe(callsAtCompletion)
+      expect(mockIsExecutionCancelled.mock.calls.length).toBe(callsAtCompletion)
     })
   })
 
@@ -1811,8 +1794,7 @@ describe('ExecutionEngine', () => {
     })
 
     it('should cache Redis cancellation result', async () => {
-      ;(isRedisCancellationEnabled as Mock).mockReturnValue(true)
-      ;(isExecutionCancelled as Mock).mockResolvedValue(true)
+      mockIsExecutionCancelled.mockResolvedValue(true)
 
       const nodes = Array.from({ length: 5 }, (_, i) => createMockNode(`node${i}`, 'function'))
       const dag = createMockDAG(nodes)
@@ -1823,7 +1805,7 @@ describe('ExecutionEngine', () => {
       const engine = new ExecutionEngine(context, dag, edgeManager, nodeOrchestrator)
       await engine.run('node0')
 
-      expect((isExecutionCancelled as Mock).mock.calls.length).toBeLessThanOrEqual(3)
+      expect(mockIsExecutionCancelled.mock.calls.length).toBeLessThanOrEqual(3)
     })
   })
 })

apps/sim/executor/execution/engine.ts

@@ -1,11 +1,7 @@
 import { createLogger, type Logger } from '@sim/logger'
 import { toError } from '@sim/utils/errors'
 import { combineExecutionAbortSignals } from '@/lib/core/execution-limits'
-import {
-  getCancellationChannel,
-  isExecutionCancelled,
-  isRedisCancellationEnabled,
-} from '@/lib/execution/cancellation'
+import { subscribeToExecutionCancellation } from '@/lib/execution/cancellation'
 import { BlockType, EDGE } from '@/executor/constants'
 import type { DAG } from '@/executor/dag/builder'
 import type { EdgeManager } from '@/executor/execution/edge-manager'
@@ -25,9 +21,6 @@ import { projectResolvedSecretDiagnosticError } from '@/executor/utils/resolved-
 
 const logger = createLogger('ExecutionEngine')
 
-/** Cadence of the Redis fallback poll that covers a lost pub/sub cancellation. */
-const CANCELLATION_POLL_INTERVAL_MS = 500
-
 export class ExecutionEngine {
   private readyQueue: string[] = []
   private executing = new Set<Promise<void>>()
@@ -45,8 +38,6 @@ export class ExecutionEngine {
   private cancellationController = new AbortController()
   private abortSignalListener: (() => void) | null = null
   private cancellationUnsubscribe: (() => void) | null = null
-  private cancellationPollTimer: ReturnType<typeof setInterval> | null = null
-  private cancellationPollInFlight = false
   private execLogger: Logger
 
   constructor(
@@ -69,15 +60,13 @@ export class ExecutionEngine {
         : [this.cancellationController.signal]
     )
     this.initializeAbortHandler()
-    this.subscribeToCancellationChannel()
   }
 
-  private subscribeToCancellationChannel(): void {
+  private async subscribeToCancellationSignal(): Promise<void> {
     if (!this.context.executionId) return
     const executionId = this.context.executionId
-    this.cancellationUnsubscribe = getCancellationChannel().subscribe((event) => {
-      if (event.executionId !== executionId) return
-      this.execLogger.info('Execution cancelled via pub/sub', { executionId })
+    this.cancellationUnsubscribe = await subscribeToExecutionCancellation(executionId, () => {
+      this.execLogger.info('Execution cancelled via Redis signal', { executionId })
       this.signalCancelled()
     })
   }
@@ -102,7 +91,6 @@ export class ExecutionEngine {
   private signalCancelled(reason: unknown = new DOMException('user', 'AbortError')): void {
     if (this.cancelledFlag) return
     this.cancelledFlag = true
-    this.stopCancellationPolling()
     if (!this.cancellationController.signal.aborted) {
       this.cancellationController.abort(reason)
     }
@@ -113,77 +101,11 @@ export class ExecutionEngine {
     return this.cancelledFlag
   }
 
-  /** Reads the durable cancellation flag; false when Redis is not the cancellation store. */
-  private async readDurableCancellation(): Promise<boolean> {
-    const executionId = this.context.executionId
-    if (!executionId || !isRedisCancellationEnabled()) return false
-    return isExecutionCancelled(executionId)
-  }
-
-  /** Catches cancellations published before this engine subscribed (e.g. resume from snapshot). */
-  private async checkCancellationBackstop(): Promise<void> {
-    if (!(await this.readDurableCancellation())) return
-    this.execLogger.info('Execution already cancelled at engine start (Redis backstop)', {
-      executionId: this.context.executionId,
-    })
-    this.signalCancelled()
-  }
-
-  /**
-   * Polls the durable flag for the life of the run.
-   *
-   * Pub/sub delivery is the fast path but is at-most-once: a dropped subscriber connection, or a
-   * publish that races this engine reaching its own last block, would otherwise let a cancelled
-   * run finish as successful. `markExecutionCancelled` writes the durable key before publishing
-   * precisely so a reader that misses the event can still observe the cancellation.
-   *
-   * This and {@link checkCancellationBackstop} are the only places a durable cancellation becomes
-   * run status. A block handler or orchestrator that reads the flag itself and then returns
-   * normally leaves `cancelledFlag` false, which reports a cancelled run as successful. Handlers
-   * that abort their own I/O off `ctx.abortSignal` are fine: that surfaces as a throw, which the
-   * cancelled branch of `run` classifies.
-   */
-  private startCancellationPolling(): void {
-    if (this.cancelledFlag || !this.context.executionId || !isRedisCancellationEnabled()) return
-    this.cancellationPollTimer = setInterval(() => {
-      if (this.cancellationPollInFlight) return
-      this.cancellationPollInFlight = true
-      void this.pollDurableCancellation()
-        .catch((error) => {
-          this.execLogger.warn('Durable cancellation poll failed', {
-            executionId: this.context.executionId,
-            error: toError(error).message,
-          })
-        })
-        .finally(() => {
-          this.cancellationPollInFlight = false
-        })
-    }, CANCELLATION_POLL_INTERVAL_MS)
-  }
-
-  private async pollDurableCancellation(): Promise<void> {
-    const cancelled = await this.readDurableCancellation()
-    // `signalCancelled` and `cleanup` both null the timer, so it doubles as "polling is still
-    // live" — a run that settled while this read was in flight must not be cancelled after.
-    if (!cancelled || !this.cancellationPollTimer) return
-    this.execLogger.info('Execution cancelled via Redis poll', {
-      executionId: this.context.executionId,
-    })
-    this.signalCancelled()
-  }
-
-  private stopCancellationPolling(): void {
-    if (!this.cancellationPollTimer) return
-    clearInterval(this.cancellationPollTimer)
-    this.cancellationPollTimer = null
-  }
-
   async run(triggerBlockId?: string): Promise<ExecutionResult> {
     const startTime = performance.now()
     try {
       this.initializeQueue(triggerBlockId)
-      await this.checkCancellationBackstop()
-      this.startCancellationPolling()
+      await this.subscribeToCancellationSignal()
 
       while (this.hasWork()) {
         if (this.checkCancellation() || this.errorFlag || this.stoppedEarlyFlag) {
@@ -273,7 +195,6 @@ export class ExecutionEngine {
   }
 
   private cleanup(): void {
-    this.stopCancellationPolling()
     if (this.abortSignalListener && this.context.abortSignal) {
       this.context.abortSignal.removeEventListener('abort', this.abortSignalListener)
       this.abortSignalListener = null

apps/sim/executor/handlers/condition/condition-handler.test.ts

@@ -3,6 +3,7 @@
  */
 import { loggerMock } from '@sim/testing'
 import { beforeEach, describe, expect, it, vi } from 'vitest'
+import { NonRetryableExecutionError } from '@/lib/execution/non-retryable-error'
 import { BlockType } from '@/executor/constants'
 import { ConditionBlockHandler } from '@/executor/handlers/condition/condition-handler'
 import type { BlockState, ExecutionContext } from '@/executor/types'
@@ -550,6 +551,59 @@ describe('ConditionBlockHandler', () => {
       expect(mockExecuteTool).toHaveBeenCalledOnce()
     })
 
+    it('does not replay an indeterminate Function execution', async () => {
+      mockExecuteTool.mockResolvedValue({
+        success: false,
+        error: 'The sandbox may have started this Function',
+        retryable: false,
+      })
+
+      await expect(
+        handler.execute(mockContext, mockBlock, { conditions: JSON.stringify(manyConditions) })
+      ).rejects.toMatchObject({
+        name: 'NonRetryableExecutionError',
+        retryable: false,
+      })
+      expect(mockExecuteTool).toHaveBeenCalledOnce()
+    })
+
+    it('does not enter per-branch fallback when the batched Function call throws indeterminate', async () => {
+      mockExecuteTool.mockRejectedValueOnce(
+        new NonRetryableExecutionError('The sandbox may have started this Function')
+      )
+
+      await expect(
+        handler.execute(mockContext, mockBlock, { conditions: JSON.stringify(manyConditions) })
+      ).rejects.toMatchObject({
+        name: 'NonRetryableExecutionError',
+        retryable: false,
+      })
+      expect(mockExecuteTool).toHaveBeenCalledOnce()
+    })
+
+    it('preserves an indeterminate result during individual fallback', async () => {
+      mockExecuteTool
+        .mockResolvedValueOnce({
+          success: false,
+          error: 'Invalid JavaScript syntax: Unexpected token',
+        })
+        .mockResolvedValueOnce({
+          success: false,
+          error: 'The sandbox may have started this Function',
+          retryable: false,
+        })
+
+      await expect(
+        handler.execute(mockContext, mockBlock, { conditions: JSON.stringify(manyConditions) })
+      ).rejects.toMatchObject({
+        cause: expect.objectContaining({
+          name: 'NonRetryableExecutionError',
+          retryable: false,
+        }),
+      })
+      expect(mockExecuteTool).toHaveBeenCalledTimes(2)
+    })
+
     it('falls back when the batch reports a branch index outside the list', async () => {
       mockExecuteTool.mockResolvedValueOnce(matchedAt(99))
       mockExecuteTool.mockResolvedValueOnce({ success: true, output: { result: true } })

apps/sim/executor/handlers/condition/condition-handler.ts

@@ -1,6 +1,10 @@
 import { createLogger } from '@sim/logger'
 import { getErrorMessage, toError } from '@sim/utils/errors'
 import { normalizeStringRecord, normalizeWorkflowVariables } from '@/lib/core/utils/records'
+import {
+  isNonRetryableExecutionError,
+  NonRetryableExecutionError,
+} from '@/lib/execution/non-retryable-error'
 import { isElseConditionTitle } from '@/lib/workflows/conditions'
 import type { BlockOutput } from '@/blocks/types'
 import { BlockType, DEFAULTS, EDGE } from '@/executor/constants'
@@ -31,7 +35,7 @@ type ConditionEvaluation =
   | { status: 'matched'; index: number }
   | { status: 'no-match' }
   | { status: 'expression-threw'; index: number; message: string }
-  | { status: 'no-verdict'; message: string; timedOut: boolean }
+  | { status: 'no-verdict'; message: string; retryable: boolean; timedOut: boolean }
 
 /**
  * Wraps one expression as a boolean test, on its own line so a trailing line
@@ -151,7 +155,12 @@ async function evaluateConditionList(
 
   if (!result.success) {
     const message = result.error ?? 'Condition evaluation failed'
-    return { status: 'no-verdict', message, timedOut: isTimeoutFailure(result.error) }
+    return {
+      status: 'no-verdict',
+      message,
+      retryable: result.retryable !== false,
+      timedOut: isTimeoutFailure(result.error),
+    }
   }
 
   const output = result.output?.result
@@ -159,6 +168,7 @@ async function evaluateConditionList(
     return {
       status: 'no-verdict',
       message: 'Condition evaluation returned no verdict',
+      retryable: true,
       timedOut: false,
     }
   }
@@ -174,6 +184,7 @@ async function evaluateConditionList(
       return {
         status: 'no-verdict',
         message: 'Condition evaluation reported an unknown branch',
+        retryable: true,
         timedOut: false,
       }
     }
@@ -191,6 +202,7 @@ async function evaluateConditionList(
     return {
       status: 'no-verdict',
       message: 'Condition evaluation returned an unrecognized verdict',
+      retryable: true,
       timedOut: false,
     }
   }
@@ -201,6 +213,7 @@ async function evaluateConditionList(
     return {
       status: 'no-verdict',
       message: 'Condition evaluation matched an unknown branch',
+      retryable: true,
       timedOut: false,
     }
   }
@@ -224,14 +237,17 @@ async function evaluateSingleCondition(
   const result = await runConditionCode(ctx, code, currentNodeId)
 
   if (!result.success) {
+    if (result.retryable === false) {
+      throw new NonRetryableExecutionError(result.error ?? 'Condition evaluation is indeterminate')
+    }
     throw new Error(result.error ?? 'Condition evaluation failed')
   }
 
   return Boolean(result.output?.result)
 }
 
-function conditionError(condition: ConditionEntry, message: string): Error {
-  return new Error(`Evaluation error in condition "${condition.title}": ${message}`)
+function conditionError(condition: ConditionEntry, message: string, cause?: unknown): Error {
+  return new Error(`Evaluation error in condition "${condition.title}": ${message}`, { cause })
 }
 
 /**
@@ -408,9 +424,11 @@ export class ConditionBlockHandler implements BlockHandler {
     try {
       evaluation = await evaluateConditionList(ctx, expressions, evalContext, currentNodeId)
     } catch (error) {
+      if (isNonRetryableExecutionError(error)) throw error
       evaluation = {
         status: 'no-verdict',
         message: getErrorMessage(error, 'Condition evaluation failed'),
+        retryable: true,
         timedOut: false,
       }
     }
@@ -424,6 +442,11 @@ export class ConditionBlockHandler implements BlockHandler {
         logger.error('Failed to evaluate condition', { conditionCount: conditions.length })
         throw conditionError(conditions[evaluation.index], evaluation.message)
       case 'no-verdict':
+        if (!evaluation.retryable) {
+          throw new NonRetryableExecutionError(
+            `Evaluation error in condition "${conditions[0].title}": ${evaluation.message}`
+          )
+        }
         // Retrying one branch at a time is what recovers a batch the sandbox
         // could not parse. It is the wrong move for a stalled transport, which
         // would re-run every branch against the same stall, or for a cancelled
@@ -458,7 +481,11 @@ export class ConditionBlockHandler implements BlockHandler {
         if (conditionMet) return condition
       } catch (error) {
         logger.error('Failed to evaluate condition', { errorName: toError(error).name })
-        throw conditionError(condition, getErrorMessage(error, 'Condition evaluation failed'))
+        throw conditionError(
+          condition,
+          getErrorMessage(error, 'Condition evaluation failed'),
+          error
+        )
       }
     }
 

apps/sim/executor/handlers/function/function-handler.ts

@@ -7,6 +7,7 @@ import {
 } from '@/lib/core/utils/records'
 import { DEFAULT_EXECUTION_TIMEOUT_MS } from '@/lib/execution/constants'
 import { DEFAULT_CODE_LANGUAGE } from '@/lib/execution/languages'
+import { NonRetryableExecutionError } from '@/lib/execution/non-retryable-error'
 import { mergeFileKeys, mergeLargeValueKeys } from '@/lib/execution/payloads/access-keys'
 import { BlockType } from '@/executor/constants'
 import type { BlockHandler, ExecutionContext } from '@/executor/types'
@@ -110,6 +111,9 @@ export class FunctionBlockHandler implements BlockHandler {
     const result = await executeTool('function_execute', toolParams, { executionContext: ctx })
 
     if (!result.success) {
+      if (result.retryable === false) {
+        throw new NonRetryableExecutionError(result.error || 'Function execution is indeterminate')
+      }
       throw new Error(result.error || 'Function execution failed')
     }
 

apps/sim/executor/handlers/mothership/mothership-handler.test.ts

@@ -35,8 +35,6 @@ const {
   mockBuildAPIUrl,
   mockExtractAPIErrorMessage,
   mockGenerateId,
-  mockIsExecutionCancelled,
-  mockIsRedisCancellationEnabled,
   mockReadUserFileContent,
 } = vi.hoisted(() => ({
   mockAreModelSafeWorkspaceFileKeys: vi.fn(),
@@ -44,8 +42,6 @@ const {
   mockBuildAPIUrl: vi.fn(),
   mockExtractAPIErrorMessage: vi.fn(),
   mockGenerateId: vi.fn(),
-  mockIsExecutionCancelled: vi.fn(),
-  mockIsRedisCancellationEnabled: vi.fn(),
   mockReadUserFileContent: vi.fn(),
 }))
 
@@ -65,11 +61,6 @@ vi.mock('@sim/utils/id', () => ({
   generateId: mockGenerateId,
 }))
 
-vi.mock('@/lib/execution/cancellation', () => ({
-  isExecutionCancelled: mockIsExecutionCancelled,
-  isRedisCancellationEnabled: mockIsRedisCancellationEnabled,
-}))
-
 vi.mock('@/lib/execution/payloads/materialization.server', () => ({
   readUserFileContent: mockReadUserFileContent,
 }))
@@ -158,9 +149,6 @@ describe('MothershipBlockHandler', () => {
     mockBuildAPIUrl.mockReturnValue(new URL('/api/mothership/execute', 'http://localhost:3000'))
     mockExtractAPIErrorMessage.mockResolvedValue('boom')
     mockGenerateId.mockReset()
-    mockIsExecutionCancelled.mockReset()
-    mockIsRedisCancellationEnabled.mockReset()
-    mockIsRedisCancellationEnabled.mockReturnValue(false)
     mockReadUserFileContent.mockReset()
     mockAreModelSafeWorkspaceFileKeys.mockReset()
     mockAreModelSafeWorkspaceFileKeys.mockResolvedValue(true)
@@ -1895,28 +1883,6 @@ describe('MothershipBlockHandler', () => {
     await expect(abortedExecution).resolves.toMatchObject({ name: 'AbortError' })
   })
 
-  it('propagates durable workflow cancellation to the mothership request', async () => {
-    vi.useFakeTimers()
-
-    mockGenerateId.mockReturnValueOnce('chat-uuid')
-    mockGenerateId.mockReturnValueOnce('message-uuid')
-    mockGenerateId.mockReturnValueOnce('request-uuid')
-    mockIsRedisCancellationEnabled.mockReturnValue(true)
-    mockIsExecutionCancelled.mockResolvedValueOnce(false).mockResolvedValueOnce(true)
-
-    fetchMock.mockImplementation((_url: string, options?: RequestInit) =>
-      createAbortableFetchPromise(options?.signal as AbortSignal | undefined)
-    )
-
-    const executionPromise = handler.execute(context, block, { prompt: 'Cancel me durably' })
-    const abortedExecution = executionPromise.catch((error) => error)
-
-    await vi.advanceTimersByTimeAsync(1000)
-
-    await expect(abortedExecution).resolves.toMatchObject({ name: 'AbortError' })
-    expect(mockIsExecutionCancelled).toHaveBeenCalledWith('execution-1')
-  })
-
   it('aborts the mothership request when selected-output streaming is cancelled', async () => {
     context.stream = true
     context.selectedOutputs = [`${block.id}_content`]

apps/sim/executor/handlers/mothership/mothership-handler.ts

@@ -1,5 +1,4 @@
 import { createLogger } from '@sim/logger'
-import { toError } from '@sim/utils/errors'
 import { generateId } from '@sim/utils/id'
 import { isPlainRecord } from '@sim/utils/object'
 import {
@@ -8,7 +7,6 @@ import {
 } from '@/lib/billing/core/billing-attribution'
 import { normalizeSecretMountPolicy } from '@/lib/copilot/secret-mount-policy'
 import { env } from '@/lib/core/config/env'
-import { isExecutionCancelled, isRedisCancellationEnabled } from '@/lib/execution/cancellation'
 import {
   projectModelSchemaAnnotations,
   projectResolvedModelInput,
@@ -55,7 +53,6 @@ const logger = createLogger('MothershipBlockHandler')
 const MOTHERSHIP_INPUT_REFUSAL = 'Mothership input could not be safely projected'
 const MOTHERSHIP_SKILL_SELECTOR_REFUSAL =
   'Mothership skill selector could not be safely projected for display'
-const CANCELLATION_CHECK_INTERVAL_MS = 500
 const MAX_MOTHERSHIP_ATTACHMENT_BYTES = 10 * 1024 * 1024
 const MOTHERSHIP_EXECUTE_STREAM_HEADER = 'X-Mothership-Execute-Stream'
 const MOTHERSHIP_EXECUTE_STREAM_VALUE = 'ndjson'
@@ -896,38 +893,7 @@ export class MothershipBlockHandler implements BlockHandler {
       ctx.abortSignal?.addEventListener('abort', onAbort, { once: true })
     }
 
-    const executionId = ctx.executionId
-    const useRedisCancellation = isRedisCancellationEnabled() && !!executionId
-    let pollInFlight = false
-    const cancellationPoller =
-      useRedisCancellation && executionId
-        ? setInterval(() => {
-            if (pollInFlight || abortController.signal.aborted) {
-              return
-            }
-            pollInFlight = true
-            void isExecutionCancelled(executionId)
-              .then((cancelled) => {
-                if (cancelled && !abortController.signal.aborted) {
-                  abortController.abort('workflow_execution_cancelled')
-                }
-              })
-              .catch((error) => {
-                logger.warn('Failed to poll workflow cancellation for Mothership block', {
-                  blockId: block.id,
-                  executionId,
-                  error: toError(error).message,
-                })
-              })
-              .finally(() => {
-                pollInFlight = false
-              })
-          }, CANCELLATION_CHECK_INTERVAL_MS)
-        : undefined
     const cleanupAbortListeners = () => {
-      if (cancellationPoller) {
-        clearInterval(cancellationPoller)
-      }
       ctx.abortSignal?.removeEventListener('abort', onAbort)
     }
 

apps/sim/executor/handlers/pi/cloud/babysit/backend.test.ts

@@ -31,10 +31,6 @@ const {
 vi.mock('@/lib/execution/remote-sandbox', () => ({
   withPiSandbox: mockWithPiSandbox,
 }))
-vi.mock('@/lib/execution/cancellation', () => ({
-  isRedisCancellationEnabled: () => false,
-  isExecutionCancelled: vi.fn().mockResolvedValue(false),
-}))
 vi.mock('@/lib/data-drains/destinations/utils', () => ({
   sleepUntilAborted: mockSleepUntilAborted,
 }))

apps/sim/executor/handlers/pi/cloud/babysit/backend.ts

@@ -8,7 +8,6 @@ import { createLogger } from '@sim/logger'
 import { getErrorMessage } from '@sim/utils/errors'
 import { truncate } from '@sim/utils/string'
 import { sleepUntilAborted } from '@/lib/data-drains/destinations/utils'
-import { isExecutionCancelled, isRedisCancellationEnabled } from '@/lib/execution/cancellation'
 import { type PiSandboxRunner, withPiSandbox } from '@/lib/execution/remote-sandbox'
 import {
   resolvePiRunLifetimeMs,
@@ -87,7 +86,6 @@ import { getPiProviderId } from '@/providers/pi-providers'
 const logger = createLogger('PiBabysitBackend')
 
 const ROUND_WAIT_MS = 5 * 60 * 1000
-const CANCELLATION_POLL_MS = 5_000
 const CONVERGENCE_WAIT_MS = 2_000
 const CONVERGENCE_ATTEMPTS = 3
 const MAX_CHANGED_FILES = 50
@@ -211,14 +209,12 @@ echo "__PUSHED__=1"`
 
 interface BabysitBackendOptions {
   roundWaitMs: number
-  cancellationPollMs: number
   convergenceWaitMs: number
   convergenceAttempts: number
 }
 
 const DEFAULT_OPTIONS: BabysitBackendOptions = {
   roundWaitMs: ROUND_WAIT_MS,
-  cancellationPollMs: CANCELLATION_POLL_MS,
   convergenceWaitMs: CONVERGENCE_WAIT_MS,
   convergenceAttempts: CONVERGENCE_ATTEMPTS,
 }
@@ -430,43 +426,18 @@ function buildRoundPrompt(
   }
 }
 
-function createCancellationSignal(
-  parent: AbortSignal | undefined,
-  executionId: string | undefined,
-  pollMs: number
-): { signal: AbortSignal; cleanup: () => void } {
+function createCancellationSignal(parent: AbortSignal | undefined): {
+  signal: AbortSignal
+  cleanup: () => void
+} {
   const controller = new AbortController()
   const onAbort = () => controller.abort(parent?.reason ?? 'workflow_abort')
   if (parent?.aborted) onAbort()
   else parent?.addEventListener('abort', onAbort, { once: true })
 
-  let pollInFlight = false
-  const poller =
-    executionId && isRedisCancellationEnabled()
-      ? setInterval(() => {
-          if (pollInFlight || controller.signal.aborted) return
-          pollInFlight = true
-          void isExecutionCancelled(executionId)
-            .then((cancelled) => {
-              if (cancelled && !controller.signal.aborted) {
-                controller.abort('workflow_execution_cancelled')
-              }
-            })
-            .catch((error) => {
-              logger.warn('Failed to poll Babysit execution cancellation', {
-                executionId,
-                error: getErrorMessage(error),
-              })
-            })
-            .finally(() => {
-              pollInFlight = false
-            })
-        }, pollMs)
-      : undefined
   return {
     signal: controller.signal,
     cleanup: () => {
-      if (poller) clearInterval(poller)
       parent?.removeEventListener('abort', onAbort)
     },
   }
@@ -728,11 +699,7 @@ export async function runBabysitPiWithOptions(
     diff: '',
     notes: [],
   }
-  const cancellation = createCancellationSignal(
-    context.signal,
-    params.executionId,
-    options.cancellationPollMs
-  )
+  const cancellation = createCancellationSignal(context.signal)
   const { signal } = cancellation
   const startedAt = Date.now()
   const lifetime = resolveBabysitExecutionBudgetMs(params.executionBudgetMs)

apps/sim/lib/api/contracts/hotspots.test.ts

@@ -24,4 +24,35 @@ describe('function execute contract', () => {
 
     expect(result.success).toBe(true)
   })
+
+  it('requires a workspace destination for a sandbox file export', () => {
+    const result = bodySchema.safeParse({
+      code: 'return 1',
+      outputSandboxPath: '/tmp/result.csv',
+    })
+
+    expect(result.success).toBe(false)
+    expect(result.error?.issues).toContainEqual(expect.objectContaining({ path: ['outputPath'] }))
+  })
+
+  it('rejects a whitespace-only workspace destination for a sandbox file export', () => {
+    const result = bodySchema.safeParse({
+      code: 'return 1',
+      outputSandboxPath: '/tmp/result.csv',
+      outputPath: '   ',
+    })
+
+    expect(result.success).toBe(false)
+    expect(result.error?.issues).toContainEqual(expect.objectContaining({ path: ['outputPath'] }))
+  })
+
+  it('accepts a sandbox file export with its workspace destination', () => {
+    const result = bodySchema.safeParse({
+      code: 'return 1',
+      outputSandboxPath: '/tmp/result.csv',
+      outputPath: 'files/result.csv',
+    })
+
+    expect(result.success).toBe(true)
+  })
 })

apps/sim/lib/api/contracts/hotspots.ts

@@ -156,10 +156,8 @@ const functionOutputFileSchema = z
   })
   .strict()
 
-export const functionExecuteContract = defineRouteContract({
-  method: 'POST',
-  path: '/api/function/execute',
-  body: z.object({
+export const functionExecuteBodySchema = z
+  .object({
     code: z.string().min(1, 'Code is required'),
     sourceCode: z.string().optional(),
     params: unknownRecordSchema.optional().default({}),
@@ -232,7 +230,25 @@ export const functionExecuteContract = defineRouteContract({
         ])
       )
       .optional(),
-  }),
+  })
+  .superRefine((body, context) => {
+    if (body.outputSandboxPath && !body.outputPath?.trim()) {
+      context.addIssue({
+        code: z.ZodIssueCode.custom,
+        path: ['outputPath'],
+        message:
+          'outputSandboxPath requires outputPath. Set outputPath to the destination workspace file, e.g. "files/result.csv".',
+      })
+    }
+  })
+
+export type FunctionExecuteBody = z.input<typeof functionExecuteBodySchema>
+export type ParsedFunctionExecuteBody = z.output<typeof functionExecuteBodySchema>
+
+export const functionExecuteContract = defineRouteContract({
+  method: 'POST',
+  path: '/api/function/execute',
+  body: functionExecuteBodySchema,
   response: {
     mode: 'json',
     schema: unknownRecordSchema,

apps/sim/lib/execution/cancellation.test.ts

@@ -1,23 +1,31 @@
 import { redisConfigMockFns } from '@sim/testing'
 import { beforeEach, describe, expect, it, vi } from 'vitest'
 
-const { mockRedisSet, mockPublish, mockSubscribe, mockRecordCancellationResult } = vi.hoisted(
-  () => ({
-    mockRedisSet: vi.fn(),
-    mockPublish: vi.fn(),
-    mockSubscribe: vi.fn(),
-    mockRecordCancellationResult: vi.fn(),
-  })
-)
+const {
+  mockRedisEval,
+  mockRedisExists,
+  mockRedisPublish,
+  mockSignalSubscribe,
+  mockSignalUnsubscribe,
+  mockPublishLocalSignal,
+  mockRecordCancellationResult,
+} = vi.hoisted(() => ({
+  mockRedisEval: vi.fn(),
+  mockRedisExists: vi.fn(),
+  mockRedisPublish: vi.fn(),
+  mockSignalSubscribe: vi.fn(),
+  mockSignalUnsubscribe: vi.fn(),
+  mockPublishLocalSignal: vi.fn(),
+  mockRecordCancellationResult: vi.fn(),
+}))
 
 const mockGetRedisClient = redisConfigMockFns.mockGetRedisClient
 
-vi.mock('@/lib/events/pubsub', () => ({
-  createPubSubChannel: () => ({
-    publish: mockPublish,
-    subscribe: mockSubscribe,
-    dispose: vi.fn(),
-  }),
+vi.mock('@/lib/execution/execution-signal', () => ({
+  getExecutionSignalChannel: (executionId: string) => `execution:signal:${executionId}`,
+  getExecutionSignalHub: () => ({ subscribe: mockSignalSubscribe }),
+  LEGACY_EXECUTION_CANCEL_CHANNEL: 'execution:cancel',
+  publishLocalExecutionSignal: mockPublishLocalSignal,
 }))
 
 vi.mock('@/lib/core/execution-limits/metrics', () => ({
@@ -26,8 +34,8 @@ vi.mock('@/lib/core/execution-limits/metrics', () => ({
 
 import {
   EXECUTION_CANCEL_MIN_RETENTION_MS,
-  getCancellationChannel,
   markExecutionCancelled,
+  subscribeToExecutionCancellation,
 } from './cancellation'
 import {
   abortManualExecution,
@@ -38,6 +46,9 @@ import {
 describe('markExecutionCancelled', () => {
   beforeEach(() => {
     vi.clearAllMocks()
+    mockSignalSubscribe.mockResolvedValue(mockSignalUnsubscribe)
+    mockRedisExists.mockResolvedValue(0)
+    mockRedisPublish.mockResolvedValue(1)
   })
 
   it('returns redis_unavailable when no Redis client exists', async () => {
@@ -47,97 +58,248 @@ describe('markExecutionCancelled', () => {
       durablyRecorded: false,
       reason: 'redis_unavailable',
     })
+    expect(mockPublishLocalSignal).toHaveBeenCalledWith('execution-1', 'cancelled')
   })
 
   it('returns recorded when Redis write succeeds', async () => {
-    mockRedisSet.mockResolvedValue('OK')
-    mockGetRedisClient.mockReturnValue({ set: mockRedisSet })
+    mockRedisEval.mockResolvedValue(1)
+    mockGetRedisClient.mockReturnValue({ eval: mockRedisEval })
 
     await expect(markExecutionCancelled('execution-1')).resolves.toEqual({
       durablyRecorded: true,
       reason: 'recorded',
     })
 
-    const expiryAt = mockRedisSet.mock.calls[0]?.[3]
-    expect(mockRedisSet).toHaveBeenCalledWith(
+    expect(mockRedisEval).toHaveBeenCalledWith(
+      expect.stringContaining("redis.call('PUBLISH'"),
+      1,
       'execution:cancel:execution-1',
-      '1',
-      'PXAT',
-      expect.any(Number)
+      expect.any(Number),
+      'execution:signal:execution-1',
+      'execution:cancel',
+      JSON.stringify({ executionId: 'execution-1', executionSignalPublished: true })
+    )
+    const redisExpiryAt = mockRedisEval.mock.calls[0]?.[3]
+    expect(redisExpiryAt).toBeGreaterThanOrEqual(
+      Date.now() + EXECUTION_CANCEL_MIN_RETENTION_MS - 100
     )
-    expect(expiryAt).toBeGreaterThanOrEqual(Date.now() + EXECUTION_CANCEL_MIN_RETENTION_MS - 100)
   })
 
   it('uses the exact execution deadline when it is later than queue retention', async () => {
     const executionDeadlineAt = new Date(Date.now() + EXECUTION_CANCEL_MIN_RETENTION_MS * 2)
-    mockRedisSet.mockResolvedValue('OK')
-    mockGetRedisClient.mockReturnValue({ set: mockRedisSet })
+    mockRedisEval.mockResolvedValue(1)
+    mockGetRedisClient.mockReturnValue({ eval: mockRedisEval })
 
     await markExecutionCancelled('execution-deadline', { executionDeadlineAt })
 
-    expect(mockRedisSet).toHaveBeenCalledWith(
+    expect(mockRedisEval).toHaveBeenCalledWith(
+      expect.any(String),
+      1,
       'execution:cancel:execution-deadline',
-      '1',
-      'PXAT',
-      executionDeadlineAt.getTime()
+      executionDeadlineAt.getTime(),
+      'execution:signal:execution-deadline',
+      'execution:cancel',
+      JSON.stringify({ executionId: 'execution-deadline', executionSignalPublished: true })
     )
   })
 
   it('keeps the 14-day minimum when the execution deadline is sooner', async () => {
     const before = Date.now()
     const executionDeadlineAt = new Date(before + 60_000)
-    mockRedisSet.mockResolvedValue('OK')
-    mockGetRedisClient.mockReturnValue({ set: mockRedisSet })
+    mockRedisEval.mockResolvedValue(1)
+    mockGetRedisClient.mockReturnValue({ eval: mockRedisEval })
 
     await markExecutionCancelled('execution-short-deadline', { executionDeadlineAt })
 
-    const expiryAt = mockRedisSet.mock.calls[0]?.[3]
+    const expiryAt = mockRedisEval.mock.calls[0]?.[3]
     expect(expiryAt).toBeGreaterThanOrEqual(before + EXECUTION_CANCEL_MIN_RETENTION_MS)
   })
 
   it('returns redis_write_failed when Redis write throws', async () => {
-    mockRedisSet.mockRejectedValue(new Error('set failed'))
-    mockGetRedisClient.mockReturnValue({ set: mockRedisSet })
+    mockRedisEval.mockRejectedValue(new Error('set failed'))
+    mockGetRedisClient.mockReturnValue({ eval: mockRedisEval, publish: mockRedisPublish })
 
     await expect(markExecutionCancelled('execution-1')).resolves.toEqual({
       durablyRecorded: false,
       reason: 'redis_write_failed',
     })
-  })
-
-  it('publishes even when the Redis write fails so local subscribers wake up', async () => {
-    mockRedisSet.mockRejectedValue(new Error('set failed'))
-    mockGetRedisClient.mockReturnValue({ set: mockRedisSet })
-
-    await markExecutionCancelled('execution-write-failed')
-
-    expect(mockPublish).toHaveBeenCalledWith({ executionId: 'execution-write-failed' })
-  })
-
-  it('publishes a cancellation event after a successful Redis write', async () => {
-    mockRedisSet.mockResolvedValue('OK')
-    mockGetRedisClient.mockReturnValue({ set: mockRedisSet })
-
-    await markExecutionCancelled('execution-2')
-
-    expect(mockPublish).toHaveBeenCalledWith({ executionId: 'execution-2' })
-    expect(mockRedisSet.mock.invocationCallOrder[0]).toBeLessThan(
-      mockPublish.mock.invocationCallOrder[0]
+    await vi.waitFor(() => expect(mockRedisPublish).toHaveBeenCalledTimes(2))
+    expect(mockRedisPublish).toHaveBeenNthCalledWith(1, 'execution:signal:execution-1', 'cancelled')
+    expect(mockRedisPublish).toHaveBeenNthCalledWith(
+      2,
+      'execution:cancel',
+      JSON.stringify({ executionId: 'execution-1', executionSignalPublished: true })
     )
   })
 
-  it('publishes even when Redis is unavailable so local subscribers wake up', async () => {
+  it('uses the legacy signal when the exact best-effort signal also fails', async () => {
+    mockRedisEval.mockRejectedValue(new Error('set failed'))
+    mockRedisPublish
+      .mockRejectedValueOnce(new Error('exact publish failed'))
+      .mockResolvedValueOnce(1)
+    mockGetRedisClient.mockReturnValue({ eval: mockRedisEval, publish: mockRedisPublish })
+
+    await expect(markExecutionCancelled('execution-1')).resolves.toEqual({
+      durablyRecorded: false,
+      reason: 'redis_write_failed',
+    })
+    await vi.waitFor(() => expect(mockRedisPublish).toHaveBeenCalledTimes(2))
+    expect(mockRedisPublish).toHaveBeenNthCalledWith(
+      2,
+      'execution:cancel',
+      JSON.stringify({ executionId: 'execution-1' })
+    )
+  })
+
+  it('does not wait for best-effort notification after the Redis write fails', async () => {
+    mockRedisEval.mockRejectedValue(new Error('set failed'))
+    let resolveExactPublish!: (value: number) => void
+    mockRedisPublish
+      .mockReturnValueOnce(
+        new Promise<number>((resolve) => {
+          resolveExactPublish = resolve
+        })
+      )
+      .mockResolvedValueOnce(1)
+    mockGetRedisClient.mockReturnValue({ eval: mockRedisEval, publish: mockRedisPublish })
+
+    await expect(markExecutionCancelled('execution-1')).resolves.toEqual({
+      durablyRecorded: false,
+      reason: 'redis_write_failed',
+    })
+    expect(mockRedisPublish).toHaveBeenCalledOnce()
+
+    resolveExactPublish(1)
+    await vi.waitFor(() => expect(mockRedisPublish).toHaveBeenCalledTimes(2))
+  })
+
+  it('publishes through the configured process-local provider when Redis is unavailable', async () => {
     mockGetRedisClient.mockReturnValue(null)
 
     await markExecutionCancelled('execution-3')
 
-    expect(mockPublish).toHaveBeenCalledWith({ executionId: 'execution-3' })
+    expect(mockRedisEval).not.toHaveBeenCalled()
+    expect(mockPublishLocalSignal).toHaveBeenCalledWith('execution-3', 'cancelled')
+  })
+
+  it('still reports Redis unavailable when the local signal provider rejects the notification', async () => {
+    mockGetRedisClient.mockReturnValue(null)
+    mockPublishLocalSignal.mockImplementationOnce(() => {
+      throw new Error('local signal unavailable')
+    })
+
+    await expect(markExecutionCancelled('execution-local-signal-error')).resolves.toEqual({
+      durablyRecorded: false,
+      reason: 'redis_unavailable',
+    })
+  })
+
+  it('expires process-local cancellation records after their retention window', async () => {
+    vi.useFakeTimers()
+    try {
+      mockGetRedisClient.mockReturnValue(null)
+      const onCancelled = vi.fn()
+
+      await markExecutionCancelled('execution-expiring-local-record')
+      await vi.advanceTimersByTimeAsync(EXECUTION_CANCEL_MIN_RETENTION_MS)
+      await subscribeToExecutionCancellation('execution-expiring-local-record', onCancelled)
+
+      expect(onCancelled).not.toHaveBeenCalled()
+    } finally {
+      vi.useRealTimers()
+    }
   })
 })
 
-describe('getCancellationChannel', () => {
-  it('returns the same channel instance across calls', () => {
-    expect(getCancellationChannel()).toBe(getCancellationChannel())
+describe('subscribeToExecutionCancellation', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    mockSignalSubscribe.mockResolvedValue(mockSignalUnsubscribe)
+    mockRedisExists.mockResolvedValue(0)
+    mockGetRedisClient.mockReturnValue({ exists: mockRedisExists })
+  })
+
+  it('subscribes before reading the durable cancellation flag', async () => {
+    const order: string[] = []
+    mockSignalSubscribe.mockImplementation(async () => {
+      order.push('subscribe')
+      return mockSignalUnsubscribe
+    })
+    mockRedisExists.mockImplementation(async () => {
+      order.push('read')
+      return 0
+    })
+
+    await subscribeToExecutionCancellation('execution-1', vi.fn())
+
+    expect(order).toEqual(['subscribe', 'read'])
+  })
+
+  it('delivers a durable cancellation that predates the subscription', async () => {
+    mockRedisExists.mockResolvedValue(1)
+    const onCancelled = vi.fn()
+
+    const unsubscribe = await subscribeToExecutionCancellation('execution-1', onCancelled)
+
+    expect(onCancelled).toHaveBeenCalledTimes(1)
+    unsubscribe()
+    expect(mockSignalUnsubscribe).toHaveBeenCalledTimes(1)
+  })
+
+  it('re-reads the durable cancellation flag after the signal connection recovers', async () => {
+    let signalHandler: ((reason: string) => void) | undefined
+    mockSignalSubscribe.mockImplementation(async (_executionId, handler) => {
+      signalHandler = handler
+      return mockSignalUnsubscribe
+    })
+    mockRedisExists.mockResolvedValueOnce(0).mockResolvedValueOnce(1)
+    const onCancelled = vi.fn()
+    await subscribeToExecutionCancellation('execution-1', onCancelled)
+
+    signalHandler?.('reconnected')
+
+    await vi.waitFor(() => expect(onCancelled).toHaveBeenCalledOnce())
+    expect(mockRedisExists).toHaveBeenCalledTimes(2)
+  })
+
+  it('queues a fresh durable read when reconnect occurs during an in-flight read', async () => {
+    let signalHandler: ((reason: string) => void) | undefined
+    mockSignalSubscribe.mockImplementation(async (_executionId, handler) => {
+      signalHandler = handler
+      return mockSignalUnsubscribe
+    })
+    let resolveInitialRead!: (value: number) => void
+    mockRedisExists
+      .mockReturnValueOnce(
+        new Promise<number>((resolve) => {
+          resolveInitialRead = resolve
+        })
+      )
+      .mockResolvedValueOnce(1)
+    const onCancelled = vi.fn()
+
+    const subscription = subscribeToExecutionCancellation('execution-1', onCancelled)
+    await vi.waitFor(() => expect(mockRedisExists).toHaveBeenCalledOnce())
+    signalHandler?.('reconnected')
+    resolveInitialRead(0)
+    await subscription
+
+    expect(mockRedisExists).toHaveBeenCalledTimes(2)
+    expect(onCancelled).toHaveBeenCalledOnce()
+  })
+
+  it('fails closed when the signal subscription cannot be restored', async () => {
+    let signalHandler: ((reason: string) => void) | undefined
+    mockSignalSubscribe.mockImplementation(async (_executionId, handler) => {
+      signalHandler = handler
+      return mockSignalUnsubscribe
+    })
+    const onCancelled = vi.fn()
+    await subscribeToExecutionCancellation('execution-1', onCancelled)
+
+    signalHandler?.('unavailable')
+
+    expect(onCancelled).toHaveBeenCalledOnce()
   })
 })
 

apps/sim/lib/execution/cancellation.ts

@@ -1,21 +1,32 @@
 import { createLogger } from '@sim/logger'
 import { getRedisClient } from '@/lib/core/config/redis'
-import { createPubSubChannel, type PubSubChannel } from '@/lib/events/pubsub'
+import {
+  getExecutionSignalChannel,
+  getExecutionSignalHub,
+  LEGACY_EXECUTION_CANCEL_CHANNEL,
+  publishLocalExecutionSignal,
+} from '@/lib/execution/execution-signal'
 
 const logger = createLogger('ExecutionCancellation')
 
 const EXECUTION_CANCEL_PREFIX = 'execution:cancel:'
 export const EXECUTION_CANCEL_MIN_RETENTION_MS = 14 * 24 * 60 * 60 * 1000
-const EXECUTION_CANCEL_CHANNEL = 'execution:cancel'
+const MAX_TIMER_DELAY_MS = 2_147_483_647
+const MAX_LOCAL_CANCELLATION_RECORDS = 50_000
+const localCancelledExecutions = new Map<string, number>()
+let localCancellationCleanupTimer: NodeJS.Timeout | undefined
+let localCancellationCleanupAt: number | undefined
+const MARK_EXECUTION_CANCELLED_SCRIPT = `
+redis.call('SET', KEYS[1], '1', 'PXAT', ARGV[1])
+redis.call('PUBLISH', ARGV[2], 'cancelled')
+redis.call('PUBLISH', ARGV[3], ARGV[4])
+return 1
+`
 
 export interface MarkExecutionCancelledOptions {
   executionDeadlineAt?: Date | null
 }
 
-export interface ExecutionCancelEvent {
-  executionId: string
-}
-
 export type ExecutionCancellationRecordResult =
   | { durablyRecorded: true; reason: 'recorded' }
   | {
@@ -23,24 +34,85 @@ export type ExecutionCancellationRecordResult =
       reason: 'redis_unavailable' | 'redis_write_failed'
     }
 
-type CancellationGlobal = typeof globalThis & {
-  _executionCancelChannel?: PubSubChannel<ExecutionCancelEvent>
+function getCancellationExpiryAt(options: MarkExecutionCancelledOptions): number {
+  const minimumExpiryAt = Date.now() + EXECUTION_CANCEL_MIN_RETENTION_MS
+  const deadlineExpiryAt = options.executionDeadlineAt?.getTime()
+  return deadlineExpiryAt !== undefined && Number.isFinite(deadlineExpiryAt)
+    ? Math.max(minimumExpiryAt, deadlineExpiryAt)
+    : minimumExpiryAt
 }
 
-const _g = globalThis as CancellationGlobal
+function serializeLegacyCancellationSignal(
+  executionId: string,
+  executionSignalPublished: boolean
+): string {
+  return JSON.stringify(
+    executionSignalPublished ? { executionId, executionSignalPublished: true } : { executionId }
+  )
+}
 
-export function getCancellationChannel(): PubSubChannel<ExecutionCancelEvent> {
-  if (!_g._executionCancelChannel) {
-    _g._executionCancelChannel = createPubSubChannel<ExecutionCancelEvent>({
-      channel: EXECUTION_CANCEL_CHANNEL,
-      label: 'execution-cancel',
+async function publishCancellationSignalBestEffort(
+  redis: NonNullable<ReturnType<typeof getRedisClient>>,
+  executionId: string
+): Promise<void> {
+  let executionSignalPublished = false
+  try {
+    await redis.publish(getExecutionSignalChannel(executionId), 'cancelled')
+    executionSignalPublished = true
+  } catch (error) {
+    logger.warn('Failed to publish best-effort execution cancellation signal', {
+      executionId,
+      error,
+    })
+  }
+
+  try {
+    await redis.publish(
+      LEGACY_EXECUTION_CANCEL_CHANNEL,
+      serializeLegacyCancellationSignal(executionId, executionSignalPublished)
+    )
+  } catch (error) {
+    logger.warn('Failed to publish best-effort legacy execution cancellation signal', {
+      executionId,
+      error,
     })
   }
-  return _g._executionCancelChannel
 }
 
-export function isRedisCancellationEnabled(): boolean {
-  return getRedisClient() !== null
+function scheduleLocalCancellationCleanup(candidateExpiryAt?: number): void {
+  if (
+    candidateExpiryAt !== undefined &&
+    localCancellationCleanupTimer &&
+    localCancellationCleanupAt !== undefined &&
+    localCancellationCleanupAt <= candidateExpiryAt
+  ) {
+    return
+  }
+  if (localCancellationCleanupTimer) clearTimeout(localCancellationCleanupTimer)
+  localCancellationCleanupTimer = undefined
+  localCancellationCleanupAt = undefined
+  if (localCancelledExecutions.size === 0) return
+
+  const now = Date.now()
+  let nextExpiryAt = candidateExpiryAt
+  if (nextExpiryAt === undefined) {
+    nextExpiryAt = Number.POSITIVE_INFINITY
+    for (const expiryAt of localCancelledExecutions.values()) {
+      nextExpiryAt = Math.min(nextExpiryAt, expiryAt)
+    }
+  }
+  const delayMs = Math.min(Math.max(0, nextExpiryAt - now), MAX_TIMER_DELAY_MS)
+  localCancellationCleanupAt = now + delayMs
+  localCancellationCleanupTimer = setTimeout(() => {
+    localCancellationCleanupTimer = undefined
+    localCancellationCleanupAt = undefined
+    const cleanupAt = Date.now()
+    for (const [executionId, expiryAt] of localCancelledExecutions) {
+      if (expiryAt <= cleanupAt) localCancelledExecutions.delete(executionId)
+    }
+    scheduleLocalCancellationCleanup()
+  }, delayMs)
+  localCancellationCleanupTimer.unref?.()
 }
 
 /** Writes the durable key first, then publishes — so a late subscriber still sees the flag on backstop check. */
@@ -49,50 +121,143 @@ export async function markExecutionCancelled(
   options: MarkExecutionCancelledOptions = {}
 ): Promise<ExecutionCancellationRecordResult> {
   const redis = getRedisClient()
+  const expiryAt = getCancellationExpiryAt(options)
   if (!redis) {
-    getCancellationChannel().publish({ executionId })
+    if (!localCancelledExecutions.has(executionId)) {
+      const oldestExecutionId = localCancelledExecutions.keys().next().value
+      if (
+        localCancelledExecutions.size >= MAX_LOCAL_CANCELLATION_RECORDS &&
+        oldestExecutionId !== undefined
+      ) {
+        localCancelledExecutions.delete(oldestExecutionId)
+        logger.warn('Evicted oldest process-local cancellation record at capacity', {
+          maxRecords: MAX_LOCAL_CANCELLATION_RECORDS,
+        })
+      }
+    } else {
+      localCancelledExecutions.delete(executionId)
+    }
+    localCancelledExecutions.set(executionId, expiryAt)
+    scheduleLocalCancellationCleanup(expiryAt)
+    try {
+      publishLocalExecutionSignal(executionId, 'cancelled')
+    } catch (error) {
+      logger.warn('Failed to publish process-local cancellation signal', { executionId, error })
+    }
     return { durablyRecorded: false, reason: 'redis_unavailable' }
   }
 
   try {
-    const minimumExpiryAt = Date.now() + EXECUTION_CANCEL_MIN_RETENTION_MS
-    const deadlineExpiryAt = options.executionDeadlineAt?.getTime()
-    const expiryAt =
-      deadlineExpiryAt !== undefined && Number.isFinite(deadlineExpiryAt)
-        ? Math.max(minimumExpiryAt, deadlineExpiryAt)
-        : minimumExpiryAt
-    await redis.set(`${EXECUTION_CANCEL_PREFIX}${executionId}`, '1', 'PXAT', expiryAt)
+    await redis.eval(
+      MARK_EXECUTION_CANCELLED_SCRIPT,
+      1,
+      `${EXECUTION_CANCEL_PREFIX}${executionId}`,
+      expiryAt,
+      getExecutionSignalChannel(executionId),
+      LEGACY_EXECUTION_CANCEL_CHANNEL,
+      serializeLegacyCancellationSignal(executionId, true)
+    )
     logger.info('Marked execution as cancelled', {
       executionId,
       expiresAt: new Date(expiryAt).toISOString(),
     })
-    getCancellationChannel().publish({ executionId })
     return { durablyRecorded: true, reason: 'recorded' }
   } catch (error) {
     logger.error('Failed to mark execution as cancelled', { executionId, error })
-    getCancellationChannel().publish({ executionId })
+    void publishCancellationSignalBestEffort(redis, executionId)
     return { durablyRecorded: false, reason: 'redis_write_failed' }
   }
 }
 
-export async function isExecutionCancelled(executionId: string): Promise<boolean> {
+async function readExecutionCancelledStrict(executionId: string): Promise<boolean> {
   const redis = getRedisClient()
   if (!redis) {
-    return false
+    const expiryAt = localCancelledExecutions.get(executionId)
+    if (expiryAt === undefined) return false
+    if (expiryAt <= Date.now()) {
+      localCancelledExecutions.delete(executionId)
+      if (expiryAt <= (localCancellationCleanupAt ?? Number.POSITIVE_INFINITY)) {
+        scheduleLocalCancellationCleanup()
+      }
+      return false
+    }
+    return true
+  }
+  return (await redis.exists(`${EXECUTION_CANCEL_PREFIX}${executionId}`)) === 1
+}
+
+/**
+ * Subscribes before reading the durable flag, then re-reads after subscriber reconnects.
+ * This closes both the initial subscribe race and a Redis disconnect gap without polling.
+ */
+export async function subscribeToExecutionCancellation(
+  executionId: string,
+  onCancelled: () => void
+): Promise<() => void> {
+  let disposed = false
+  let readInFlight: Promise<void> | undefined
+  let readAgain = false
+  const readAndCancel = () => {
+    if (disposed) return undefined
+    if (readInFlight) {
+      readAgain = true
+      return readInFlight
+    }
+    readInFlight = (async () => {
+      do {
+        readAgain = false
+        const cancelled = await readExecutionCancelledStrict(executionId)
+        if (cancelled && !disposed) onCancelled()
+      } while (readAgain && !disposed)
+    })().finally(() => {
+      readInFlight = undefined
+    })
+    return readInFlight
   }
 
+  const unsubscribe = await getExecutionSignalHub().subscribe(executionId, (reason) => {
+    if (reason === 'cancelled') {
+      onCancelled()
+      return
+    }
+    if (reason === 'unavailable') {
+      logger.error('Execution cancellation signal became unavailable', { executionId })
+      onCancelled()
+      return
+    }
+    if (reason === 'reconnected') {
+      void readAndCancel()?.catch((error) => {
+        logger.error('Failed to restore cancellation state after signal reconnect', {
+          executionId,
+          error,
+        })
+        if (!disposed) onCancelled()
+      })
+    }
+  })
   try {
-    const result = await redis.exists(`${EXECUTION_CANCEL_PREFIX}${executionId}`)
-    return result === 1
+    await readAndCancel()
   } catch (error) {
-    logger.error('Failed to check execution cancellation', { executionId, error })
-    return false
+    unsubscribe()
+    throw error
+  }
+  return () => {
+    disposed = true
+    unsubscribe()
   }
 }
 
 export async function clearExecutionCancellation(executionId: string): Promise<void> {
   const redis = getRedisClient()
   if (!redis) {
+    const expiryAt = localCancelledExecutions.get(executionId)
+    localCancelledExecutions.delete(executionId)
+    if (
+      expiryAt !== undefined &&
+      expiryAt <= (localCancellationCleanupAt ?? Number.POSITIVE_INFINITY)
+    ) {
+      scheduleLocalCancellationCleanup()
+    }
     return
   }
 

apps/sim/lib/execution/event-buffer.test.ts

@@ -55,8 +55,12 @@ import {
   createExecutionEventWriter,
   flushExecutionStreamReplayBuffer,
   initializeExecutionStreamMeta,
+  markExecutionStreamTerminal,
   readExecutionEventsState,
+  readExecutionMetaState,
   resetExecutionStreamBuffer,
+  setExecutionActiveBlockStarts,
+  setExecutionMeta,
 } from '@/lib/execution/event-buffer'
 
 function makeEvent(blockId: string): ExecutionEvent {
@@ -141,6 +145,136 @@ describe('execution event buffer', () => {
     }))
   })
 
+  it('restores the bounded active block snapshot from stream metadata', async () => {
+    mockRedis.hgetall.mockResolvedValueOnce({
+      status: 'active',
+      workflowId: 'wf-1',
+      activeBlockStarts: JSON.stringify([
+        {
+          eventId: 7,
+          data: {
+            blockId: 'function-1',
+            blockName: 'Qualify',
+            blockType: 'function',
+            executionOrder: 2,
+          },
+        },
+      ]),
+    })
+
+    await expect(readExecutionMetaState('exec-1')).resolves.toMatchObject({
+      status: 'found',
+      meta: {
+        activeBlockStarts: [{ eventId: 7, data: { blockId: 'function-1' } }],
+      },
+    })
+  })
+
+  it('uses the configured in-memory provider when database cache mode is selected', async () => {
+    setEnv({ REDIS_URL: undefined })
+    mockGetRedisClient.mockReturnValue(null)
+    const executionId = 'exec-database-provider'
+
+    await expect(
+      initializeExecutionStreamMeta(executionId, {
+        workflowId: 'wf-1',
+        userId: 'user-1',
+      })
+    ).resolves.toBe(true)
+    const writer = createExecutionEventWriter(executionId)
+    const entry = await writer.write({
+      ...makeEvent('function-local'),
+      executionId,
+    })
+
+    await expect(readExecutionEventsState(executionId, 0)).resolves.toMatchObject({
+      status: 'ok',
+      events: [{ eventId: entry.eventId, event: { type: 'block:started' } }],
+    })
+    await expect(readExecutionMetaState(executionId)).resolves.toMatchObject({
+      status: 'found',
+      meta: { status: 'active', protocolVersion: 1 },
+    })
+  })
+
+  it('retains active runs through the execution ceiling but expires terminal streams after an hour', async () => {
+    await setExecutionMeta('exec-1', { status: 'active' })
+    await setExecutionMeta('exec-1', { status: 'complete' })
+
+    expect(mockRedis.expire.mock.calls[0]?.[1]).toBeGreaterThan(60 * 60)
+    expect(mockRedis.expire.mock.calls[1]?.[1]).toBe(60 * 60)
+  })
+
+  it('atomically marks a degraded terminal stream and wakes its observers', async () => {
+    await expect(markExecutionStreamTerminal('exec-1', 'error')).resolves.toBe(true)
+
+    const [script, keyCount, metaKey, signalChannel, status, , ttlSeconds] =
+      mockRedis.eval.mock.calls.at(-1) ?? []
+    expect(script).toContain("redis.call('HSET'")
+    expect(script).toContain("redis.call('PUBLISH'")
+    expect(keyCount).toBe(2)
+    expect(metaKey).toBe('execution:stream:exec-1:meta')
+    expect(signalChannel).toBe('execution:signal:exec-1')
+    expect(status).toBe('error')
+    expect(ttlSeconds).toBe(60 * 60)
+  })
+
+  it('atomically persists active block starts and wakes reconnecting observers', async () => {
+    const activeBlockStarts = [
+      {
+        eventId: 7,
+        data: {
+          blockId: 'function-1',
+          blockName: 'Qualify',
+          blockType: 'function',
+          executionOrder: 2,
+        },
+      },
+    ]
+
+    await expect(setExecutionActiveBlockStarts('exec-1', activeBlockStarts)).resolves.toBe(true)
+
+    const [script, keyCount, metaKey, signalChannel, snapshot, , ttlSeconds] =
+      mockRedis.eval.mock.calls.at(-1) ?? []
+    expect(script).toContain("redis.call('HSET'")
+    expect(script).toContain("redis.call('HGET'")
+    expect(script).toContain("status == 'complete'")
+    expect(script).toContain("status == 'error'")
+    expect(script).toContain("status == 'cancelled'")
+    expect(script).toContain("redis.call('PUBLISH'")
+    expect(keyCount).toBe(2)
+    expect(metaKey).toBe('execution:stream:exec-1:meta')
+    expect(signalChannel).toBe('execution:signal:exec-1')
+    expect(snapshot).toBe(JSON.stringify(activeBlockStarts))
+    expect(ttlSeconds).toBeGreaterThan(60 * 60)
+  })
+
+  it('does not replace a terminal process-local snapshot with a late block event', async () => {
+    setEnv({ REDIS_URL: undefined })
+    mockGetRedisClient.mockReturnValue(null)
+    const executionId = 'exec-terminal-snapshot'
+
+    await expect(markExecutionStreamTerminal(executionId, 'cancelled')).resolves.toBe(true)
+    await expect(
+      setExecutionActiveBlockStarts(executionId, [
+        {
+          eventId: 9,
+          data: {
+            blockId: 'function-late',
+            blockName: 'Late',
+            blockType: 'function',
+            executionOrder: 3,
+          },
+        },
+      ])
+    ).resolves.toBe(true)
+
+    await expect(readExecutionMetaState(executionId)).resolves.toMatchObject({
+      status: 'found',
+      meta: { status: 'cancelled', activeBlockStarts: [] },
+    })
+  })
+
   it('serializes event id reservation so reconnect replay preserves write order', async () => {
     let releaseReservation: ((value: number) => void) | undefined
     mockRedis.incrby.mockReturnValueOnce(
@@ -493,7 +627,7 @@ describe('execution event buffer', () => {
   /**
    * A timer-driven flush carries no terminal status of its own. If it is the
    * loop that drains the final chunk, the terminal event lands without a status
-   * and readers poll an `active` stream forever — while `writeTerminal` reports
+   * and readers wait on an `active` stream forever — while `writeTerminal` reports
    * success, so nothing degrades.
    */
   it('applies terminal status even when a concurrent scheduled flush drains the final chunk', async () => {
@@ -896,8 +1030,8 @@ describe('execution event buffer', () => {
     await writer.writeTerminal(makeEvent('terminal'), 'complete')
 
     expect(flushScript).not.toBe('')
-    const userKeyExpires = countOccurrences(flushScript, "redis.call('EXPIRE', KEYS[5]")
-    const userKeyTtlGuards = countOccurrences(flushScript, "redis.call('TTL', KEYS[5]) < 0")
+    const userKeyExpires = countOccurrences(flushScript, "redis.call('EXPIRE', KEYS[6]")
+    const userKeyTtlGuards = countOccurrences(flushScript, "redis.call('TTL', KEYS[6]) < 0")
     expect(userKeyExpires).toBeGreaterThan(0)
     expect(userKeyTtlGuards).toBe(userKeyExpires)
   })
@@ -912,8 +1046,8 @@ describe('execution event buffer', () => {
     const writer = createExecutionEventWriter('exec-1', { userId: 'user-1' })
     await writer.writeTerminal(makeEvent('terminal'), 'complete')
 
-    expect(countOccurrences(flushScript, "redis.call('TTL', KEYS[4]) < 0")).toBe(0)
-    expect(countOccurrences(flushScript, "redis.call('EXPIRE', KEYS[4]")).toBeGreaterThan(0)
+    expect(countOccurrences(flushScript, "redis.call('TTL', KEYS[5]) < 0")).toBe(0)
+    expect(countOccurrences(flushScript, "redis.call('EXPIRE', KEYS[5]")).toBeGreaterThan(0)
   })
 
   it('reports pruned replay buffers before reading incomplete events', async () => {

apps/sim/lib/execution/event-buffer.ts

@@ -3,6 +3,11 @@ import { toError } from '@sim/utils/errors'
 import { randomInt } from '@sim/utils/random'
 import { getConfiguredCacheProvider } from '@/lib/core/config/env-capabilities.server'
 import { getRedisClient } from '@/lib/core/config/redis'
+import { getMaxExecutionTimeout } from '@/lib/core/execution-limits'
+import {
+  getExecutionSignalChannel,
+  publishLocalExecutionSignal,
+} from '@/lib/execution/execution-signal'
 import { LARGE_VALUE_THRESHOLD_BYTES } from '@/lib/execution/payloads/large-value-ref'
 import { compactExecutionPayload } from '@/lib/execution/payloads/serializer'
 import type { LargeValueStoreContext } from '@/lib/execution/payloads/store'
@@ -15,18 +20,21 @@ import {
   ExecutionResourceLimitError,
   isExecutionResourceLimitError,
 } from '@/lib/execution/resource-errors'
-import type { ExecutionEvent } from '@/lib/workflows/executor/execution-events'
+import type { BlockStartedData, ExecutionEvent } from '@/lib/workflows/executor/execution-events'
 
 const logger = createLogger('ExecutionEventBuffer')
 
 const REDIS_PREFIX = 'execution:stream:'
-const TTL_SECONDS = 60 * 60 // 1 hour
+export const EXECUTION_STREAM_PROTOCOL_VERSION = 1
+const TERMINAL_TTL_SECONDS = 60 * 60
+const ACTIVE_TTL_SECONDS = Math.ceil(getMaxExecutionTimeout() / 1000) + TERMINAL_TTL_SECONDS
 const EVENT_LIMIT = 1000
 const RESERVE_BATCH = 100
 const FLUSH_INTERVAL_MS = 15
 const FLUSH_MAX_RETRY_INTERVAL_MS = 1000
 const FLUSH_MAX_BATCH = 200
 const MAX_PENDING_EVENTS = 1000
+const MAX_ACTIVE_BLOCK_SNAPSHOT_BYTES = 256 * 1024
 /**
  * Bytes a single execution may buffer before its events start offloading
  * aggressively, and the per-value threshold applied once it does.
@@ -85,48 +93,48 @@ for i = 1, new_prune_count do
 end
 local net_bytes = new_bytes - pruned_bytes
 if net_bytes > 0 then
-  local execution_current = tonumber(redis.call('GET', KEYS[4]) or '0')
+  local execution_current = tonumber(redis.call('GET', KEYS[5]) or '0')
   if execution_limit > 0 and execution_current + net_bytes > execution_limit then
     return {0, 'execution_redis_bytes', execution_current, pruned_bytes}
   end
   local user_current = 0
-  if #KEYS >= 5 then
-    user_current = tonumber(redis.call('GET', KEYS[5]) or '0')
+  if #KEYS >= 6 then
+    user_current = tonumber(redis.call('GET', KEYS[6]) or '0')
     if user_limit > 0 and user_current + net_bytes > user_limit then
       return {0, 'user_redis_bytes', user_current, pruned_bytes}
     end
   end
-  redis.call('INCRBY', KEYS[4], net_bytes)
-  redis.call('EXPIRE', KEYS[4], budget_ttl_seconds)
-  if #KEYS >= 5 then
-    redis.call('INCRBY', KEYS[5], net_bytes)
-    if redis.call('TTL', KEYS[5]) < 0 then
-      redis.call('EXPIRE', KEYS[5], budget_ttl_seconds)
+  redis.call('INCRBY', KEYS[5], net_bytes)
+  redis.call('EXPIRE', KEYS[5], budget_ttl_seconds)
+  if #KEYS >= 6 then
+    redis.call('INCRBY', KEYS[6], net_bytes)
+    if redis.call('TTL', KEYS[6]) < 0 then
+      redis.call('EXPIRE', KEYS[6], budget_ttl_seconds)
     end
   end
 elseif net_bytes < 0 then
   local release_bytes = -net_bytes
-  local execution_next = redis.call('DECRBY', KEYS[4], release_bytes)
+  local execution_next = redis.call('DECRBY', KEYS[5], release_bytes)
   if execution_next <= 0 then
-    redis.call('DEL', KEYS[4])
+    redis.call('DEL', KEYS[5])
   else
-    redis.call('EXPIRE', KEYS[4], budget_ttl_seconds)
+    redis.call('EXPIRE', KEYS[5], budget_ttl_seconds)
   end
-  if #KEYS >= 5 then
-    local user_next = redis.call('DECRBY', KEYS[5], release_bytes)
+  if #KEYS >= 6 then
+    local user_next = redis.call('DECRBY', KEYS[6], release_bytes)
     if user_next <= 0 then
-      redis.call('DEL', KEYS[5])
-    elseif redis.call('TTL', KEYS[5]) < 0 then
-      redis.call('EXPIRE', KEYS[5], budget_ttl_seconds)
+      redis.call('DEL', KEYS[6])
+    elseif redis.call('TTL', KEYS[6]) < 0 then
+      redis.call('EXPIRE', KEYS[6], budget_ttl_seconds)
     end
   end
 else
-  if redis.call('EXISTS', KEYS[4]) == 1 then
-    redis.call('EXPIRE', KEYS[4], budget_ttl_seconds)
-  end
-  if #KEYS >= 5 and redis.call('EXISTS', KEYS[5]) == 1 and redis.call('TTL', KEYS[5]) < 0 then
+  if redis.call('EXISTS', KEYS[5]) == 1 then
     redis.call('EXPIRE', KEYS[5], budget_ttl_seconds)
   end
+  if #KEYS >= 6 and redis.call('EXISTS', KEYS[6]) == 1 and redis.call('TTL', KEYS[6]) < 0 then
+    redis.call('EXPIRE', KEYS[6], budget_ttl_seconds)
+  end
 end
 for i = 9, #ARGV, 2 do
   redis.call('ZADD', KEYS[1], ARGV[i], ARGV[i + 1])
@@ -136,15 +144,32 @@ redis.call('EXPIRE', KEYS[2], tonumber(ARGV[1]))
 redis.call('ZREMRANGEBYRANK', KEYS[1], 0, -tonumber(ARGV[2]) - 1)
 local oldest = redis.call('ZRANGE', KEYS[1], 0, 0, 'WITHSCORES')
 if terminal_status ~= '' then
-  redis.call('HSET', KEYS[3], 'status', terminal_status, 'updatedAt', ARGV[3])
+  redis.call('HSET', KEYS[3], 'status', terminal_status, 'updatedAt', ARGV[3], 'activeBlockStarts', '[]')
   redis.call('EXPIRE', KEYS[3], tonumber(ARGV[1]))
 end
 if oldest[2] then
   redis.call('HSET', KEYS[3], 'earliestEventId', tostring(math.floor(tonumber(oldest[2]))), 'updatedAt', ARGV[3])
   redis.call('EXPIRE', KEYS[3], tonumber(ARGV[1]))
 end
+redis.call('PUBLISH', KEYS[4], 'event')
 return {1, oldest[2] or false, pruned_bytes}
 `
+const MARK_TERMINAL_META_SCRIPT = `
+redis.call('HSET', KEYS[1], 'status', ARGV[1], 'updatedAt', ARGV[2], 'activeBlockStarts', '[]')
+redis.call('EXPIRE', KEYS[1], tonumber(ARGV[3]))
+redis.call('PUBLISH', KEYS[2], 'event')
+return 1
+`
+const SET_ACTIVE_BLOCK_STARTS_SCRIPT = `
+local status = redis.call('HGET', KEYS[1], 'status')
+if status == 'complete' or status == 'error' or status == 'cancelled' then
+  return 1
+end
+redis.call('HSET', KEYS[1], 'activeBlockStarts', ARGV[1], 'updatedAt', ARGV[2])
+redis.call('EXPIRE', KEYS[1], tonumber(ARGV[3]))
+redis.call('PUBLISH', KEYS[2], 'event')
+return 1
+`
 const RESET_STREAM_SCRIPT = `
 local entries = redis.call('ZRANGE', KEYS[1], 0, -1)
 local retained_bytes = 0
@@ -252,11 +277,18 @@ function trimFinalBlockLogsForEventData(data: unknown): unknown {
 
 export interface ExecutionStreamMeta {
   status: ExecutionStreamStatus
+  protocolVersion?: number
   userId?: string
   workflowId?: string
   updatedAt?: string
   earliestEventId?: number
   replayStartEventId?: number
+  activeBlockStarts?: ActiveBlockStartSnapshot[]
+}
+
+export interface ActiveBlockStartSnapshot {
+  eventId: number
+  data: BlockStartedData
 }
 
 export type TerminalExecutionStreamStatus = Exclude<ExecutionStreamStatus, 'active'>
@@ -369,9 +401,7 @@ function canUseMemoryEventBuffer(): boolean {
 
 function pruneExpiredMemoryStreams(now = Date.now()): void {
   for (const [executionId, stream] of memoryExecutionStreams) {
-    if (stream.expiresAt <= now) {
-      memoryExecutionStreams.delete(executionId)
-    }
+    if (stream.expiresAt <= now) memoryExecutionStreams.delete(executionId)
   }
 }
 
@@ -383,7 +413,7 @@ function getMemoryStream(executionId: string): MemoryExecutionStream {
       events: [],
       meta: null,
       nextEventId: 1,
-      expiresAt: Date.now() + TTL_SECONDS * 1000,
+      expiresAt: Date.now() + ACTIVE_TTL_SECONDS * 1000,
     }
     memoryExecutionStreams.set(executionId, stream)
   }
@@ -391,23 +421,8 @@ function getMemoryStream(executionId: string): MemoryExecutionStream {
 }
 
 function touchMemoryStream(stream: MemoryExecutionStream): void {
-  stream.expiresAt = Date.now() + TTL_SECONDS * 1000
-}
-
-function isReplayBeforeAvailableEvents(
-  afterEventId: number,
-  earliestEventId?: number,
-  replayStartEventId?: number
-): earliestEventId is number {
-  if (earliestEventId === undefined || !Number.isFinite(earliestEventId)) return false
-  if (
-    afterEventId === 0 &&
-    replayStartEventId !== undefined &&
-    Number.isFinite(replayStartEventId)
-  ) {
-    return earliestEventId > replayStartEventId
-  }
-  return afterEventId + 1 < earliestEventId
+  const terminal = stream.meta?.status && stream.meta.status !== 'active'
+  stream.expiresAt = Date.now() + (terminal ? TERMINAL_TTL_SECONDS : ACTIVE_TTL_SECONDS) * 1000
 }
 
 function readMemoryMeta(executionId: string): ExecutionMetaReadResult {
@@ -437,7 +452,7 @@ function createMemoryExecutionEventWriter(
   executionId: string,
   context: ExecutionEventWriterContext = {}
 ): ExecutionEventWriter {
-  const writeMemoryEvent = async (event: ExecutionEvent) => {
+  const writeMemoryEvent = async (event: ExecutionEvent, publish: boolean) => {
     const stream = getMemoryStream(executionId)
     const compactEvent = await compactEventForBuffer(event, context)
     const entry = {
@@ -458,20 +473,23 @@ function createMemoryExecutionEventWriter(
       }
     }
     touchMemoryStream(stream)
+    if (publish) publishLocalExecutionSignal(executionId, 'event')
     return entry
   }
 
   return {
-    write: writeMemoryEvent,
+    write: (event) => writeMemoryEvent(event, true),
     writeTerminal: async (event, status) => {
-      const entry = await writeMemoryEvent(event)
+      const entry = await writeMemoryEvent(event, false)
       const stream = getMemoryStream(executionId)
       stream.meta = {
         ...stream.meta,
         status,
         updatedAt: new Date().toISOString(),
+        activeBlockStarts: [],
       }
       touchMemoryStream(stream)
+      publishLocalExecutionSignal(executionId, 'event')
       return entry
     },
     flush: async () => {},
@@ -479,6 +497,22 @@ function createMemoryExecutionEventWriter(
   }
 }
 
+function isReplayBeforeAvailableEvents(
+  afterEventId: number,
+  earliestEventId?: number,
+  replayStartEventId?: number
+): earliestEventId is number {
+  if (earliestEventId === undefined || !Number.isFinite(earliestEventId)) return false
+  if (
+    afterEventId === 0 &&
+    replayStartEventId !== undefined &&
+    Number.isFinite(replayStartEventId)
+  ) {
+    return earliestEventId > replayStartEventId
+  }
+  return afterEventId + 1 < earliestEventId
+}
+
 export async function flushExecutionStreamReplayBuffer(
   executionId: string,
   writer: ExecutionEventWriter
@@ -505,19 +539,21 @@ export async function flushExecutionStreamReplayBuffer(
 export async function resetExecutionStreamBuffer(executionId: string): Promise<boolean> {
   const redis = getRedisClient()
   if (!redis) {
-    if (!canUseMemoryEventBuffer()) {
-      logger.warn('resetExecutionStreamBuffer: Redis client unavailable', { executionId })
-      return false
+    if (canUseMemoryEventBuffer()) {
+      const stream = getMemoryStream(executionId)
+      stream.events = []
+      stream.meta = {
+        status: 'active',
+        protocolVersion: EXECUTION_STREAM_PROTOCOL_VERSION,
+        replayStartEventId: stream.nextEventId,
+        updatedAt: new Date().toISOString(),
+        activeBlockStarts: [],
+      }
+      touchMemoryStream(stream)
+      return true
     }
-    const stream = getMemoryStream(executionId)
-    stream.events = []
-    stream.meta = {
-      status: 'active',
-      replayStartEventId: stream.nextEventId,
-      updatedAt: new Date().toISOString(),
-    }
-    stream.expiresAt = Date.now() + TTL_SECONDS * 1000
-    return true
+    logger.warn('resetExecutionStreamBuffer: Redis client unavailable', { executionId })
+    return false
   }
 
   try {
@@ -543,7 +579,7 @@ export async function resetExecutionStreamBuffer(executionId: string): Promise<b
       ...budgetKeys,
       String(replayStartEventId),
       new Date().toISOString(),
-      TTL_SECONDS,
+      ACTIVE_TTL_SECONDS,
       getExecutionRedisBudgetLimits().ttlSeconds
     )
     return true
@@ -566,6 +602,15 @@ export async function setExecutionMeta(
       const stream = getMemoryStream(executionId)
       const status = meta.status ?? stream.meta?.status
       if (!status) return false
+      if (meta.activeBlockStarts !== undefined) {
+        const activeBlockStarts = JSON.stringify(meta.activeBlockStarts)
+        if (
+          meta.activeBlockStarts.length > EVENT_LIMIT ||
+          Buffer.byteLength(activeBlockStarts, 'utf8') > MAX_ACTIVE_BLOCK_SNAPSHOT_BYTES
+        ) {
+          return false
+        }
+      }
       stream.meta = {
         ...stream.meta,
         ...meta,
@@ -584,14 +629,33 @@ export async function setExecutionMeta(
       updatedAt: new Date().toISOString(),
     }
     if (meta.status) payload.status = meta.status
+    if (meta.protocolVersion !== undefined) {
+      payload.protocolVersion = String(meta.protocolVersion)
+    }
     if (meta.userId) payload.userId = meta.userId
     if (meta.workflowId) payload.workflowId = meta.workflowId
     if (meta.earliestEventId !== undefined) payload.earliestEventId = String(meta.earliestEventId)
     if (meta.replayStartEventId !== undefined) {
       payload.replayStartEventId = String(meta.replayStartEventId)
     }
+    if (meta.activeBlockStarts !== undefined) {
+      const activeBlockStarts = JSON.stringify(meta.activeBlockStarts)
+      if (
+        meta.activeBlockStarts.length > EVENT_LIMIT ||
+        Buffer.byteLength(activeBlockStarts, 'utf8') > MAX_ACTIVE_BLOCK_SNAPSHOT_BYTES
+      ) {
+        logger.warn('Active execution snapshot exceeds its retention bound', {
+          executionId,
+          activeBlocks: meta.activeBlockStarts.length,
+        })
+        return false
+      }
+      payload.activeBlockStarts = activeBlockStarts
+    }
     await redis.hset(key, payload)
-    await redis.expire(key, TTL_SECONDS)
+    const ttlSeconds =
+      meta.status && meta.status !== 'active' ? TERMINAL_TTL_SECONDS : ACTIVE_TTL_SECONDS
+    await redis.expire(key, ttlSeconds)
     return true
   } catch (error) {
     logger.warn('Failed to update execution meta', {
@@ -602,6 +666,65 @@ export async function setExecutionMeta(
   }
 }
 
+export async function setExecutionActiveBlockStarts(
+  executionId: string,
+  activeBlockStarts: ActiveBlockStartSnapshot[]
+): Promise<boolean> {
+  if (activeBlockStarts.length > EVENT_LIMIT) {
+    logger.warn('Active execution snapshot exceeds its retention bound', {
+      executionId,
+      activeBlocks: activeBlockStarts.length,
+    })
+    return false
+  }
+  const serializedSnapshot = JSON.stringify(activeBlockStarts)
+  if (Buffer.byteLength(serializedSnapshot, 'utf8') > MAX_ACTIVE_BLOCK_SNAPSHOT_BYTES) {
+    logger.warn('Active execution snapshot exceeds its retention bound', {
+      executionId,
+      activeBlocks: activeBlockStarts.length,
+    })
+    return false
+  }
+
+  const redis = getRedisClient()
+  if (!redis) {
+    if (canUseMemoryEventBuffer()) {
+      const stream = getMemoryStream(executionId)
+      if (!stream.meta?.status) return false
+      if (stream.meta.status !== 'active') return true
+      stream.meta = {
+        ...stream.meta,
+        activeBlockStarts,
+        updatedAt: new Date().toISOString(),
+      }
+      touchMemoryStream(stream)
+      publishLocalExecutionSignal(executionId, 'event')
+      return true
+    }
+    logger.warn('setExecutionActiveBlockStarts: Redis client unavailable', { executionId })
+    return false
+  }
+
+  try {
+    await redis.eval(
+      SET_ACTIVE_BLOCK_STARTS_SCRIPT,
+      2,
+      getMetaKey(executionId),
+      getExecutionSignalChannel(executionId),
+      serializedSnapshot,
+      new Date().toISOString(),
+      ACTIVE_TTL_SECONDS
+    )
+    return true
+  } catch (error) {
+    logger.warn('Failed to update active execution snapshot', {
+      executionId,
+      error: toError(error).message,
+    })
+    return false
+  }
+}
+
 export async function initializeExecutionStreamMeta(
   executionId: string,
   meta: Omit<ExecutionStreamMeta, 'status' | 'updatedAt'> & { status?: 'active' }
@@ -610,6 +733,7 @@ export async function initializeExecutionStreamMeta(
     const metaPersisted = await setExecutionMeta(executionId, {
       ...meta,
       status: 'active',
+      protocolVersion: EXECUTION_STREAM_PROTOCOL_VERSION,
     })
     if (metaPersisted) return true
     logger.warn('Failed to persist active execution meta during initialization', {
@@ -620,14 +744,56 @@ export async function initializeExecutionStreamMeta(
   return false
 }
 
+/** Records a terminal state and wakes observers when the terminal event itself could not persist. */
+export async function markExecutionStreamTerminal(
+  executionId: string,
+  status: TerminalExecutionStreamStatus
+): Promise<boolean> {
+  const redis = getRedisClient()
+  if (!redis) {
+    if (canUseMemoryEventBuffer()) {
+      const stream = getMemoryStream(executionId)
+      stream.meta = {
+        ...stream.meta,
+        status,
+        protocolVersion: EXECUTION_STREAM_PROTOCOL_VERSION,
+        updatedAt: new Date().toISOString(),
+        activeBlockStarts: [],
+      }
+      touchMemoryStream(stream)
+      publishLocalExecutionSignal(executionId, 'event')
+      return true
+    }
+    logger.warn('markExecutionStreamTerminal: Redis client unavailable', { executionId })
+    return false
+  }
+  try {
+    await redis.eval(
+      MARK_TERMINAL_META_SCRIPT,
+      2,
+      getMetaKey(executionId),
+      getExecutionSignalChannel(executionId),
+      status,
+      new Date().toISOString(),
+      TERMINAL_TTL_SECONDS
+    )
+    return true
+  } catch (error) {
+    logger.warn('Failed to mark execution stream terminal', {
+      executionId,
+      status,
+      error: toError(error).message,
+    })
+    return false
+  }
+}
+
 export async function readExecutionMetaState(
   executionId: string
 ): Promise<ExecutionMetaReadResult> {
   const redis = getRedisClient()
   if (!redis) {
-    if (canUseMemoryEventBuffer()) {
-      return readMemoryMeta(executionId)
-    }
+    if (canUseMemoryEventBuffer()) return readMemoryMeta(executionId)
     logger.warn('readExecutionMetaState: Redis client unavailable', { executionId })
     return { status: 'unavailable', error: 'Redis client unavailable' }
   }
@@ -640,6 +806,8 @@ export async function readExecutionMetaState(
       status: 'found',
       meta: {
         status: meta.status,
+        protocolVersion:
+          meta.protocolVersion !== undefined ? Number(meta.protocolVersion) : undefined,
         userId: meta.userId,
         workflowId: meta.workflowId,
         updatedAt: meta.updatedAt,
@@ -647,6 +815,7 @@ export async function readExecutionMetaState(
           meta.earliestEventId !== undefined ? Number(meta.earliestEventId) : undefined,
         replayStartEventId:
           meta.replayStartEventId !== undefined ? Number(meta.replayStartEventId) : undefined,
+        activeBlockStarts: parseActiveBlockStarts(meta.activeBlockStarts),
       },
     }
   } catch (error) {
@@ -659,15 +828,30 @@ export async function readExecutionMetaState(
   }
 }
 
+function parseActiveBlockStarts(value: string | undefined): ActiveBlockStartSnapshot[] | undefined {
+  if (!value) return undefined
+  try {
+    const parsed: unknown = JSON.parse(value)
+    if (!Array.isArray(parsed)) return undefined
+    return parsed.filter(
+      (entry): entry is ActiveBlockStartSnapshot =>
+        typeof entry === 'object' &&
+        entry !== null &&
+        typeof (entry as ActiveBlockStartSnapshot).eventId === 'number' &&
+        typeof (entry as ActiveBlockStartSnapshot).data?.blockId === 'string'
+    )
+  } catch {
+    return undefined
+  }
+}
+
 export async function readExecutionEventsState(
   executionId: string,
   afterEventId: number
 ): Promise<ExecutionEventsReadResult> {
   const redis = getRedisClient()
   if (!redis) {
-    if (canUseMemoryEventBuffer()) {
-      return readMemoryEvents(executionId, afterEventId)
-    }
+    if (canUseMemoryEventBuffer()) return readMemoryEvents(executionId, afterEventId)
     return { status: 'unavailable', error: 'Redis client unavailable' }
   }
   try {
@@ -726,20 +910,7 @@ export function createExecutionEventWriter(
       logger.info('createExecutionEventWriter: using in-memory event buffer', { executionId })
       return createMemoryExecutionEventWriter(executionId, context)
     }
-    logger.warn(
-      'createExecutionEventWriter: Redis client unavailable, events will not be buffered',
-      {
-        executionId,
-      }
-    )
-    return {
-      write: async (event) => ({ eventId: 0, executionId, event }),
-      writeTerminal: async () => {
-        throw new Error(`Execution event buffer unavailable for ${executionId}`)
-      },
-      flush: async () => {},
-      close: async () => {},
-    }
+    throw new Error(`Redis execution event buffer unavailable for ${executionId}`)
   }
 
   let pending: ExecutionEventEntry[] = []
@@ -901,12 +1072,13 @@ export function createExecutionEventWriter(
       const flushResult = getFlushScriptResult(
         await redis.eval(
           FLUSH_EVENTS_SCRIPT,
-          3 + budgetKeys.length,
+          4 + budgetKeys.length,
           key,
           getSeqKey(executionId),
           getMetaKey(executionId),
+          getExecutionSignalChannel(executionId),
           ...budgetKeys,
-          TTL_SECONDS,
+          chunkTerminalStatus ? TERMINAL_TTL_SECONDS : ACTIVE_TTL_SECONDS,
           EVENT_LIMIT,
           new Date().toISOString(),
           chunkTerminalStatus ?? '',
@@ -1118,9 +1290,8 @@ export function createExecutionEventWriter(
         throw error
       }
       // `pendingTerminalStatus` still being set means the status never reached
-      // Redis even though the events did, which would leave readers polling an
-      // `active` stream. Treat it as a failed terminal publish so the caller
-      // degrades instead of reporting a success that readers cannot observe.
+      // Redis even though the events did. Treat it as a failed terminal publish
+      // instead of reporting a success that readers cannot observe.
       if (!ok || pendingTerminalStatus) {
         discardTerminalEntry(entry)
         // Report why when the budget was the cause, so the run surfaces the

apps/sim/lib/execution/execution-signal.test.ts

@@ -0,0 +1,143 @@
+/**
+ * @vitest-environment node
+ */
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const { listeners, mockRedisUrl, mockSubscribe, mockUnsubscribe } = vi.hoisted(() => ({
+  listeners: new Map<string, (...args: unknown[]) => void>(),
+  mockRedisUrl: { value: 'redis://localhost:6379' as string | undefined },
+  mockSubscribe: vi.fn(),
+  mockUnsubscribe: vi.fn(),
+}))
+
+vi.mock('ioredis', () => ({
+  default: class {
+    on(event: string, handler: (...args: unknown[]) => void) {
+      listeners.set(event, handler)
+      return this
+    }
+
+    subscribe = mockSubscribe
+    unsubscribe = mockUnsubscribe
+  },
+}))
+
+vi.mock('@/lib/core/config/redis', () => ({
+  getConfiguredRedisUrl: () => mockRedisUrl.value,
+  getRedisConnectionDefaults: () => ({}),
+}))
+
+import {
+  getExecutionSignalHub,
+  publishLocalExecutionSignal,
+} from '@/lib/execution/execution-signal'
+
+describe('ExecutionSignalHub', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    listeners.clear()
+    mockSubscribe.mockResolvedValue(1)
+    mockUnsubscribe.mockResolvedValue(0)
+    mockRedisUrl.value = 'redis://localhost:6379'
+    const signalGlobal = globalThis as typeof globalThis & { _executionSignalHub?: unknown }
+    signalGlobal._executionSignalHub = undefined
+  })
+
+  it('waits for one shared subscription acknowledgement before resolving concurrent subscribers', async () => {
+    let acknowledge: (() => void) | undefined
+    mockSubscribe.mockReturnValueOnce(
+      new Promise<number>((resolve) => {
+        acknowledge = () => resolve(2)
+      })
+    )
+    const hub = getExecutionSignalHub()
+    const first = hub.subscribe('execution-1', vi.fn())
+    const second = hub.subscribe('execution-1', vi.fn())
+    let secondResolved = false
+    void second.then(() => {
+      secondResolved = true
+    })
+
+    await Promise.resolve()
+    expect(mockSubscribe).toHaveBeenCalledOnce()
+    expect(secondResolved).toBe(false)
+
+    acknowledge?.()
+    await Promise.all([first, second])
+    expect(secondResolved).toBe(true)
+  })
+
+  it('marks every affected subscription unavailable when reconnect acknowledgement fails', async () => {
+    const hub = getExecutionSignalHub()
+    listeners.get('ready')?.()
+    const handler = vi.fn()
+    await hub.subscribe('execution-1', handler)
+    mockSubscribe.mockRejectedValueOnce(new Error('Redis unavailable'))
+
+    listeners.get('ready')?.()
+
+    await vi.waitFor(() => expect(handler).toHaveBeenCalledWith('unavailable'))
+  })
+
+  it('delivers legacy rolling-deployment cancellations to the matching execution', async () => {
+    const hub = getExecutionSignalHub()
+    const firstHandler = vi.fn()
+    const secondHandler = vi.fn()
+    await hub.subscribe('execution-1', firstHandler)
+    await hub.subscribe('execution-2', secondHandler)
+
+    expect(mockSubscribe).toHaveBeenCalledWith('execution:signal:execution-1', 'execution:cancel')
+    listeners.get('message')?.('execution:cancel', JSON.stringify({ executionId: 'execution-1' }))
+
+    expect(firstHandler).toHaveBeenCalledWith('cancelled')
+    expect(secondHandler).not.toHaveBeenCalled()
+  })
+
+  it('deduplicates cancellations published to both rollout channels', async () => {
+    const hub = getExecutionSignalHub()
+    const handler = vi.fn()
+    await hub.subscribe('execution-1', handler)
+
+    listeners.get('message')?.(
+      'execution:cancel',
+      JSON.stringify({ executionId: 'execution-1', executionSignalPublished: true })
+    )
+    listeners.get('message')?.('execution:signal:execution-1', 'cancelled')
+
+    expect(handler).toHaveBeenCalledOnce()
+    expect(handler).toHaveBeenCalledWith('cancelled')
+  })
+
+  it('does not deliver a stale reconnect failure to a replacement subscriber', async () => {
+    const hub = getExecutionSignalHub()
+    listeners.get('ready')?.()
+    const oldHandler = vi.fn()
+    const unsubscribeOld = await hub.subscribe('execution-1', oldHandler)
+    let rejectOldReconnect!: (error: Error) => void
+    mockSubscribe.mockReturnValueOnce(
+      new Promise<number>((_resolve, reject) => {
+        rejectOldReconnect = reject
+      })
+    )
+
+    listeners.get('ready')?.()
+    unsubscribeOld()
+    mockSubscribe.mockResolvedValueOnce(1)
+    const replacement = vi.fn()
+    await hub.subscribe('execution-1', replacement)
+    rejectOldReconnect(new Error('stale reconnect failed'))
+
+    await vi.waitFor(() => expect(mockSubscribe).toHaveBeenCalledTimes(3))
+    expect(replacement).not.toHaveBeenCalledWith('unavailable')
+  })
+
+  it('uses a process-local signal hub when Redis is not configured', async () => {
+    mockRedisUrl.value = undefined
+    const handler = vi.fn()
+    await getExecutionSignalHub().subscribe('execution-local', handler)
+
+    publishLocalExecutionSignal('execution-local', 'event')
+
+    expect(handler).toHaveBeenCalledWith('event')
+  })
+})

apps/sim/lib/execution/execution-signal.ts

@@ -0,0 +1,212 @@
+import { createLogger } from '@sim/logger'
+import { toError } from '@sim/utils/errors'
+import { isRecordLike } from '@sim/utils/object'
+import Redis, { type RedisOptions } from 'ioredis'
+import { getConfiguredRedisUrl, getRedisConnectionDefaults } from '@/lib/core/config/redis'
+
+const logger = createLogger('ExecutionSignalHub')
+const EXECUTION_SIGNAL_PREFIX = 'execution:signal:'
+export const LEGACY_EXECUTION_CANCEL_CHANNEL = 'execution:cancel'
+
+export type ExecutionSignalReason = 'event' | 'cancelled' | 'reconnected' | 'unavailable'
+export type ExecutionSignalHandler = (reason: ExecutionSignalReason) => void
+
+export interface ExecutionSignalHub {
+  subscribe(executionId: string, handler: ExecutionSignalHandler): Promise<() => void>
+}
+
+export function getExecutionSignalChannel(executionId: string): string {
+  return `${EXECUTION_SIGNAL_PREFIX}${executionId}`
+}
+
+class RedisExecutionSignalHub implements ExecutionSignalHub {
+  private readonly subscriber: Redis
+  private readonly handlers = new Map<string, Set<ExecutionSignalHandler>>()
+  private readonly subscriptionReady = new Map<string, Promise<void>>()
+  private connectedOnce = false
+
+  constructor(redisUrl: string) {
+    const options = {
+      ...getRedisConnectionDefaults(redisUrl),
+      commandTimeout: 5000,
+      connectionName: 'execution-signal-hub',
+      maxRetriesPerRequest: null,
+      retryStrategy: (attempt: number) => Math.min(attempt * 500, 5000),
+    } satisfies RedisOptions
+    this.subscriber = new Redis(redisUrl, options)
+    this.subscriber.on('message', (channel: string, message: string) => {
+      if (channel === LEGACY_EXECUTION_CANCEL_CHANNEL) {
+        try {
+          const event: unknown = JSON.parse(message)
+          if (isRecordLike(event) && typeof event.executionId === 'string') {
+            if (event.executionSignalPublished === true) return
+            this.dispatch(getExecutionSignalChannel(event.executionId), 'cancelled')
+          }
+        } catch (error) {
+          logger.warn('Ignored malformed legacy execution cancellation signal', {
+            error: toError(error).message,
+          })
+        }
+        return
+      }
+      this.dispatch(channel, message === 'cancelled' ? 'cancelled' : 'event')
+    })
+    this.subscriber.on('ready', () => {
+      void this.handleReady()
+    })
+    this.subscriber.on('error', (error: Error) => {
+      logger.error('Execution signal subscriber error', { error: error.message })
+    })
+  }
+
+  async subscribe(executionId: string, handler: ExecutionSignalHandler): Promise<() => void> {
+    const channel = getExecutionSignalChannel(executionId)
+    let channelHandlers = this.handlers.get(channel)
+    if (!channelHandlers) {
+      channelHandlers = new Set()
+      this.handlers.set(channel, channelHandlers)
+    }
+    channelHandlers.add(handler)
+
+    let ready = this.subscriptionReady.get(channel)
+    if (!ready) {
+      ready = this.subscriber
+        .subscribe(channel, LEGACY_EXECUTION_CANCEL_CHANNEL)
+        .then(() => undefined)
+      this.subscriptionReady.set(channel, ready)
+    }
+    try {
+      await ready
+    } catch (error) {
+      if (this.subscriptionReady.get(channel) === ready) {
+        this.subscriptionReady.delete(channel)
+      }
+      channelHandlers.delete(handler)
+      if (channelHandlers.size === 0) this.handlers.delete(channel)
+      throw new Error(`Execution signal subscription failed: ${toError(error).message}`, {
+        cause: error,
+      })
+    }
+
+    return () => {
+      const current = this.handlers.get(channel)
+      if (!current) return
+      current.delete(handler)
+      if (current.size > 0) return
+      this.handlers.delete(channel)
+      this.subscriptionReady.delete(channel)
+      void this.subscriber.unsubscribe(channel).catch((error) => {
+        logger.warn('Execution signal unsubscribe failed', {
+          channel,
+          error: toError(error).message,
+        })
+      })
+    }
+  }
+
+  private async handleReady(): Promise<void> {
+    const reconnect = this.connectedOnce
+    this.connectedOnce = true
+    if (!reconnect || this.handlers.size === 0) return
+
+    const channels = [...this.handlers.keys()]
+    const ready = this.subscriber
+      .subscribe(...channels, LEGACY_EXECUTION_CANCEL_CHANNEL)
+      .then(() => undefined)
+    for (const channel of channels) {
+      if (this.handlers.has(channel)) this.subscriptionReady.set(channel, ready)
+    }
+    try {
+      await ready
+      for (const channel of channels) {
+        if (this.handlers.has(channel) && this.subscriptionReady.get(channel) === ready) {
+          this.dispatch(channel, 'reconnected')
+        } else if (!this.handlers.has(channel)) {
+          void this.subscriber.unsubscribe(channel)
+        }
+      }
+    } catch (error) {
+      logger.error('Execution signal resubscription failed', { error: toError(error).message })
+      for (const channel of channels) {
+        if (this.subscriptionReady.get(channel) !== ready) continue
+        this.subscriptionReady.delete(channel)
+        this.dispatch(channel, 'unavailable')
+      }
+    }
+  }
+
+  private dispatch(channel: string, reason: ExecutionSignalReason): void {
+    const channelHandlers = this.handlers.get(channel)
+    if (!channelHandlers) return
+    for (const handler of channelHandlers) {
+      try {
+        handler(reason)
+      } catch (error) {
+        logger.error('Execution signal handler failed', { error: toError(error).message })
+      }
+    }
+  }
+}
+
+class LocalExecutionSignalHub implements ExecutionSignalHub {
+  private readonly handlers = new Map<string, Set<ExecutionSignalHandler>>()
+
+  async subscribe(executionId: string, handler: ExecutionSignalHandler): Promise<() => void> {
+    const channel = getExecutionSignalChannel(executionId)
+    let channelHandlers = this.handlers.get(channel)
+    if (!channelHandlers) {
+      channelHandlers = new Set()
+      this.handlers.set(channel, channelHandlers)
+    }
+    channelHandlers.add(handler)
+    return () => {
+      const current = this.handlers.get(channel)
+      if (!current) return
+      current.delete(handler)
+      if (current.size === 0) this.handlers.delete(channel)
+    }
+  }
+
+  publish(
+    executionId: string,
+    reason: Extract<ExecutionSignalReason, 'event' | 'cancelled'>
+  ): void {
+    const channelHandlers = this.handlers.get(getExecutionSignalChannel(executionId))
+    if (!channelHandlers) return
+    for (const handler of channelHandlers) {
+      try {
+        handler(reason)
+      } catch (error) {
+        logger.error('Local execution signal handler failed', { error: toError(error).message })
+      }
+    }
+  }
+}
+
+type ExecutionSignalGlobal = typeof globalThis & {
+  _executionSignalHub?: ExecutionSignalHub
+}
+
+const executionSignalGlobal = globalThis as ExecutionSignalGlobal
+
+export function getExecutionSignalHub(): ExecutionSignalHub {
+  if (executionSignalGlobal._executionSignalHub) return executionSignalGlobal._executionSignalHub
+  const redisUrl = getConfiguredRedisUrl()
+  if (!redisUrl) {
+    executionSignalGlobal._executionSignalHub = new LocalExecutionSignalHub()
+    return executionSignalGlobal._executionSignalHub
+  }
+  executionSignalGlobal._executionSignalHub = new RedisExecutionSignalHub(redisUrl)
+  return executionSignalGlobal._executionSignalHub
+}
+
+export function publishLocalExecutionSignal(
+  executionId: string,
+  reason: Extract<ExecutionSignalReason, 'event' | 'cancelled'>
+): void {
+  const hub = getExecutionSignalHub()
+  if (!(hub instanceof LocalExecutionSignalHub)) {
+    throw new Error('Local execution signals are unavailable while Redis is configured')
+  }
+  hub.publish(executionId, reason)
+}

apps/sim/lib/execution/isolated-vm.test.ts

@@ -498,7 +498,7 @@ describe('isolated-vm scheduler', () => {
     expect(result.error?.message).toContain('Too many concurrent')
   })
 
-  it('falls back to local execution when Redis is configured but unavailable', async () => {
+  it('fails closed when Redis is configured but unavailable', async () => {
     const { executeInIsolatedVM } = await loadExecutionModule({
       envOverrides: {
         REDIS_URL: 'redis://localhost:6379',
@@ -516,11 +516,14 @@ describe('isolated-vm scheduler', () => {
       ownerKey: 'user:redis-down',
     })
 
-    expect(result.error).toBeUndefined()
-    expect(result.result).toBe('ok')
+    expect(result.error).toMatchObject({
+      isSystemError: true,
+      message: 'Code execution coordination is temporarily unavailable. Please try again later.',
+    })
+    expect(result.result).toBeNull()
   })
 
-  it('falls back to local execution when Redis lease evaluation errors', async () => {
+  it('fails closed when Redis lease evaluation errors', async () => {
     const { executeInIsolatedVM } = await loadExecutionModule({
       envOverrides: {
         REDIS_URL: 'redis://localhost:6379',
@@ -545,8 +548,59 @@ describe('isolated-vm scheduler', () => {
       ownerKey: 'user:redis-error',
     })
 
-    expect(result.error).toBeUndefined()
-    expect(result.result).toBe('ok')
+    expect(result.error).toMatchObject({
+      isSystemError: true,
+      message: 'Code execution coordination is temporarily unavailable. Please try again later.',
+    })
+    expect(result.result).toBeNull()
+  })
+
+  it('reports cancellation when abort races a rejected distributed lease', async () => {
+    let resolveLease!: (value: number) => void
+    let markLeaseRequested!: () => void
+    const leaseResult = new Promise<number>((resolve) => {
+      resolveLease = resolve
+    })
+    const leaseRequested = new Promise<void>((resolve) => {
+      markLeaseRequested = resolve
+    })
+    const { executeInIsolatedVM, spawnMock } = await loadExecutionModule({
+      envOverrides: {
+        REDIS_URL: 'redis://localhost:6379',
+      },
+      spawns: [() => createReadyProc('unused')],
+      redisEvalImpl: (...args: unknown[]) => {
+        const script = String(args[0] ?? '')
+        if (script.includes('ZREMRANGEBYSCORE')) {
+          markLeaseRequested()
+          return leaseResult
+        }
+        return 1
+      },
+    })
+    const controller = new AbortController()
+
+    const execution = executeInIsolatedVM(
+      {
+        code: 'return "unused"',
+        params: {},
+        envVars: {},
+        contextVariables: {},
+        timeoutMs: 100,
+        requestId: 'req-abort-lease',
+        ownerKey: 'user:cancelled',
+      },
+      { signal: controller.signal }
+    )
+    await leaseRequested
+    controller.abort(new DOMException('user', 'AbortError'))
+    resolveLease(0)
+
+    await expect(execution).resolves.toMatchObject({
+      termination: 'cancelled',
+      error: { name: 'AbortError' },
+    })
+    expect(spawnMock).not.toHaveBeenCalled()
   })
 
   it('applies weighted owner scheduling when draining queued executions', async () => {

apps/sim/lib/execution/isolated-vm.ts

@@ -407,7 +407,7 @@ async function tryAcquireDistributedLease(
     ])
     return Number(result) === 1 ? 'acquired' : 'limit_exceeded'
   } catch (error) {
-    logger.warn('Failed to acquire distributed owner lease — falling back to local execution', {
+    logger.error('Failed to acquire distributed owner lease; execution will be rejected', {
       ownerKey,
       error,
     })
@@ -1390,6 +1390,15 @@ export async function executeInIsolatedVM(
     distributedLeaseId,
     req.timeoutMs
   )
+  if (leaseAcquireResult !== 'acquired' && signal?.aborted) {
+    maybeCleanupOwner(ownerKey)
+    return {
+      result: null,
+      stdout: '',
+      error: { message: 'Execution cancelled', name: 'AbortError' },
+      termination: 'cancelled',
+    }
+  }
   if (leaseAcquireResult === 'limit_exceeded') {
     logger.warn('Isolated-vm saturation: distributed lease limit exceeded', {
       reason: 'distributed_lease_limit',
@@ -1408,8 +1417,19 @@ export async function executeInIsolatedVM(
     }
   }
   if (leaseAcquireResult === 'unavailable') {
-    logger.warn('Distributed lease unavailable, falling back to local execution', { ownerKey })
-    // Continue execution — local pool still enforces per-process concurrency limits
+    logger.error('Isolated-vm execution rejected because its distributed lease is unavailable', {
+      ownerKey,
+    })
+    maybeCleanupOwner(ownerKey)
+    return {
+      result: null,
+      stdout: '',
+      error: {
+        message: 'Code execution coordination is temporarily unavailable. Please try again later.',
+        name: 'Error',
+        isSystemError: true,
+      },
+    }
   }
 
   let settled = false

apps/sim/lib/execution/non-retryable-error.ts

@@ -0,0 +1,47 @@
+import { findCause, getErrorMessage } from '@sim/utils/errors'
+
+/** A failure whose operation may already have produced externally visible side effects. */
+export class NonRetryableExecutionError extends Error {
+  readonly retryable = false as const
+
+  constructor(message: string, options?: ErrorOptions) {
+    super(message, options)
+    this.name = 'NonRetryableExecutionError'
+  }
+}
+
+/** The provider may have started user code but did not return a recoverable process identity. */
+export class SandboxLaunchIndeterminateError extends NonRetryableExecutionError {
+  readonly code = 'sandbox_launch_indeterminate' as const
+
+  constructor(provider: string, options?: ErrorOptions) {
+    const providerDetail = options?.cause
+      ? ` Provider detail: ${getErrorMessage(options.cause)}`
+      : ''
+    super(
+      `${provider} may have started this Function, but Sim could not recover its process identity. The outcome is indeterminate and the code was not run again.${providerDetail}`,
+      options
+    )
+    this.name = 'SandboxLaunchIndeterminateError'
+  }
+}
+
+export function isNonRetryableExecutionError(error: unknown): boolean {
+  return Boolean(
+    findCause(error, (cause): cause is NonRetryableExecutionError => {
+      return (
+        cause instanceof NonRetryableExecutionError ||
+        (typeof cause === 'object' &&
+          cause !== null &&
+          'retryable' in cause &&
+          cause.retryable === false)
+      )
+    })
+  )
+}
+
+export function isSandboxLaunchIndeterminateError(
+  error: unknown
+): error is SandboxLaunchIndeterminateError {
+  return Boolean(findCause(error, (cause) => cause instanceof SandboxLaunchIndeterminateError))
+}

apps/sim/lib/execution/remote-sandbox/conformance.test.ts

@@ -17,6 +17,7 @@ const {
   mockE2BCreate,
   mockE2BRunCode,
   mockE2BCommandsRun,
+  mockE2BCommandsConnect,
   mockE2BFilesGetInfo,
   mockE2BFilesRead,
   mockE2BFilesRemove,
@@ -37,6 +38,7 @@ const {
   mockExecuteSessionCommand,
   mockGetSessionCommandLogs,
   mockGetSessionCommand,
+  mockSendSessionCommandInput,
   mockDeleteSession,
   mockRecordSandboxProviderLimit,
   mockRecordSandboxTeardownFailure,
@@ -64,6 +66,7 @@ const {
   mockE2BCreate: vi.fn(),
   mockE2BRunCode: vi.fn(),
   mockE2BCommandsRun: vi.fn(),
+  mockE2BCommandsConnect: vi.fn(),
   mockE2BFilesGetInfo: vi.fn(),
   mockE2BFilesRead: vi.fn(),
   mockE2BFilesRemove: vi.fn(),
@@ -84,6 +87,7 @@ const {
   mockExecuteSessionCommand: vi.fn(),
   mockGetSessionCommandLogs: vi.fn(),
   mockGetSessionCommand: vi.fn(),
+  mockSendSessionCommandInput: vi.fn(),
   mockDeleteSession: vi.fn(),
   mockRecordSandboxProviderLimit: vi.fn(),
   mockRecordSandboxTeardownFailure: vi.fn(),
@@ -145,6 +149,13 @@ function providerTimeoutError(provider: Provider): Error {
   return error
 }
 
+function emitDaytonaStreamReady(onStdout: (chunk: string) => void): void {
+  const command = mockExecuteSessionCommand.mock.calls.at(-1)?.[1]?.command
+  if (typeof command === 'string' && command.includes('__SIM_DAYTONA_STREAM_READY__')) {
+    onStdout('__SIM_DAYTONA_STREAM_READY__\n')
+  }
+}
+
 /** Stubs a code execution that prints `stdout` and emits `result` via the marker. */
 function stubCodeRun(provider: Provider, stdout: string) {
   if (provider === 'e2b') {
@@ -152,6 +163,7 @@ function stubCodeRun(provider: Provider, stdout: string) {
   } else {
     mockGetSessionCommandLogs.mockImplementation(
       async (_sessionId: string, _commandId: string, onStdout: (chunk: string) => void) => {
+        emitDaytonaStreamReady(onStdout)
         if (stdout) onStdout(stdout)
       }
     )
@@ -200,6 +212,7 @@ function stubShellCommand(provider: Provider, stdout: string, stderr: string, ex
       onStdout: (chunk: string) => void,
       onStderr: (chunk: string) => void
     ) => {
+      emitDaytonaStreamReady(onStdout)
       if (stdout) onStdout(stdout)
       if (stderr) onStderr(stderr)
     }
@@ -221,7 +234,7 @@ function capturedSandboxWrites(provider: Provider): CapturedSandboxWrite[] {
         content: call[1],
         order: mockE2BFilesWrite.mock.invocationCallOrder[index],
       }))
-      .filter((write) => !write.path.startsWith('.sim-function-'))
+      .filter((write) => !write.path.includes('.sim-function-'))
   }
   return mockUploadFile.mock.calls
     .map((call, index) => ({
@@ -229,7 +242,7 @@ function capturedSandboxWrites(provider: Provider): CapturedSandboxWrite[] {
       content: call[0],
       order: mockUploadFile.mock.invocationCallOrder[index],
     }))
-    .filter((write) => !write.path.startsWith('.sim-function-'))
+    .filter((write) => !write.path.includes('.sim-function-'))
 }
 
 function sandboxWriteBuffer(content: unknown): Buffer {
@@ -247,7 +260,7 @@ beforeEach(() => {
   mockE2BCreate.mockResolvedValue({
     sandboxId: 'sb_1',
     runCode: mockE2BRunCode,
-    commands: { run: mockE2BCommandsRun },
+    commands: { run: mockE2BCommandsRun, connect: mockE2BCommandsConnect },
     files: {
       getInfo: mockE2BFilesGetInfo,
       read: mockE2BFilesRead,
@@ -257,6 +270,9 @@ beforeEach(() => {
     kill: mockE2BKill,
   })
   mockE2BCommandsRun.mockResolvedValue({ stdout: '', stderr: '', exitCode: 0 })
+  mockE2BCommandsConnect.mockResolvedValue({
+    wait: vi.fn().mockResolvedValue({ stdout: '', stderr: '', exitCode: 0 }),
+  })
   mockE2BFilesRemove.mockResolvedValue(undefined)
   mockE2BKill.mockResolvedValue(undefined)
 
@@ -270,6 +286,7 @@ beforeEach(() => {
       executeSessionCommand: mockExecuteSessionCommand,
       getSessionCommandLogs: mockGetSessionCommandLogs,
       getSessionCommand: mockGetSessionCommand,
+      sendSessionCommandInput: mockSendSessionCommandInput,
       deleteSession: mockDeleteSession,
     },
     fs: {
@@ -292,6 +309,7 @@ beforeEach(() => {
   mockProvisionRuntime.mockResolvedValue(undefined)
   mockExecuteSessionCommand.mockResolvedValue({ cmdId: 'cmd_1' })
   mockGetSessionCommand.mockResolvedValue({ exitCode: 0 })
+  mockSendSessionCommandInput.mockResolvedValue(undefined)
 })
 
 /** Headroom for the note `tailStreamedSandboxOutput` prepends when it truncates. */
@@ -363,6 +381,7 @@ describe.each(PROVIDERS)('sandbox conformance [%s]', (provider) => {
       })
     } else {
       mockGetSessionCommandLogs.mockImplementationOnce(async (_sessionId, _commandId, onStdout) => {
+        emitDaytonaStreamReady(onStdout)
         onStdout(oversized)
       })
     }
@@ -812,6 +831,7 @@ describe.each(PROVIDERS)('sandbox conformance [%s]', (provider) => {
       })
     } else {
       mockGetSessionCommandLogs.mockImplementationOnce(async (_sessionId, _commandId, onStdout) => {
+        emitDaytonaStreamReady(onStdout)
         onStdout(oversized)
       })
     }
@@ -1183,6 +1203,220 @@ describe('E2B streamed output safety', () => {
   })
 })
 
+describe('provider stream recovery', () => {
+  it('fails an at-most-once E2B run closed when its output stream disconnects', async () => {
+    const wait = vi.fn().mockRejectedValueOnce(new Error('stream disconnected'))
+    mockE2BCommandsRun.mockResolvedValueOnce({ pid: 42, wait })
+    const sandbox = await e2bProvider.create('code')
+
+    await expect(
+      sandbox.runCommand('node function.js', { timeoutMs: 1000, atMostOnce: true })
+    ).rejects.toMatchObject({
+      name: 'SandboxLaunchIndeterminateError',
+      retryable: false,
+      code: 'sandbox_launch_indeterminate',
+    })
+
+    expect(mockE2BCommandsRun).toHaveBeenCalledTimes(1)
+    expect(mockE2BCommandsConnect).not.toHaveBeenCalled()
+  })
+
+  it('forwards E2B output received after reconnecting to the exact process', async () => {
+    const wait = vi.fn().mockRejectedValueOnce(new Error('stream disconnected'))
+    mockE2BCommandsRun.mockImplementationOnce(async (_command, options) => {
+      options.onStdout?.('before')
+      return { pid: 42, wait }
+    })
+    mockE2BCommandsConnect.mockImplementationOnce(async (_pid, options) => {
+      options.onStdout?.('after')
+      options.onStderr?.('warning')
+      return {
+        wait: vi.fn().mockResolvedValue({ stdout: 'after', stderr: 'warning', exitCode: 0 }),
+      }
+    })
+    const sandbox = await e2bProvider.create('code')
+    const stdout: string[] = []
+    const stderr: string[] = []
+
+    await expect(
+      sandbox.runCommand('node function.js', {
+        timeoutMs: 1000,
+        onStdout: (chunk) => stdout.push(chunk),
+        onStderr: (chunk) => stderr.push(chunk),
+      })
+    ).resolves.toMatchObject({ stdout: 'beforeafter', stderr: 'warning', exitCode: 0 })
+
+    expect(stdout).toEqual(['before', 'after'])
+    expect(stderr).toEqual(['warning'])
+    expect(mockE2BCommandsRun).toHaveBeenCalledTimes(1)
+    expect(mockE2BCommandsConnect).toHaveBeenCalledWith(42, expect.any(Object))
+  })
+
+  it('bounds E2B output received while reconnecting to the exact process', async () => {
+    const wait = vi.fn().mockRejectedValueOnce(new Error('stream disconnected'))
+    mockE2BCommandsRun.mockResolvedValueOnce({ pid: 42, wait })
+    mockE2BCommandsConnect.mockImplementationOnce(async (_pid, options) => {
+      options.onStdout?.('12345')
+      return { wait: vi.fn() }
+    })
+    const sandbox = await e2bProvider.create('code')
+
+    await expect(
+      sandbox.runCommand('pi run', {
+        timeoutMs: 1000,
+        maxOutputBytes: 4,
+      })
+    ).rejects.toMatchObject({
+      code: 'sandbox_output_limit_exceeded',
+      attemptedBytes: 5,
+      limitBytes: 4,
+    })
+    expect(mockE2BCommandsRun).toHaveBeenCalledTimes(1)
+    expect(mockE2BCommandsConnect).toHaveBeenCalledTimes(1)
+  })
+
+  it('marks an ambiguous E2B start as indeterminate without retrying', async () => {
+    mockE2BCommandsRun.mockRejectedValueOnce(new Error('connection reset during start'))
+    const sandbox = await e2bProvider.create('code')
+
+    await expect(
+      sandbox.runCommand('node function.js', { timeoutMs: 1000, atMostOnce: true })
+    ).rejects.toMatchObject({
+      name: 'SandboxLaunchIndeterminateError',
+      retryable: false,
+      code: 'sandbox_launch_indeterminate',
+    })
+    expect(mockE2BCommandsRun).toHaveBeenCalledTimes(1)
+    expect(mockE2BCommandsConnect).not.toHaveBeenCalled()
+  })
+
+  it('reattaches to the exact Daytona session command after a stream disconnect', async () => {
+    mockGetSessionCommandLogs
+      .mockRejectedValueOnce(new Error('stream disconnected'))
+      .mockImplementationOnce(
+        async (_sessionId: string, _commandId: string, onStdout: (chunk: string) => void) => {
+          onStdout('__SIM_DAYTONA_STREAM_READY__\ndone')
+        }
+      )
+    const sandbox = await daytonaProvider.create('code')
+
+    await expect(
+      sandbox.runCommand('node function.js', { timeoutMs: 1000, atMostOnce: true })
+    ).resolves.toMatchObject({ stdout: 'done', exitCode: 0 })
+
+    expect(mockExecuteSessionCommand).toHaveBeenCalledTimes(1)
+    expect(mockExecuteSessionCommand.mock.calls[0]?.[1]).toMatchObject({
+      runAsync: true,
+      suppressInputEcho: true,
+    })
+    expect(mockGetSessionCommandLogs).toHaveBeenCalledTimes(2)
+    expect(mockGetSessionCommand).toHaveBeenCalledWith(expect.any(String), 'cmd_1')
+  })
+
+  it('keeps the original Daytona deadline while recovering a disconnected stream', async () => {
+    mockGetSessionCommandLogs
+      .mockRejectedValueOnce(new Error('stream disconnected'))
+      .mockImplementationOnce(() => new Promise<void>(() => {}))
+    const sandbox = await daytonaProvider.create('code')
+
+    await expect(
+      sandbox.runCommand('node function.js', { timeoutMs: 25, atMostOnce: true })
+    ).resolves.toMatchObject({ exitCode: 124, timedOut: true })
+
+    expect(mockExecuteSessionCommand).toHaveBeenCalledTimes(1)
+    expect(mockGetSessionCommandLogs).toHaveBeenCalledTimes(2)
+    expect(mockGetSessionCommand).not.toHaveBeenCalled()
+    expect(mockDeleteSession).toHaveBeenCalledTimes(1)
+  })
+
+  it('preserves a legitimate leading newline in Daytona stdout', async () => {
+    mockGetSessionCommandLogs.mockImplementationOnce(
+      async (_sessionId: string, _commandId: string, onStdout: (chunk: string) => void) => {
+        onStdout('__SIM_DAYTONA_STREAM_READY__\n')
+        onStdout('\nhello')
+      }
+    )
+    const sandbox = await daytonaProvider.create('code')
+
+    await expect(
+      sandbox.runCommand('printf "\\nhello"', { timeoutMs: 1000, atMostOnce: true })
+    ).resolves.toMatchObject({ stdout: '\nhello', exitCode: 0 })
+  })
+
+  it('bounds Daytona output replayed while reconnecting to the exact command', async () => {
+    mockGetSessionCommandLogs
+      .mockRejectedValueOnce(new Error('stream disconnected'))
+      .mockImplementationOnce(
+        async (_sessionId: string, _commandId: string, onStdout: (chunk: string) => void) => {
+          onStdout('__SIM_DAYTONA_STREAM_READY__\n12345')
+        }
+      )
+    const sandbox = await daytonaProvider.create('code')
+
+    await expect(
+      sandbox.runCommand('node function.js', {
+        timeoutMs: 1000,
+        maxOutputBytes: 4,
+        atMostOnce: true,
+      })
+    ).rejects.toMatchObject({
+      code: 'sandbox_output_limit_exceeded',
+      attemptedBytes: 5,
+      limitBytes: 4,
+    })
+    expect(mockExecuteSessionCommand).toHaveBeenCalledTimes(1)
+    expect(mockGetSessionCommandLogs).toHaveBeenCalledTimes(2)
+  })
+
+  it('does not redeliver callback-consumed Daytona output beyond the retained diagnostic tail', async () => {
+    const beforeDisconnect = 'x'.repeat(MAX_SANDBOX_STREAMED_OUTPUT_TAIL_BYTES * 2)
+    mockGetSessionCommandLogs
+      .mockImplementationOnce(
+        async (_sessionId: string, _commandId: string, onStdout: (chunk: string) => void) => {
+          onStdout('__SIM_DAYTONA_STREAM_READY__\n')
+          onStdout(beforeDisconnect)
+          throw new Error('stream disconnected')
+        }
+      )
+      .mockImplementationOnce(
+        async (_sessionId: string, _commandId: string, onStdout: (chunk: string) => void) => {
+          onStdout(`__SIM_DAYTONA_STREAM_READY__\n${beforeDisconnect}after`)
+        }
+      )
+    const sandbox = await daytonaProvider.create('code')
+    const stdout: string[] = []
+
+    await expect(
+      sandbox.runCommand('node function.js', {
+        timeoutMs: 1000,
+        atMostOnce: true,
+        onStdout: (chunk) => stdout.push(chunk),
+      })
+    ).resolves.toMatchObject({ stdout: expect.stringContaining('after'), exitCode: 0 })
+
+    expect(stdout.join('')).toBe(`${beforeDisconnect}after`)
+    expect(mockExecuteSessionCommand).toHaveBeenCalledTimes(1)
+    expect(mockGetSessionCommandLogs).toHaveBeenCalledTimes(2)
+  })
+
+  it.each([{}, null, undefined])(
+    'marks a Daytona start without a command identity as indeterminate',
+    async (started) => {
+      mockExecuteSessionCommand.mockResolvedValueOnce(started)
+      const sandbox = await daytonaProvider.create('code')
+
+      await expect(
+        sandbox.runCommand('node function.js', { timeoutMs: 1000, atMostOnce: true })
+      ).rejects.toMatchObject({
+        name: 'SandboxLaunchIndeterminateError',
+        retryable: false,
+        code: 'sandbox_launch_indeterminate',
+      })
+      expect(mockExecuteSessionCommand).toHaveBeenCalledTimes(1)
+    }
+  )
+})
+
 describe('custom dependency sets', () => {
   it.each(PROVIDERS)('honors imageRef for code executions [%s]', async (provider) => {
     useProvider(provider)
@@ -1702,11 +1936,28 @@ describe('provider selection', () => {
         /^\.\/\.sim-function-[A-Za-z0-9_-]+\.cjs$/
       )
     } else {
+      const uploadedFunctionPaths = mockUploadFile.mock.calls
+        .map(([, path]) => String(path))
+        .filter((path) => path.includes('.sim-function-'))
+      expect(uploadedFunctionPaths).toHaveLength(3)
+      expect(uploadedFunctionPaths).toEqual(
+        expect.arrayContaining([
+          expect.stringMatching(/^\.sim-function-[A-Za-z0-9_-]+\.mjs$/),
+          expect.stringMatching(/^\.sim-function-[A-Za-z0-9_-]+\.cjs$/),
+          expect.stringMatching(/^\.sim-function-[A-Za-z0-9_-]+\.sh$/),
+        ])
+      )
       const environmentWrite = mockUploadFile.mock.calls.find(([, path]) =>
         String(path).startsWith('/tmp/.sim-env-')
       )?.[0]
-      expect(sandboxWriteBuffer(environmentWrite).toString('utf8')).toMatch(
-        /SIM_PRELOAD_PATH='\.\/\.sim-function-[A-Za-z0-9_-]+\.cjs'/
+      const environment = sandboxWriteBuffer(environmentWrite).toString('utf8')
+      expect(environment).toMatch(/SIM_CODE_PATH='\.sim-function-[A-Za-z0-9_-]+\.mjs'/)
+      expect(environment).toMatch(/SIM_PRELOAD_PATH='\.\/\.sim-function-[A-Za-z0-9_-]+\.cjs'/)
+      const launcher = mockUploadFile.mock.calls.find(([, path]) =>
+        String(path).endsWith('.sh')
+      )?.[0]
+      expect(sandboxWriteBuffer(launcher).toString('utf8')).toContain(
+        'ln -s "$SIM_NODE_MODULES_PATH" node_modules'
       )
     }
   })
@@ -1715,6 +1966,7 @@ describe('provider selection', () => {
     useProvider('daytona')
     const oversized = 'x'.repeat(MAX_SANDBOX_PROCESS_OUTPUT_BYTES + 1)
     mockGetSessionCommandLogs.mockImplementationOnce(async (_sessionId, _commandId, onStdout) => {
+      emitDaytonaStreamReady(onStdout)
       onStdout(oversized)
     })
 
@@ -1729,7 +1981,7 @@ describe('provider selection', () => {
       outputKind: 'process',
     })
     expect(mockProcessCodeRun).not.toHaveBeenCalled()
-    expect(mockDeleteFile).toHaveBeenCalledTimes(2)
+    expect(mockDeleteFile).toHaveBeenCalledTimes(3)
     expect(mockDelete).toHaveBeenCalledTimes(1)
   })
 
@@ -1737,6 +1989,7 @@ describe('provider selection', () => {
     useProvider('daytona')
     const oversized = 'x'.repeat(MAX_SANDBOX_PROCESS_OUTPUT_BYTES + 1)
     mockGetSessionCommandLogs.mockImplementationOnce(async (_sessionId, _commandId, onStdout) => {
+      emitDaytonaStreamReady(onStdout)
       onStdout(oversized)
     })
     const sandbox = await daytonaProvider.create('shell')

apps/sim/lib/execution/remote-sandbox/daytona.ts

@@ -12,8 +12,13 @@ import {
   readNodeStreamToBufferWithLimit,
 } from '@/lib/core/utils/stream-limits'
 import { CodeLanguage } from '@/lib/execution/languages'
+import {
+  isNonRetryableExecutionError,
+  SandboxLaunchIndeterminateError,
+} from '@/lib/execution/non-retryable-error'
 import {
   appendStreamedSandboxOutput,
+  assertSandboxProcessOutputWithinLimit,
   isSandboxOutputLimitError,
   MAX_SANDBOX_OUTPUT_BYTES,
   MAX_SANDBOX_PROCESS_OUTPUT_BYTES,
@@ -34,6 +39,7 @@ import type {
 
 const logger = createLogger('DaytonaSandboxProvider')
 const DAYTONA_DEFAULT_SANDBOX_TTL_MS = 24 * 60 * 60 * 1000
+const DAYTONA_STREAM_READY_MARKER = '__SIM_DAYTONA_STREAM_READY__'
 
 /** Daytona expresses every timeout in seconds; the rest of Sim works in milliseconds. */
 function toSeconds(timeoutMs: number): number {
@@ -171,6 +177,7 @@ class DaytonaSandboxHandle implements SandboxHandle {
     const isPython = this.language === CodeLanguage.Python
     const codePath = `.sim-function-${runId}.${isPython ? 'py' : 'mjs'}`
     const preloadPath = `.sim-function-${runId}.cjs`
+    const launchPath = `.sim-function-${runId}.sh`
 
     try {
       await this.writeFile(codePath, code)
@@ -179,16 +186,19 @@ class DaytonaSandboxHandle implements SandboxHandle {
           .filter(Boolean)
           .join('\n')
         await this.writeFile(preloadPath, `${preload}\n`)
+        await this.writeFile(
+          launchPath,
+          'set -e\nif [ -n "$SIM_NODE_MODULES_PATH" ] && [ ! -e node_modules ]; then ln -s "$SIM_NODE_MODULES_PATH" node_modules; fi\nnode --require "$SIM_PRELOAD_PATH" "$SIM_CODE_PATH" < /dev/null\n'
+        )
       }
 
       const result = await this.runStreamingCommand(
-        isPython
-          ? 'python3 "$SIM_CODE_PATH"'
-          : 'set -e; if [ -n "$SIM_NODE_MODULES_PATH" ] && [ ! -e node_modules ]; then ln -s "$SIM_NODE_MODULES_PATH" node_modules; fi; node --require "$SIM_PRELOAD_PATH" "$SIM_CODE_PATH"',
+        isPython ? 'python3 "$SIM_CODE_PATH"' : 'bash "$SIM_LAUNCH_PATH"',
         {
           timeoutMs: options.timeoutMs,
           maxOutputBytes: options.maxOutputBytes,
           signal: options.signal,
+          atMostOnce: true,
           envs: {
             ...options.envs,
             SIM_CODE_PATH: codePath,
@@ -196,6 +206,7 @@ class DaytonaSandboxHandle implements SandboxHandle {
               ? {
                   SIM_NODE_MODULES_PATH: options.envs?.NODE_PATH ?? '',
                   SIM_PRELOAD_PATH: `./${preloadPath}`,
+                  SIM_LAUNCH_PATH: launchPath,
                 }
               : {}),
           },
@@ -212,13 +223,34 @@ class DaytonaSandboxHandle implements SandboxHandle {
       return { text: '', stdout: result.stdout, stderr: result.stderr }
     } finally {
       await this.sandbox.fs.deleteFile(codePath, false).catch(() => {})
-      if (!isPython) await this.sandbox.fs.deleteFile(preloadPath, false).catch(() => {})
+      if (!isPython) {
+        await Promise.all([
+          this.sandbox.fs.deleteFile(preloadPath, false).catch(() => {}),
+          this.sandbox.fs.deleteFile(launchPath, false).catch(() => {}),
+        ])
+      }
     }
   }
 
   async runCommand(command: string, options: RunCommandOptions): Promise<SandboxCommandResult> {
-    // `rootUser` needs no handling: Daytona already executes commands as uid 0.
-    return this.runStreamingCommand(command, options, 'command')
+    if (!options.atMostOnce) {
+      return this.runStreamingCommand(command, options, 'command')
+    }
+
+    const commandPath = `/tmp/.sim-command-${generateShortId(12)}.sh`
+    try {
+      await this.writeFile(commandPath, command)
+      return await this.runStreamingCommand(
+        'bash "$SIM_COMMAND_PATH"',
+        {
+          ...options,
+          envs: { ...options.envs, SIM_COMMAND_PATH: commandPath },
+        },
+        'command'
+      )
+    } finally {
+      await this.sandbox.fs.deleteFile(commandPath, false).catch(() => {})
+    }
   }
 
   /**
@@ -253,6 +285,8 @@ class DaytonaSandboxHandle implements SandboxHandle {
     // stdout but not stderr still has stderr fully bounded. The failover must not change behavior.
     const retainStdout = options.onStdout === undefined
     const retainStderr = options.onStderr === undefined
+    let observedStdoutLength = 0
+    let observedStderrLength = 0
     // The appender keeps the accumulator under twice the tail so it is not re-cut on every chunk,
     // which leaves it anywhere in that band when the stream ends. E2B tails the value it returns,
     // so the final cut has to happen here too or a stream finishing between one and two tails comes
@@ -272,13 +306,32 @@ class DaytonaSandboxHandle implements SandboxHandle {
         await this.writeFile(envPath, envFile)
         script = `set -a; . ${envPath}; set +a; rm -f ${envPath}; ${command}`
       }
+      if (options.atMostOnce) {
+        script = `printf '%s\\n' '${DAYTONA_STREAM_READY_MARKER}'; IFS= read -r _ || exit 78; ${script}`
+      }
 
-      const started = await this.sandbox.process.executeSessionCommand(
-        sessionId,
-        { command: script, runAsync: true },
-        toSeconds(options.timeoutMs)
-      )
-      const commandId: string = started.cmdId ?? started.commandId
+      let started
+      try {
+        started = await this.sandbox.process.executeSessionCommand(
+          sessionId,
+          {
+            command: script,
+            runAsync: true,
+            ...(options.atMostOnce ? { suppressInputEcho: true } : {}),
+          },
+          toSeconds(options.timeoutMs)
+        )
+      } catch (error) {
+        if (options.signal?.aborted || isDaytonaExecutionTimeout(error)) throw error
+        if (options.atMostOnce) {
+          throw new SandboxLaunchIndeterminateError('Daytona', { cause: error })
+        }
+        throw error
+      }
+      const commandId = started?.cmdId ?? started?.commandId
+      if (typeof commandId !== 'string' || commandId.length === 0) {
+        throw new SandboxLaunchIndeterminateError('Daytona')
+      }
       // Accumulate the streamed chunks as well as forwarding them: callers read
       // markers out of stdout (the Pi cloud flow parses __BASE_SHA__/__CHANGED__)
       // and format failures from stderr, so returning empty strings here would
@@ -289,6 +342,9 @@ class DaytonaSandboxHandle implements SandboxHandle {
       // promise is abandoned and deleteSession (finally) tears the session down,
       // which rejects it; without the handler that would be an unhandledRejection.
       let streamError: unknown
+      let handshakeError: unknown
+      let releaseRequested = false
+      let initialHandshakeBuffer = ''
       let resolveOutputLimit!: () => void
       const outputLimitExceeded = new Promise<'output-limit'>((resolve) => {
         resolveOutputLimit = () => resolve('output-limit')
@@ -311,21 +367,53 @@ class DaytonaSandboxHandle implements SandboxHandle {
         append(chunk)
         callback?.(chunk)
       }
+      let resolveHandshakeError!: () => void
+      const handshakeFailed = new Promise<'handshake-error'>((resolve) => {
+        resolveHandshakeError = () => resolve('handshake-error')
+      })
+      const releaseCommand = (): void => {
+        if (!options.atMostOnce || releaseRequested) return
+        releaseRequested = true
+        void this.sandbox.process
+          .sendSessionCommandInput(sessionId, commandId, '\n')
+          .catch((error: unknown) => {
+            handshakeError = error
+            resolveHandshakeError()
+          })
+      }
+      const appendInitialStdout = (chunk: string): void => {
+        if (!options.atMostOnce || releaseRequested) {
+          if (!retainStdout) observedStdoutLength += chunk.length
+          appendOutput(
+            chunk,
+            (value) => {
+              stdout = retainStdout ? stdout + value : appendStreamedSandboxOutput(stdout, value)
+            },
+            retainStdout,
+            options.onStdout
+          )
+          return
+        }
+
+        initialHandshakeBuffer += chunk
+        const marker = `${DAYTONA_STREAM_READY_MARKER}\n`
+        const markerIndex = initialHandshakeBuffer.indexOf(marker)
+        if (markerIndex < 0) return
+        const remaining = initialHandshakeBuffer.slice(markerIndex + marker.length)
+        initialHandshakeBuffer = ''
+        releaseCommand()
+        if (remaining) appendInitialStdout(remaining)
+      }
+      const streamDeadlineAt = Date.now() + options.timeoutMs
       const streamed = this.sandbox.process
         .getSessionCommandLogs(
           sessionId,
           commandId,
           (chunk: string) => {
-            appendOutput(
-              chunk,
-              (value) => {
-                stdout = retainStdout ? stdout + value : appendStreamedSandboxOutput(stdout, value)
-              },
-              retainStdout,
-              options.onStdout
-            )
+            appendInitialStdout(chunk)
           },
           (chunk: string) => {
+            if (!retainStderr) observedStderrLength += chunk.length
             appendOutput(
               chunk,
               (value) => {
@@ -348,7 +436,7 @@ class DaytonaSandboxHandle implements SandboxHandle {
       // deleteSession terminates the still-running command.
       let timer: ReturnType<typeof setTimeout> | undefined
       const timedOut = new Promise<'timeout'>((resolve) => {
-        timer = setTimeout(() => resolve('timeout'), options.timeoutMs)
+        timer = setTimeout(() => resolve('timeout'), Math.max(0, streamDeadlineAt - Date.now()))
       })
       let abortListener: (() => void) | undefined
       const aborted = new Promise<'aborted'>((resolve) => {
@@ -359,9 +447,15 @@ class DaytonaSandboxHandle implements SandboxHandle {
         options.signal?.addEventListener('abort', abortListener, { once: true })
         if (options.signal?.aborted) abortListener()
       })
-      let outcome: 'done' | 'error' | 'timeout' | 'aborted' | 'output-limit'
+      let outcome: 'done' | 'error' | 'timeout' | 'aborted' | 'output-limit' | 'handshake-error'
       try {
-        outcome = await Promise.race([streamed, timedOut, aborted, outputLimitExceeded])
+        outcome = await Promise.race([
+          streamed,
+          timedOut,
+          aborted,
+          outputLimitExceeded,
+          handshakeFailed,
+        ])
       } finally {
         if (timer) clearTimeout(timer)
         if (abortListener) options.signal?.removeEventListener('abort', abortListener)
@@ -372,6 +466,9 @@ class DaytonaSandboxHandle implements SandboxHandle {
           : new DOMException('Execution cancelled', 'AbortError')
       }
       if (outcome === 'output-limit' || outputBudget.error) throw outputBudget.error
+      if (outcome === 'handshake-error') {
+        throw new SandboxLaunchIndeterminateError('Daytona', { cause: handshakeError })
+      }
       if (outcome === 'timeout') {
         return {
           stdout: finalStdout(),
@@ -383,13 +480,176 @@ class DaytonaSandboxHandle implements SandboxHandle {
       if (outcome === 'error') {
         if (outputBudget.error) throw outputBudget.error
         const timedOut = isDaytonaExecutionTimeout(streamError)
-        if (!timedOut) throw streamError
-        return {
-          stdout: finalStdout(),
-          stderr: finalStderr() || getErrorMessage(streamError),
-          exitCode: 124,
-          timedOut,
+        if (timedOut) {
+          return {
+            stdout: finalStdout(),
+            stderr: finalStderr() || getErrorMessage(streamError),
+            exitCode: 124,
+            timedOut,
+          }
         }
+
+        if (!options.atMostOnce) throw streamError
+
+        logger.warn('Daytona command event stream disconnected; reconnecting to the same process', {
+          sandboxId: this.sandboxId,
+          sessionId,
+          commandId,
+        })
+        let recoveredStdout = ''
+        let recoveredStderr = ''
+        let recoveredStdoutPrefix = ''
+        let stdoutReplayRemaining = retainStdout ? 0 : observedStdoutLength
+        let stderrReplayRemaining = retainStderr ? 0 : observedStderrLength
+        let recoveryMarkerPending = options.atMostOnce === true
+        const recoveryOutputBudget = new SandboxProcessOutputBudget(
+          options.maxOutputBytes ?? MAX_SANDBOX_PROCESS_OUTPUT_BYTES
+        )
+        const appendRecoveredOutput = (
+          chunk: string,
+          append: (value: string) => void,
+          retain: boolean,
+          callback?: (value: string) => void
+        ): void => {
+          if (retain) {
+            try {
+              recoveryOutputBudget.add(chunk)
+            } catch (outputError) {
+              void this.kill().catch(() => {})
+              throw outputError
+            }
+          }
+          append(chunk)
+          if (!retain) callback?.(chunk)
+        }
+        const skipObservedReplay = (chunk: string, remaining: number): [string, number] => {
+          const skipped = Math.min(chunk.length, remaining)
+          return [chunk.slice(skipped), remaining - skipped]
+        }
+        const appendRecoveredStdout = (chunk: string): void => {
+          if (!recoveryMarkerPending) {
+            const [suffix, remaining] = skipObservedReplay(chunk, stdoutReplayRemaining)
+            stdoutReplayRemaining = remaining
+            if (!suffix) return
+            appendRecoveredOutput(
+              suffix,
+              (value) => {
+                recoveredStdout = retainStdout
+                  ? recoveredStdout + value
+                  : appendStreamedSandboxOutput(recoveredStdout, value)
+              },
+              retainStdout,
+              options.onStdout
+            )
+            return
+          }
+
+          recoveredStdoutPrefix += chunk
+          const marker = `${DAYTONA_STREAM_READY_MARKER}\n`
+          if (
+            marker.startsWith(recoveredStdoutPrefix) &&
+            recoveredStdoutPrefix.length < marker.length
+          ) {
+            return
+          }
+
+          recoveryMarkerPending = false
+          const markerWasPresent = recoveredStdoutPrefix.startsWith(marker)
+          const remaining = markerWasPresent
+            ? recoveredStdoutPrefix.slice(marker.length)
+            : recoveredStdoutPrefix
+          recoveredStdoutPrefix = ''
+          if (!releaseRequested && markerWasPresent) releaseCommand()
+          if (remaining) {
+            appendRecoveredStdout(remaining)
+          }
+        }
+        let recoveryStreamError: unknown
+        const recoveryStreamed = this.sandbox.process
+          .getSessionCommandLogs(
+            sessionId,
+            commandId,
+            (chunk: string) => {
+              appendRecoveredStdout(chunk)
+            },
+            (chunk: string) => {
+              const [suffix, remaining] = skipObservedReplay(chunk, stderrReplayRemaining)
+              stderrReplayRemaining = remaining
+              if (!suffix) return
+              appendRecoveredOutput(
+                suffix,
+                (value) => {
+                  recoveredStderr = retainStderr
+                    ? recoveredStderr + value
+                    : appendStreamedSandboxOutput(recoveredStderr, value)
+                },
+                retainStderr,
+                options.onStderr
+              )
+            }
+          )
+          .then(() => 'done' as const)
+          .catch((error: unknown) => {
+            recoveryStreamError = error
+            return 'error' as const
+          })
+        let recoveryTimer: ReturnType<typeof setTimeout> | undefined
+        const recoveryTimedOut = new Promise<'timeout'>((resolve) => {
+          recoveryTimer = setTimeout(
+            () => resolve('timeout'),
+            Math.max(0, streamDeadlineAt - Date.now())
+          )
+        })
+        let recoveryAbortListener: (() => void) | undefined
+        const recoveryAborted = new Promise<'aborted'>((resolve) => {
+          recoveryAbortListener = () => {
+            void this.kill().catch(() => {})
+            resolve('aborted')
+          }
+          options.signal?.addEventListener('abort', recoveryAbortListener, { once: true })
+          if (options.signal?.aborted) recoveryAbortListener()
+        })
+        let recoveryOutcome: 'done' | 'error' | 'timeout' | 'aborted'
+        try {
+          recoveryOutcome = await Promise.race([
+            recoveryStreamed,
+            recoveryTimedOut,
+            recoveryAborted,
+          ])
+        } finally {
+          if (recoveryTimer) clearTimeout(recoveryTimer)
+          if (recoveryAbortListener) {
+            options.signal?.removeEventListener('abort', recoveryAbortListener)
+          }
+        }
+        if (recoveryOutcome === 'aborted') {
+          throw options.signal?.reason instanceof Error
+            ? options.signal.reason
+            : new DOMException('Execution cancelled', 'AbortError')
+        }
+        if (recoveryOutcome === 'timeout') {
+          return {
+            stdout: finalStdout(),
+            stderr: finalStderr() || `Command timed out after ${options.timeoutMs}ms`,
+            exitCode: 124,
+            timedOut: true,
+          }
+        }
+        if (recoveryOutcome === 'error') {
+          if (recoveryOutputBudget.error) throw recoveryOutputBudget.error
+          throw new SandboxLaunchIndeterminateError('Daytona', { cause: recoveryStreamError })
+        }
+
+        const mergeRecoveredStream = (existing: string, recovered: string, retain: boolean) => {
+          if (!retain) {
+            return appendStreamedSandboxOutput(existing, recovered)
+          }
+          const recoveredIncludesExisting = recovered.startsWith(existing)
+          return recoveredIncludesExisting ? recovered : existing + recovered
+        }
+        stdout = mergeRecoveredStream(stdout, recoveredStdout, retainStdout)
+        stderr = mergeRecoveredStream(stderr, recoveredStderr, retainStderr)
+        assertSandboxProcessOutputWithinLimit([stdout, stderr], options.maxOutputBytes)
       }
 
       const finished = await this.sandbox.process.getSessionCommand(sessionId, commandId)
@@ -400,6 +660,7 @@ class DaytonaSandboxHandle implements SandboxHandle {
         void this.kill().catch(() => {})
         throw error
       }
+      if (isNonRetryableExecutionError(error)) throw error
       if (options.signal?.aborted) {
         throw options.signal.reason instanceof Error
           ? options.signal.reason

apps/sim/lib/execution/remote-sandbox/e2b.ts

@@ -18,6 +18,10 @@ import {
   readStreamToBufferWithLimit,
 } from '@/lib/core/utils/stream-limits'
 import { CodeLanguage } from '@/lib/execution/languages'
+import {
+  isNonRetryableExecutionError,
+  SandboxLaunchIndeterminateError,
+} from '@/lib/execution/non-retryable-error'
 import { classifyInstallOutput, tailBuildLog } from '@/lib/execution/remote-sandbox/build-errors'
 import {
   sandboxCliEnvironment,
@@ -31,6 +35,7 @@ import {
 } from '@/lib/execution/remote-sandbox/function-resources'
 import {
   assertSandboxProcessOutputWithinLimit,
+  isSandboxOutputLimitError,
   MAX_SANDBOX_OUTPUT_BYTES,
   MAX_SANDBOX_PROCESS_OUTPUT_BYTES,
   SandboxOutputFileError,
@@ -341,6 +346,7 @@ class E2BSandboxHandle implements SandboxHandle {
           timeoutMs: options.timeoutMs,
           maxOutputBytes: options.maxOutputBytes,
           signal: options.signal,
+          atMostOnce: true,
           envs: {
             ...options.envs,
             SIM_CODE_PATH: codePath,
@@ -400,33 +406,132 @@ class E2BSandboxHandle implements SandboxHandle {
      */
     const retainStdout = options.onStdout === undefined
     const retainStderr = options.onStderr === undefined
-    const guardOutput = (value: string, callback?: (chunk: string) => void) => {
+    let observedStdout = ''
+    let observedStderr = ''
+    const guardOutput = (
+      value: string,
+      append: (chunk: string) => void,
+      callback?: (chunk: string) => void
+    ) => {
       try {
         outputBudget.add(value)
       } catch (error) {
         void this.kill().catch(() => {})
         throw error
       }
+      append(value)
       callback?.(value)
     }
     try {
       const prepared = prepareE2BCommand(command, options.envs)
-      const result = await this.sandbox.commands.run(prepared.command, {
+      const processOptions = {
         ...(prepared.envs ? { envs: prepared.envs } : {}),
         timeoutMs: e2bTimeoutMs(options.timeoutMs),
         ...(options.signal ? { signal: options.signal } : {}),
         ...(options.rootUser ? { user: 'root' as const } : {}),
-        onStdout: (chunk) => guardOutput(chunk, options.onStdout),
-        onStderr: (chunk) => guardOutput(chunk, options.onStderr),
-      })
-      assertSandboxProcessOutputWithinLimit([result.stdout, result.stderr], options.maxOutputBytes)
+      }
+      let started: Awaited<ReturnType<E2BSandbox['commands']['run']>>
+      try {
+        started = await this.sandbox.commands.run(prepared.command, {
+          ...processOptions,
+          background: true,
+          onStdout: (chunk: string) =>
+            guardOutput(
+              chunk,
+              (value) => {
+                observedStdout += value
+              },
+              options.onStdout
+            ),
+          onStderr: (chunk: string) =>
+            guardOutput(
+              chunk,
+              (value) => {
+                observedStderr += value
+              },
+              options.onStderr
+            ),
+        })
+      } catch (error) {
+        if (options.signal?.aborted || isE2BExecutionTimeout(error)) throw error
+        const failure = error as { stdout?: string; stderr?: string; exitCode?: number }
+        if (
+          failure.stdout !== undefined ||
+          failure.stderr !== undefined ||
+          failure.exitCode !== undefined
+        ) {
+          throw error
+        }
+        if (options.atMostOnce) {
+          throw new SandboxLaunchIndeterminateError('E2B', { cause: error })
+        }
+        throw error
+      }
+
+      let result
+      if ('wait' in started && typeof started.wait === 'function') {
+        try {
+          result = await started.wait()
+        } catch (error) {
+          const failure = error as { stdout?: string; stderr?: string; exitCode?: number }
+          const isTerminalFailure =
+            failure.stdout !== undefined ||
+            failure.stderr !== undefined ||
+            failure.exitCode !== undefined ||
+            isE2BExecutionTimeout(error)
+          if (options.signal?.aborted || outputBudget.error || isTerminalFailure) throw error
+          if (options.atMostOnce) {
+            throw new SandboxLaunchIndeterminateError('E2B', { cause: error })
+          }
+
+          logger.warn('E2B command event stream disconnected; reconnecting to the same process', {
+            sandboxId: this.sandboxId,
+            pid: started.pid,
+          })
+          const recovered = await this.sandbox.commands.connect(started.pid, {
+            ...processOptions,
+            onStdout: (chunk: string) =>
+              guardOutput(
+                chunk,
+                (value) => {
+                  observedStdout += value
+                },
+                options.onStdout
+              ),
+            onStderr: (chunk: string) =>
+              guardOutput(
+                chunk,
+                (value) => {
+                  observedStderr += value
+                },
+                options.onStderr
+              ),
+          })
+          result = await recovered.wait()
+        }
+      } else {
+        // Older test doubles return the synchronous result shape. Production E2B always returns a
+        // process handle because `background: true` is fixed above.
+        result = started
+      }
+      const mergeObservedOutput = (observed: string, returned: string): string => {
+        if (!observed) return returned
+        if (!returned || observed.endsWith(returned)) return observed
+        if (returned.startsWith(observed)) return returned
+        return observed + returned
+      }
+      const stdout = mergeObservedOutput(observedStdout, result.stdout ?? '')
+      const stderr = mergeObservedOutput(observedStderr, result.stderr ?? '')
+      assertSandboxProcessOutputWithinLimit([stdout, stderr], options.maxOutputBytes)
       return {
-        stdout: retainStdout ? result.stdout : tailStreamedSandboxOutput(result.stdout),
-        stderr: retainStderr ? result.stderr : tailStreamedSandboxOutput(result.stderr),
-        exitCode: result.exitCode,
+        stdout: retainStdout ? stdout : tailStreamedSandboxOutput(stdout),
+        stderr: retainStderr ? stderr : tailStreamedSandboxOutput(stderr),
+        exitCode: result.exitCode ?? 1,
       }
     } catch (error) {
       if (outputBudget.error) throw outputBudget.error
+      if (isSandboxOutputLimitError(error)) throw error
+      if (isNonRetryableExecutionError(error)) throw error
       if (reachedE2BProviderLimit(error, this.providerLimitAtMs, options.signal)) {
         recordSandboxProviderLimit({ provider: 'e2b', operation })
         return { stdout: '', stderr: E2B_PROVIDER_LIMIT_ERROR, exitCode: 1 }

apps/sim/lib/execution/remote-sandbox/index.ts

@@ -715,6 +715,7 @@ async function executeShellInSandboxWithinBudget(
         maxOutputBytes: MAX_SANDBOX_PROCESS_OUTPUT_BYTES,
         signal,
         rootUser: true,
+        atMostOnce: true,
       })
     } catch (error) {
       throwIfAborted(signal)

apps/sim/lib/execution/remote-sandbox/resolve.test.ts

@@ -604,7 +604,11 @@ describe('provisionRuntimeDependencies', () => {
     ]
     expect(command).toContain('apt-get update')
     expect(command).toContain("'curl=7.88.1-10+deb12u8' 'jq'")
-    expect(options).toMatchObject({ rootUser: true, signal: controller.signal })
+    expect(options).toMatchObject({
+      rootUser: true,
+      signal: controller.signal,
+      atMostOnce: true,
+    })
     expect(options.timeoutMs).toBeGreaterThan(0)
     expect(options.timeoutMs).toBeLessThanOrEqual(90_000)
     expect(sandbox.writeFile).not.toHaveBeenCalled()

apps/sim/lib/execution/remote-sandbox/resolve.ts

@@ -573,6 +573,7 @@ export async function provisionRuntimeDependencies(
       maxOutputBytes: MAX_SANDBOX_PROCESS_OUTPUT_BYTES,
       signal,
       rootUser: true,
+      atMostOnce: true,
       ...(commandOptions?.envs ? { envs: commandOptions.envs } : {}),
     })
     if (signal?.aborted) {

apps/sim/lib/execution/remote-sandbox/types.ts

@@ -128,6 +128,8 @@ export interface RunCommandOptions {
   signal?: AbortSignal
   /** Run as root. The shell and Pi paths depend on this; the code path does not. */
   rootUser?: boolean
+  /** Never relaunch this command when provider acknowledgement is ambiguous. */
+  atMostOnce?: boolean
   onStdout?: (chunk: string) => void
   onStderr?: (chunk: string) => void
 }

apps/sim/lib/function-execution/application/authorization.ts

@@ -0,0 +1,21 @@
+import type { DelegatedPrincipal } from '@sim/auth/principal'
+import type { ActiveWorkspaceApplicationContext } from '@/lib/workspaces/application/workspace-context'
+
+export const FUNCTION_EXECUTION_DELEGATION_AUDIENCE = 'sim:function-executions'
+
+export interface FunctionExecutionAuthorizationContext extends ActiveWorkspaceApplicationContext {
+  executionId?: string
+}
+
+export const functionExecutionDelegationPolicy = {
+  audience: FUNCTION_EXECUTION_DELEGATION_AUDIENCE,
+  isWithinScope: (
+    principal: DelegatedPrincipal,
+    context: FunctionExecutionAuthorizationContext
+  ): boolean => {
+    const scopedExecutionId = principal.resourceScope?.executionId
+    return context.executionId
+      ? scopedExecutionId === context.executionId
+      : scopedExecutionId === undefined
+  },
+}

apps/sim/lib/function-execution/application/execute-function.ts

@@ -0,0 +1,58 @@
+import { requirePrincipalSubjectUserId } from '@sim/auth/principal'
+import { type NextRequest, NextRequest as ServerRequest } from 'next/server'
+import { type FunctionExecuteBody, functionExecuteBodySchema } from '@/lib/api/contracts'
+import { defineAuthorizedWorkspaceUseCase } from '@/lib/core/application'
+import { OrchestrationError } from '@/lib/core/orchestration/types'
+import { functionExecutionDelegationPolicy } from '@/lib/function-execution/application/authorization'
+import { functionExecutionOperations } from '@/lib/function-execution/application/operations'
+import { resolveActiveWorkspaceApplicationContext } from '@/lib/workspaces/application/workspace-context'
+
+export interface ExecuteFunctionInput {
+  workspaceId: string
+  body: FunctionExecuteBody
+  headers: Headers
+  signal?: AbortSignal
+  sandboxProfile?: 'mothership'
+}
+
+/**
+ * Runs the existing Function protocol in the authenticated owner process.
+ *
+ * Returning a Response deliberately preserves the compatibility adapter's status, headers, and
+ * private metadata while callers migrate off the HTTP transport. The Function protocol itself is
+ * still parsed by its shared boundary contract before any code executes.
+ */
+export const executeFunction = defineAuthorizedWorkspaceUseCase({
+  operation: functionExecutionOperations.execute,
+  resolveContext: async ({ input }: { input: ExecuteFunctionInput }) => {
+    if (input.body.workspaceId !== input.workspaceId) {
+      throw new OrchestrationError('not_found', 'Workspace not found')
+    }
+    return {
+      ...(await resolveActiveWorkspaceApplicationContext(input.workspaceId)),
+      ...(input.body.executionId ? { executionId: input.body.executionId } : {}),
+    }
+  },
+  authorizationOptions: {
+    delegation: functionExecutionDelegationPolicy,
+  },
+  execute: async ({ principal, input }): Promise<Response> => {
+    const parsedBody = functionExecuteBodySchema.safeParse(input.body)
+    if (!parsedBody.success) {
+      throw new OrchestrationError(
+        'validation',
+        parsedBody.error.issues[0]?.message ?? 'Function execution input is invalid'
+      )
+    }
+    const request = new ServerRequest('http://sim.internal/api/function/execute', {
+      method: 'POST',
+      headers: input.headers,
+      ...(input.signal ? { signal: input.signal } : {}),
+    }) as NextRequest
+    const { executeFunctionRequest } = await import('@/lib/function-execution/execute-request')
+    return executeFunctionRequest(request, parsedBody.data, {
+      userId: requirePrincipalSubjectUserId(principal),
+      ...(input.sandboxProfile ? { sandboxProfile: input.sandboxProfile } : {}),
+    })
+  },
+})

apps/sim/lib/function-execution/application/operations.ts

@@ -0,0 +1,11 @@
+import { defineWorkspaceOperation } from '@/lib/core/application'
+
+export const functionExecutionOperations = {
+  execute: defineWorkspaceOperation({
+    id: 'function-executions.execute',
+    minimumRole: 'read',
+    workspaceApiKey: 'deny',
+    principalKinds: ['delegated'],
+    delegatedServices: ['executor', 'copilot'],
+  }),
+} as const

apps/sim/lib/workflows/custom-blocks/child-execution.test.ts

@@ -4,18 +4,10 @@
 import { beforeEach, describe, expect, it, vi } from 'vitest'
 import { createTimeoutAbortController, getRemainingExecutionMs } from '@/lib/core/execution-limits'
 
-const {
-  mockCheckAttributedUsageLimits,
-  mockSubscribe,
-  mockUnsubscribe,
-  mockIsExecutionCancelled,
-  mockIsRedisCancellationEnabled,
-} = vi.hoisted(() => ({
+const { mockCheckAttributedUsageLimits, mockSubscribe, mockUnsubscribe } = vi.hoisted(() => ({
   mockCheckAttributedUsageLimits: vi.fn(),
   mockSubscribe: vi.fn(),
   mockUnsubscribe: vi.fn(),
-  mockIsExecutionCancelled: vi.fn(),
-  mockIsRedisCancellationEnabled: vi.fn(),
 }))
 
 vi.mock('@/lib/billing/core/billing-attribution', () => ({
@@ -23,9 +15,7 @@ vi.mock('@/lib/billing/core/billing-attribution', () => ({
 }))
 
 vi.mock('@/lib/execution/cancellation', () => ({
-  getCancellationChannel: () => ({ subscribe: mockSubscribe }),
-  isExecutionCancelled: mockIsExecutionCancelled,
-  isRedisCancellationEnabled: mockIsRedisCancellationEnabled,
+  subscribeToExecutionCancellation: mockSubscribe,
 }))
 
 import {
@@ -43,9 +33,7 @@ const attribution = { actorUserId: 'owner-1', workspaceId: 'workspace-source' }
 describe('admitCustomBlockChildExecution', () => {
   beforeEach(() => {
     vi.clearAllMocks()
-    mockSubscribe.mockReturnValue(mockUnsubscribe)
-    mockIsRedisCancellationEnabled.mockReturnValue(true)
-    mockIsExecutionCancelled.mockResolvedValue(false)
+    mockSubscribe.mockResolvedValue(mockUnsubscribe)
   })
 
   it('passes when the source payer has headroom', async () => {
@@ -134,9 +122,7 @@ describe('buildCustomBlockCorrelation', () => {
 describe('createChildCancellationSignal', () => {
   beforeEach(() => {
     vi.clearAllMocks()
-    mockSubscribe.mockReturnValue(mockUnsubscribe)
-    mockIsRedisCancellationEnabled.mockReturnValue(true)
-    mockIsExecutionCancelled.mockResolvedValue(false)
+    mockSubscribe.mockResolvedValue(mockUnsubscribe)
   })
 
   it('aborts when the parent signal aborts', async () => {
@@ -174,15 +160,13 @@ describe('createChildCancellationSignal', () => {
     expect(signal.aborted).toBe(true)
   })
 
-  it('aborts on the parent cancellation event, and ignores other runs', async () => {
+  it('aborts on the bound parent cancellation signal', async () => {
     const { signal } = await createChildCancellationSignal({ parentExecutionId: 'parent-1' })
-    const handler = mockSubscribe.mock.calls[0][0]
+    const handler = mockSubscribe.mock.calls[0][1]
 
-    handler({ executionId: 'someone-else' })
-    expect(signal.aborted).toBe(false)
-
-    handler({ executionId: 'parent-1' })
+    handler()
     expect(signal.aborted).toBe(true)
+    expect(mockSubscribe).toHaveBeenCalledWith('parent-1', handler)
   })
 
   it('unsubscribes on dispose so a looped block leaks nothing', async () => {
@@ -211,52 +195,15 @@ describe('CustomBlockAdmissionError', () => {
 describe('createChildCancellationSignal durable backstop', () => {
   beforeEach(() => {
     vi.clearAllMocks()
-    mockSubscribe.mockReturnValue(mockUnsubscribe)
-    mockIsRedisCancellationEnabled.mockReturnValue(true)
-    mockIsExecutionCancelled.mockResolvedValue(false)
+    mockSubscribe.mockResolvedValue(mockUnsubscribe)
   })
 
-  it('aborts on a cancel published before the bridge subscribed', async () => {
-    // The pub/sub event is long gone; only the durable key remains.
-    mockIsExecutionCancelled.mockResolvedValue(true)
+  it('fails closed when the durable cancellation subscription cannot be established', async () => {
+    mockSubscribe.mockRejectedValue(new Error('redis down'))
 
-    const { signal } = await createChildCancellationSignal({ parentExecutionId: 'parent-1' })
-
-    expect(mockIsExecutionCancelled).toHaveBeenCalledWith('parent-1')
-    expect(signal.aborted).toBe(true)
-  })
-
-  it('subscribes before reading the durable key so no window is left open', async () => {
-    const order: string[] = []
-    mockSubscribe.mockImplementation(() => {
-      order.push('subscribe')
-      return mockUnsubscribe
-    })
-    mockIsExecutionCancelled.mockImplementation(async () => {
-      order.push('durable-read')
-      return false
-    })
-
-    await createChildCancellationSignal({ parentExecutionId: 'parent-1' })
-
-    expect(order).toEqual(['subscribe', 'durable-read'])
-  })
-
-  it('fails open when the durable read throws', async () => {
-    mockIsExecutionCancelled.mockRejectedValue(new Error('redis down'))
-
-    const { signal } = await createChildCancellationSignal({ parentExecutionId: 'parent-1' })
-
-    expect(signal.aborted).toBe(false)
-  })
-
-  it('skips the durable read when redis cancellation is unavailable', async () => {
-    mockIsRedisCancellationEnabled.mockReturnValue(false)
-
-    const { signal } = await createChildCancellationSignal({ parentExecutionId: 'parent-1' })
-
-    expect(mockIsExecutionCancelled).not.toHaveBeenCalled()
-    expect(signal.aborted).toBe(false)
+    await expect(createChildCancellationSignal({ parentExecutionId: 'parent-1' })).rejects.toThrow(
+      'redis down'
+    )
   })
 })
 

apps/sim/lib/workflows/custom-blocks/child-execution.ts

@@ -5,11 +5,7 @@ import {
 } from '@/lib/billing/core/billing-attribution'
 import type { AsyncExecutionCorrelation } from '@/lib/core/async-jobs/types'
 import { combineExecutionAbortSignals } from '@/lib/core/execution-limits'
-import {
-  getCancellationChannel,
-  isExecutionCancelled,
-  isRedisCancellationEnabled,
-} from '@/lib/execution/cancellation'
+import { subscribeToExecutionCancellation } from '@/lib/execution/cancellation'
 import { BoundarySafeError } from '@/executor/errors/boundary'
 
 const logger = createLogger('CustomBlockChildExecution')
@@ -119,27 +115,7 @@ export async function createChildCancellationSignal(params: {
 
   let unsubscribe: (() => void) | undefined
   if (params.parentExecutionId) {
-    const parentExecutionId = params.parentExecutionId
-    // Subscribe BEFORE the durable read, mirroring `markExecutionCancelled`'s
-    // write-durable-then-publish order: a cancel published during the read is
-    // caught by the subscription, and one published earlier — while the child's
-    // session and admission were still being set up — by the read itself. The
-    // child's own engine backstop cannot cover this, since it checks the CHILD's
-    // execution id, which is never the one marked cancelled. For the same reason
-    // the child's steady-state coverage is transitive: a cancel published after
-    // setup reaches the child only via this subscription, or via the PARENT
-    // engine's durable poll aborting `parentSignal`.
-    unsubscribe = getCancellationChannel().subscribe((event) => {
-      if (event.executionId === parentExecutionId) abort()
-    })
-    if (isRedisCancellationEnabled()) {
-      try {
-        if (await isExecutionCancelled(parentExecutionId)) abort()
-      } catch {
-        // Fail open, matching the engine's own backstop: a failed read must not
-        // stop a child whose parent was never actually cancelled.
-      }
-    }
+    unsubscribe = await subscribeToExecutionCancellation(params.parentExecutionId, abort)
   }
 
   return {

apps/sim/lib/workflows/executor/execution-events.ts

@@ -326,6 +326,25 @@ export type StreamDoneData = StreamDoneEvent['data']
 export type StreamThinkingData = StreamThinkingEvent['data']
 export type StreamToolData = StreamToolEvent['data']
 
+export function getBlockInvocationKey(
+  data: Pick<
+    BlockStartedData,
+    | 'blockId'
+    | 'executionOrder'
+    | 'iterationContainerId'
+    | 'iterationCurrent'
+    | 'childWorkflowBlockId'
+  >
+): string {
+  return [
+    data.blockId,
+    data.executionOrder,
+    data.iterationContainerId ?? '',
+    data.iterationCurrent ?? '',
+    data.childWorkflowBlockId ?? '',
+  ].join(':')
+}
+
 /**
  * Helper to create SSE formatted message
  */

apps/sim/lib/workflows/executor/human-in-the-loop-manager.ts

@@ -19,8 +19,8 @@ import {
   createExecutionEventWriter,
   flushExecutionStreamReplayBuffer,
   initializeExecutionStreamMeta,
+  markExecutionStreamTerminal,
   resetExecutionStreamBuffer,
-  setExecutionMeta,
   type TerminalExecutionStreamStatus,
 } from '@/lib/execution/event-buffer'
 import {
@@ -1528,9 +1528,10 @@ export class PauseResumeManager {
           status: terminalStatus,
         })
       )
-      const metaPersisted = await setExecutionMeta(resumeExecutionId, {
-        status: terminalStatus,
-      }).catch(() => false)
+      const metaPersisted = await markExecutionStreamTerminal(
+        resumeExecutionId,
+        terminalStatus
+      ).catch(() => false)
       if (!metaPersisted) {
         logger.warn('Failed to record degraded terminal status on resume stream meta', {
           resumeExecutionId,

apps/sim/tools/function/execute.ts

@@ -1,3 +1,4 @@
+import type { FunctionExecuteBody } from '@/lib/api/contracts'
 import {
   normalizeRecord,
   normalizeRecordMap,
@@ -14,6 +15,52 @@ import {
 import type { CodeExecutionInput, CodeExecutionOutput } from '@/tools/function/types'
 import type { ToolConfig } from '@/tools/types'
 
+/** Builds the canonical Function protocol body for both HTTP compatibility and in-process calls. */
+export function buildFunctionExecuteBody(params: CodeExecutionInput): FunctionExecuteBody {
+  const codeContent = Array.isArray(params.code)
+    ? params.code.map((entry: { content: string }) => entry.content).join('\n')
+    : params.code
+
+  return {
+    code: codeContent,
+    sourceCode: params.sourceCode,
+    language: params.language || DEFAULT_CODE_LANGUAGE,
+    timeout: params.timeout || DEFAULT_EXECUTION_TIMEOUT_MS,
+    title: params.title,
+    outputPath: params.outputPath,
+    outputFormat: params.outputFormat,
+    outputTable: params.outputTable,
+    outputSandboxPath: params.outputSandboxPath,
+    outputMimeType: params.outputMimeType,
+    sandboxId: params.sandboxId,
+    secretScope: params.secretScope,
+    mountedSecrets: params.mountedSecrets,
+    unredactedSecretNames: params.unredactedSecretNames,
+    overwriteFileId: params.overwriteFileId,
+    inputs: params.inputs,
+    outputs: params.outputs,
+    envVars: normalizeStringRecord(params.envVars),
+    workflowVariables: normalizeWorkflowVariables(params.workflowVariables),
+    blockData: normalizeRecord(params.blockData),
+    blockNameMapping: normalizeStringRecord(params.blockNameMapping),
+    blockOutputSchemas: normalizeRecordMap(params.blockOutputSchemas),
+    contextVariables: normalizeRecord(params.contextVariables),
+    workflowId: params._context?.workflowId,
+    executionId: params._context?.executionId,
+    largeValueExecutionIds: params._context?.largeValueExecutionIds,
+    largeValueKeys: params._context?.largeValueKeys,
+    fileKeys: params._context?.fileKeys,
+    allowLargeValueWorkflowScope: params._context?.allowLargeValueWorkflowScope,
+    userId: params._context?.userId,
+    workspaceId: params._context?.workspaceId,
+    isCustomTool: params.isCustomTool || false,
+    ...(params[PRIVATE_SECRET_PROVENANCE_FIELD]
+      ? { [PRIVATE_SECRET_PROVENANCE_FIELD]: params[PRIVATE_SECRET_PROVENANCE_FIELD] }
+      : {}),
+    ...(params._sandboxFiles ? { _sandboxFiles: params._sandboxFiles } : {}),
+  }
+}
+
 export const functionExecuteTool: ToolConfig<CodeExecutionInput, CodeExecutionOutput> = {
   id: 'function_execute',
   name: 'Function Execute',
@@ -154,57 +201,7 @@ export const functionExecuteTool: ToolConfig<CodeExecutionInput, CodeExecutionOu
         ? { [PRIVATE_SECRET_PROVENANCE_HEADER]: PRIVATE_SECRET_PROVENANCE_BUNDLE_V1 }
         : {}),
     }),
-    body: (params: CodeExecutionInput) => {
-      const codeContent = Array.isArray(params.code)
-        ? params.code.map((c: { content: string }) => c.content).join('\n')
-        : params.code
-
-      const body: Record<string, unknown> = {
-        code: codeContent,
-        sourceCode: params.sourceCode,
-        language: params.language || DEFAULT_CODE_LANGUAGE,
-        timeout: params.timeout || DEFAULT_EXECUTION_TIMEOUT_MS,
-        title: params.title,
-        outputPath: params.outputPath,
-        outputFormat: params.outputFormat,
-        outputTable: params.outputTable,
-        outputSandboxPath: params.outputSandboxPath,
-        outputMimeType: params.outputMimeType,
-        sandboxId: params.sandboxId,
-        secretScope: params.secretScope,
-        mountedSecrets: params.mountedSecrets,
-        unredactedSecretNames: params.unredactedSecretNames,
-        overwriteFileId: params.overwriteFileId,
-        inputs: params.inputs,
-        outputs: params.outputs,
-        envVars: normalizeStringRecord(params.envVars),
-        workflowVariables: normalizeWorkflowVariables(params.workflowVariables),
-        blockData: normalizeRecord(params.blockData),
-        blockNameMapping: normalizeStringRecord(params.blockNameMapping),
-        blockOutputSchemas: normalizeRecordMap(params.blockOutputSchemas),
-        contextVariables: normalizeRecord(params.contextVariables),
-        workflowId: params._context?.workflowId,
-        executionId: params._context?.executionId,
-        largeValueExecutionIds: params._context?.largeValueExecutionIds,
-        largeValueKeys: params._context?.largeValueKeys,
-        fileKeys: params._context?.fileKeys,
-        allowLargeValueWorkflowScope: params._context?.allowLargeValueWorkflowScope,
-        userId: params._context?.userId,
-        workspaceId: params._context?.workspaceId,
-        isCustomTool: params.isCustomTool || false,
-        ...(params[PRIVATE_SECRET_PROVENANCE_FIELD]
-          ? {
-              [PRIVATE_SECRET_PROVENANCE_FIELD]: params[PRIVATE_SECRET_PROVENANCE_FIELD],
-            }
-          : {}),
-      }
-
-      if (params._sandboxFiles) {
-        body._sandboxFiles = params._sandboxFiles
-      }
-
-      return body
-    },
+    body: buildFunctionExecuteBody,
   },
 
   transformResponse: async (response: Response): Promise<CodeExecutionOutput> => {
@@ -218,6 +215,7 @@ export const functionExecuteTool: ToolConfig<CodeExecutionInput, CodeExecutionOu
           stdout: result.output?.stdout || '',
         },
         error: result.error,
+        retryable: result.retryable,
         resources: result.resources,
         largeValueKeys: result.largeValueKeys,
         fileKeys: result.fileKeys,
@@ -233,6 +231,7 @@ export const functionExecuteTool: ToolConfig<CodeExecutionInput, CodeExecutionOu
       resources: result.resources,
       largeValueKeys: result.largeValueKeys,
       fileKeys: result.fileKeys,
+      retryable: result.retryable,
     }
   },
 

apps/sim/tools/index.test.ts

@@ -38,6 +38,7 @@ import {
 import { bitbucketGetPipelineStepLogTool } from '@/tools/bitbucket/get_pipeline_step_log'
 import { ErrorExtractorId } from '@/tools/error-extractors'
 import { fileGetContentTool } from '@/tools/file/get'
+import { buildFunctionExecuteBody } from '@/tools/function/execute'
 import { memoryAddTool } from '@/tools/memory/add'
 import { tableBatchInsertRowsTool } from '@/tools/table/batch_insert_rows'
 import { customBlockExecutorTool } from '@/tools/workflow/custom-block-executor'
@@ -58,6 +59,7 @@ const {
   mockGenerateInternalToken,
   mockResolveWorkspaceFileReference,
   mockAssertPermissionsAllowed,
+  mockExecuteFunction,
 } = vi.hoisted(() => ({
   mockGetBYOKKey: vi.fn(),
   mockGetToolAsync: vi.fn(),
@@ -76,6 +78,7 @@ const {
   mockGenerateInternalToken: vi.fn(),
   mockResolveWorkspaceFileReference: vi.fn(),
   mockAssertPermissionsAllowed: vi.fn(),
+  mockExecuteFunction: vi.fn(),
 }))
 
 const mockSecureFetchWithPinnedIP = inputValidationMockFns.mockSecureFetchWithPinnedIP
@@ -121,6 +124,10 @@ vi.mock('@/lib/billing/core/usage-log', () => ({}))
 
 vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock)
 
+vi.mock('@/lib/function-execution/application/execute-function', () => ({
+  executeFunction: { execute: mockExecuteFunction },
+}))
+
 vi.mock('@/lib/core/rate-limiter/hosted-key', () => ({
   getHostedKeyRateLimiter: () => mockRateLimiterFns,
 }))
@@ -219,11 +226,7 @@ const mockRegistryTools: Record<string, any> = {
       url: '/api/function/execute',
       method: 'POST',
       headers: () => ({ 'Content-Type': 'application/json' }),
-      body: (p: any) => ({
-        code: Array.isArray(p.code) ? p.code.map((c: any) => c.content).join('\n') : p.code,
-        language: p.language || 'javascript',
-        timeout: p.timeout || 30000,
-      }),
+      body: buildFunctionExecuteBody,
     },
     transformResponse: async (response: any) => {
       const data = await response.json()
@@ -477,6 +480,12 @@ beforeEach(() => {
   mockAssertPermissionsAllowed.mockResolvedValue(undefined)
   mockGenerateInternalDelegationToken.mockResolvedValue('executor-token')
   mockRunWorkflowTool.mockResolvedValue({ success: true, output: {} })
+  mockExecuteFunction.mockResolvedValue(
+    new Response(JSON.stringify({ success: true, output: { result: 'in-process' } }), {
+      status: 200,
+      headers: { 'content-type': 'application/json' },
+    })
+  )
   // Suites below call vi.resetAllMocks(), which wipes the shared env/urls mock
   // implementations — restore their defaults and re-pin the base URL each test.
   resetEnvMock()
@@ -707,6 +716,68 @@ describe('executeTool Function', () => {
     tools.function_execute = originalFunctionTool
   })
 
+  it('executes trusted Function calls in process without dropping resolved execution context', async () => {
+    const fetchSpy = vi.fn()
+    global.fetch = Object.assign(fetchSpy, { preconnect: vi.fn() }) as typeof fetch
+
+    const largeValueRef = {
+      __simLargeValueRef: true,
+      version: 1,
+      id: 'lv_ABCDEFGHIJKL',
+      kind: 'array',
+      size: 1024,
+      executionId: 'execution-1',
+    }
+    const result = await executeTool('function_execute', {
+      code: 'return [{{API_KEY}}, __blockRef_0.field]',
+      isCustomTool: true,
+      inputs: { location: 'San Francisco' },
+      envVars: { API_KEY: 'resolved-secret' },
+      contextVariables: {
+        __blockRef_0: { field: 'resolved-output' },
+        __blockRef_1: largeValueRef,
+      },
+      _context: {
+        userId: 'user-1',
+        workspaceId: 'workspace-456',
+        workflowId: 'workflow-1',
+        executionId: 'execution-1',
+        largeValueExecutionIds: ['execution-1'],
+        largeValueKeys: ['lv_ABCDEFGHIJKL'],
+        fileKeys: ['file-1'],
+        allowLargeValueWorkflowScope: true,
+      },
+    })
+
+    expect(result.success).toBe(true)
+    expect(mockExecuteFunction).toHaveBeenCalledOnce()
+    expect(mockExecuteFunction.mock.calls[0]?.[0]).toMatchObject({
+      input: {
+        workspaceId: 'workspace-456',
+        body: {
+          code: 'return [{{API_KEY}}, __blockRef_0.field]',
+          isCustomTool: true,
+          inputs: { location: 'San Francisco' },
+          envVars: { API_KEY: 'resolved-secret' },
+          contextVariables: {
+            __blockRef_0: { field: 'resolved-output' },
+            __blockRef_1: largeValueRef,
+          },
+          workflowId: 'workflow-1',
+          executionId: 'execution-1',
+          workspaceId: 'workspace-456',
+          userId: 'user-1',
+          largeValueExecutionIds: ['execution-1'],
+          largeValueKeys: ['lv_ABCDEFGHIJKL'],
+          fileKeys: ['file-1'],
+          allowLargeValueWorkflowScope: true,
+        },
+      },
+    })
+    expect(mockGenerateInternalToken).not.toHaveBeenCalled()
+    expect(fetchSpy).not.toHaveBeenCalled()
+  })
+
   it('bounds ignored Bitbucket pipeline log ranges through the execution path', async () => {
     mockValidateUrlWithDNS.mockResolvedValue({ isValid: true, resolvedIP: '93.184.216.34' })
 
@@ -1517,6 +1588,8 @@ describe('executeTool Function', () => {
     )
 
     await expect(execution).resolves.toMatchObject({ success: true })
+    expect(mockExecuteFunction).not.toHaveBeenCalled()
+    expect(global.fetch).toHaveBeenCalledTimes(1)
     expect(registry.isComplete()).toBe(true)
     expect(
       projectToolResultForCopilot({ success: true, output: { result: secret } }, registry)

apps/sim/tools/index.ts

@@ -1,9 +1,11 @@
+import type { DelegatedPrincipal } from '@sim/auth/principal'
 import { createLogger } from '@sim/logger'
 import { describeError, findCause, getErrorMessage, toError } from '@sim/utils/errors'
 import { sleep } from '@sim/utils/helpers'
 import { isPlainRecord, isRecordLike } from '@sim/utils/object'
 import { backoffWithJitter, parseRetryAfter } from '@sim/utils/retry'
 import { DrizzleQueryError } from 'drizzle-orm/errors'
+import type { FunctionExecuteBody } from '@/lib/api/contracts'
 import { MANAGED_OAUTH_DELEGATION_HEADER } from '@/lib/api/contracts/oauth-connections'
 import { getBYOKKey } from '@/lib/api-key/byok'
 import {
@@ -55,6 +57,7 @@ import {
   RESOLVED_SECRET_PROVENANCE_FIELD,
   RESOLVED_SECRET_PROVENANCE_METADATA_V1,
 } from '@/lib/execution/private-tool-metadata'
+import { FUNCTION_EXECUTION_DELEGATION_AUDIENCE } from '@/lib/function-execution/application/authorization'
 import { parseMcpToolId } from '@/lib/mcp/utils'
 import { hostedKeyMetrics } from '@/lib/monitoring/metrics'
 import type { CredentialTokenPayload } from '@/lib/oauth/token-resolution'
@@ -2276,10 +2279,14 @@ async function executeToolImplementation(
     const endTime = new Date()
     const endTimeISO = endTime.toISOString()
     const duration = endTime.getTime() - startTime.getTime()
+    const rawResponseData =
+      error instanceof Error && 'data' in error ? (error as { data?: unknown }).data : undefined
+    const responseData = isRecordLike(rawResponseData) ? rawResponseData : undefined
     return {
       success: false,
       output: errorDetails,
       error: errorMessage,
+      ...(responseData?.retryable === false ? { retryable: false } : {}),
       // Sim's own status (hosted-key 429/503) survives the flattening from a
       // thrown error into a result object; an upstream provider's status stays
       // on `output` where it cannot be mistaken for ours.
@@ -2470,7 +2477,16 @@ async function executeToolRequest(
     const requestParams = prepareToolRequest(tool, params, resolvedSecretTraceRegistry)
     const endpointUrl = requestParams.url
     const { headers, isInternalRoute } = requestParams
-    const baseUrl = isInternalRoute ? getInternalApiBaseUrl() : getBaseUrl()
+    const runFunctionInProcess =
+      isInternalRoute &&
+      normalizeToolId(toolId) === 'function_execute' &&
+      typeof params._context?.userId === 'string' &&
+      typeof params._context?.workspaceId === 'string'
+    const baseUrl = runFunctionInProcess
+      ? 'http://sim.internal'
+      : isInternalRoute
+        ? getInternalApiBaseUrl()
+        : getBaseUrl()
 
     const fullUrlObj = new URL(endpointUrl, baseUrl)
 
@@ -2517,15 +2533,17 @@ async function executeToolRequest(
         }
       }
     }
-    await addInternalAuthIfNeeded(
-      headers,
-      isInternalRoute,
-      requestId,
-      toolId,
-      params._context?.userId,
-      internalSandboxProfile ? { sandboxProfile: internalSandboxProfile } : undefined,
-      internalExecutorDelegation
-    )
+    if (!runFunctionInProcess) {
+      await addInternalAuthIfNeeded(
+        headers,
+        isInternalRoute,
+        requestId,
+        toolId,
+        params._context?.userId,
+        internalSandboxProfile ? { sandboxProfile: internalSandboxProfile } : undefined,
+        internalExecutorDelegation
+      )
+    }
     if (isInternalRoute && params._context?.billingAttribution) {
       headers.set(
         BILLING_ATTRIBUTION_HEADER,
@@ -2559,10 +2577,51 @@ async function executeToolRequest(
       headersRecord[key] = value
     })
 
-    const retryConfig = getRetryConfig(tool.request.retry, params, requestParams.method)
-    const maxAttempts = retryConfig ? 1 + retryConfig.maxRetries : 1
+    const retryConfig = runFunctionInProcess
+      ? null
+      : getRetryConfig(tool.request.retry, params, requestParams.method)
+    const maxAttempts = runFunctionInProcess ? 0 : retryConfig ? 1 + retryConfig.maxRetries : 1
 
     let response: Response | undefined
+    if (runFunctionInProcess) {
+      if (!requestParams.body) {
+        throw new Error('Function execution requires a request body')
+      }
+      const body = JSON.parse(requestParams.body) as FunctionExecuteBody
+      const issuedAt = new Date()
+      const serializedDeadline = serializeExecutionDeadlineHeader(signal)
+      const requestedTimeout =
+        typeof body.timeout === 'number' ? body.timeout : DEFAULT_EXECUTION_TIMEOUT_MS
+      const expiresAt = serializedDeadline
+        ? new Date(Number(serializedDeadline))
+        : new Date(issuedAt.getTime() + requestedTimeout)
+      const serviceId = params._context?.copilotToolExecution === true ? 'copilot' : 'executor'
+      const principal: DelegatedPrincipal = {
+        kind: 'delegated',
+        serviceId,
+        subjectUserId: params._context.userId,
+        workspaceId: params._context.workspaceId,
+        delegationId: `function-execute:${requestId}`,
+        audience: FUNCTION_EXECUTION_DELEGATION_AUDIENCE,
+        issuedAt,
+        expiresAt,
+        ...(body.executionId ? { resourceScope: { executionId: body.executionId } } : {}),
+      }
+      const { executeFunction } = await import(
+        '@/lib/function-execution/application/execute-function'
+      )
+      response = await executeFunction.execute({
+        principal,
+        input: {
+          workspaceId: params._context.workspaceId,
+          body,
+          headers,
+          ...(signal ? { signal } : {}),
+          ...(internalSandboxProfile ? { sandboxProfile: internalSandboxProfile } : {}),
+        },
+      })
+    }
+
     let lastError: unknown
     const nullBodyStatuses = new Set([101, 204, 205, 304])
 

apps/sim/tools/types.ts

@@ -96,6 +96,8 @@ export interface ToolResponse {
   success: boolean // Whether the tool execution was successful
   output: Record<string, any> // The structured output from the tool
   error?: string // Error message if success is false
+  /** False when replaying the operation could duplicate external side effects. */
+  retryable?: boolean
   /**
    * HTTP status owned by SIM itself (e.g. hosted-key rate limiting or
    * exhaustion), carried so it survives the throw → `ToolResponse` flattening

apps/sim/trigger.config.ts

@@ -26,6 +26,25 @@ if (grafanaConfigured && !grafanaFullyConfigured) {
   )
 }
 
+const FUNCTION_EXECUTION_ENV = [
+  { name: 'REDIS_URL', secret: true },
+  { name: 'REDIS_TLS_SERVERNAME', secret: false },
+  { name: 'SANDBOX_PROVIDER', secret: false },
+  { name: 'E2B_ENABLED', secret: false },
+  { name: 'E2B_API_KEY', secret: true },
+  { name: 'E2B_FUNCTION_TEMPLATE_ID', secret: false },
+  { name: 'E2B_FUNCTION_TEMPLATE_GENERATION', secret: false },
+  { name: 'DAYTONA_API_KEY', secret: true },
+  { name: 'DAYTONA_FUNCTION_SNAPSHOT_ID', secret: false },
+] as const
+
+function getFunctionExecutionEnvVars() {
+  return FUNCTION_EXECUTION_ENV.flatMap(({ name, secret }) => {
+    const value = env[name]
+    return value ? [{ name, value, isSecret: secret }] : []
+  })
+}
+
 const grafanaTelemetry = grafanaFullyConfigured
   ? (() => {
       const baseUrl = grafanaEndpoint!.replace(/\/+$/, '')
@@ -94,6 +113,7 @@ export default defineConfig({
          * because the `init` hook above marks the run process directly.
          */
         { name: 'TRIGGER_DEV_ENABLED', value: 'TRUE' },
+        ...getFunctionExecutionEnvVars(),
       ]),
       additionalFiles({
         files: [