improvement(executor): subflows, hitl handling cleanup (#4604)

* improvement(subflows): orchestration consolidation

* address comments

* fix hitl cases

* address comments

* subflow results output extraction

* hitl fallback case

* more cleanup

* add test

* fix type issue

* add test case for hitl resume

* address comments

* fix test

* fix snapshot for nested subflows
This commit is contained in:
Vikhyath Mondreti
2026-05-26 14:03:02 -07:00
committed by GitHub
parent bc99c457b9
commit e07b1ffbbe
54 changed files with 5126 additions and 833 deletions
@@ -108,26 +108,24 @@ export function getLeftmostBlockId(workflowState: WorkflowState | null | undefin
/** Execution status for edges/nodes in the preview */
type ExecutionStatus = 'success' | 'error' | 'not-executed'
/** Calculates absolute position, handling nested subflows. */
function calculateAbsolutePosition(
/** Calculates nesting depth, handling nested subflows. */
function calculateNestingDepth(
block: BlockState,
blocks: Record<string, BlockState>
): { x: number; y: number } {
if (!block.data?.parentId) {
return block.position
}
blocks: Record<string, BlockState>,
visited: Set<string> = new Set()
): number {
const parentId = block.data?.parentId
if (!parentId) return 0
if (visited.has(parentId)) return 0
const parentBlock = blocks[block.data.parentId]
const parentBlock = blocks[parentId]
if (!parentBlock) {
logger.warn(`Parent block not found for child block`)
return block.position
logger.warn('Parent block not found for child block')
return 0
}
const parentAbsolutePosition = calculateAbsolutePosition(parentBlock, blocks)
return {
x: parentAbsolutePosition.x + block.position.x,
y: parentAbsolutePosition.y + block.position.y,
}
visited.add(parentId)
return 1 + calculateNestingDepth(parentBlock, blocks, visited)
}
interface PreviewWorkflowProps {
@@ -329,7 +327,8 @@ export function PreviewWorkflow({
if (childStatuses.length === 0) return undefined
if (childStatuses.some((s) => s === 'error')) return 'error'
return 'success'
if (childStatuses.every((s) => s === 'success')) return 'success'
return undefined
}
return derive
}, [subflowChildrenMap, blockExecutionMap, workflowState.blocks])
@@ -367,13 +366,20 @@ export function PreviewWorkflow({
const nodeArray: Node[] = []
Object.entries(workflowState.blocks || {}).forEach(([blockId, block]) => {
const sortedBlocks = Object.entries(workflowState.blocks || {}).sort(
([, left], [, right]) =>
calculateNestingDepth(left, workflowState.blocks) -
calculateNestingDepth(right, workflowState.blocks)
)
sortedBlocks.forEach(([blockId, block]) => {
if (!block || !block.type) {
logger.warn(`Skipping invalid block: ${blockId}`)
return
}
const absolutePosition = calculateAbsolutePosition(block, workflowState.blocks)
const parentId = block.data?.parentId
const nestingDepth = calculateNestingDepth(block, workflowState.blocks)
if (block.type === 'loop' || block.type === 'parallel') {
const isSelected = selectedBlockId === blockId
@@ -391,9 +397,14 @@ export function PreviewWorkflow({
nodeArray.push({
id: blockId,
type: 'subflowNode',
position: absolutePosition,
position: block.position,
parentId,
extent: block.data?.extent || undefined,
draggable: false,
zIndex: nestingDepth,
className: parentId ? 'nested-subflow-node' : undefined,
data: {
...block.data,
name: block.name,
width: dimensions.width,
height: dimensions.height,
@@ -430,9 +441,11 @@ export function PreviewWorkflow({
nodeArray.push({
id: blockId,
type: nodeType,
position: absolutePosition,
position: block.position,
parentId,
extent: block.data?.extent || undefined,
draggable: false,
zIndex: block.data?.parentId ? 10 : undefined,
zIndex: parentId ? 1000 : undefined,
data: {
type: block.type,
name: block.name,
@@ -34,6 +34,15 @@ interface BlockExecutionData {
childWorkflowSnapshotId?: string
}
function isPauseOutput(output: unknown): boolean {
return (
output !== null &&
typeof output === 'object' &&
'_pauseMetadata' in output &&
(output as Record<string, unknown>)._pauseMetadata !== undefined
)
}
/** Represents a level in the workflow navigation stack */
interface WorkflowStackEntry {
workflowState: WorkflowState
@@ -91,7 +100,7 @@ export function buildBlockExecutions(spans: TraceSpan[]): Record<string, BlockEx
blockExecutionMap[span.blockId] = {
input: redactApiKeys(span.input || {}),
output: redactApiKeys(span.output || {}),
status: span.status || 'unknown',
status: isPauseOutput(span.output) ? 'pending' : span.status || 'unknown',
durationMs: span.duration || 0,
children: span.children,
childWorkflowSnapshotId: span.childWorkflowSnapshotId,
+14
View File
@@ -74,6 +74,20 @@ export const EDGE = {
DEFAULT: 'default',
} as const
export const SUBFLOW_CONTROL_EDGE_HANDLES = new Set<string>([
EDGE.LOOP_CONTINUE,
EDGE.LOOP_CONTINUE_ALT,
EDGE.LOOP_EXIT,
EDGE.PARALLEL_CONTINUE,
EDGE.PARALLEL_EXIT,
])
export const CONTROL_BACK_EDGE_HANDLES = new Set<string>([
EDGE.LOOP_CONTINUE,
EDGE.LOOP_CONTINUE_ALT,
EDGE.PARALLEL_CONTINUE,
])
export const LOOP = {
TYPE: {
FOR: 'for' as LoopType,
+170 -27
View File
@@ -1,5 +1,6 @@
import { beforeEach, describe, expect, it } from 'vitest'
import type { DAG, DAGNode } from '@/executor/dag/builder'
import { buildBranchNodeId } from '@/executor/utils/subflow-utils'
import type { SerializedBlock, SerializedLoop, SerializedWorkflow } from '@/serializer/types'
import { EdgeConstructor } from './edges'
@@ -133,6 +134,129 @@ describe('EdgeConstructor', () => {
})
})
describe('nested subflow skip-at-start bypasses', () => {
it('wires a nested loop start exit to the next sibling inside a parallel branch', () => {
const parallelId = 'parallel-1'
const loopId = 'loop-1'
const afterId = 'after'
const loopStartId = `loop-${loopId}-sentinel-start`
const loopEndId = `loop-${loopId}-sentinel-end`
const afterTemplateId = buildBranchNodeId(afterId, 0)
const dag = createMockDAG([loopStartId, loopEndId, afterTemplateId])
dag.nodes.get(loopStartId)!.metadata = {
isSentinel: true,
sentinelType: 'start',
subflowId: loopId,
subflowType: 'loop',
}
dag.nodes.get(loopEndId)!.metadata = {
isSentinel: true,
sentinelType: 'end',
subflowId: loopId,
subflowType: 'loop',
}
dag.nodes.get(afterTemplateId)!.metadata = {
isParallelBranch: true,
subflowId: parallelId,
subflowType: 'parallel',
branchIndex: 0,
}
dag.loopConfigs.set(loopId, { id: loopId, nodes: [], iterations: 1 })
dag.parallelConfigs.set(parallelId, {
id: parallelId,
nodes: [loopId, afterId],
count: 1,
})
const workflow = createMockWorkflow(
[createMockBlock(loopId, 'loop'), createMockBlock(afterId)],
[{ source: loopId, target: afterId }]
)
edgeConstructor.execute(
workflow,
dag,
new Set([loopId, afterId]),
new Set(),
new Set([loopId, afterId]),
new Map()
)
const loopStartTargets = Array.from(dag.nodes.get(loopStartId)!.outgoingEdges.values())
expect(loopStartTargets).toContainEqual({
target: loopEndId,
sourceHandle: 'loop_exit',
targetHandle: undefined,
})
expect(Array.from(dag.nodes.get(loopEndId)!.outgoingEdges.values())).toContainEqual({
target: afterTemplateId,
sourceHandle: 'loop_exit',
targetHandle: undefined,
})
expect(dag.nodes.get(loopEndId)!.incomingEdges).not.toContain(loopStartId)
})
it('wires a parallel start exit bypass to a downstream parallel sentinel start', () => {
const sourceParallelId = 'parallel-a'
const targetParallelId = 'parallel-b'
const sourceStartId = `parallel-${sourceParallelId}-sentinel-start`
const sourceEndId = `parallel-${sourceParallelId}-sentinel-end`
const targetStartId = `parallel-${targetParallelId}-sentinel-start`
const targetEndId = `parallel-${targetParallelId}-sentinel-end`
const dag = createMockDAG([sourceStartId, sourceEndId, targetStartId, targetEndId])
dag.parallelConfigs.set(sourceParallelId, { id: sourceParallelId, nodes: [], count: 1 })
dag.parallelConfigs.set(targetParallelId, { id: targetParallelId, nodes: [], count: 1 })
const workflow = createMockWorkflow(
[
createMockBlock(sourceParallelId, 'parallel'),
createMockBlock(targetParallelId, 'parallel'),
],
[{ source: sourceParallelId, target: targetParallelId }]
)
edgeConstructor.execute(workflow, dag, new Set(), new Set(), new Set(), new Map())
const sourceStartTargets = Array.from(dag.nodes.get(sourceStartId)!.outgoingEdges.values())
expect(sourceStartTargets).toContainEqual({
target: sourceEndId,
sourceHandle: 'parallel_exit',
targetHandle: undefined,
})
expect(dag.nodes.get(sourceEndId)!.incomingEdges).not.toContain(sourceStartId)
})
it('wires terminal top-level loop start exit through its own sentinel end', () => {
const loopId = 'loop-1'
const taskId = 'task-1'
const loopStartId = `loop-${loopId}-sentinel-start`
const loopEndId = `loop-${loopId}-sentinel-end`
const dag = createMockDAG([loopStartId, loopEndId, taskId])
dag.loopConfigs.set(loopId, { id: loopId, nodes: [taskId], iterations: 1 })
const workflow = createMockWorkflow(
[createMockBlock(loopId, 'loop'), createMockBlock(taskId)],
[]
)
edgeConstructor.execute(
workflow,
dag,
new Set(),
new Set([taskId]),
new Set([taskId]),
new Map()
)
const loopStartTargets = Array.from(dag.nodes.get(loopStartId)!.outgoingEdges.values())
expect(loopStartTargets).toContainEqual({
target: loopEndId,
sourceHandle: 'loop_exit',
targetHandle: undefined,
})
expect(dag.nodes.get(loopEndId)!.incomingEdges).not.toContain(loopStartId)
})
})
describe('Condition block edge wiring', () => {
it('should wire condition block edges with proper condition prefixes', () => {
const conditionId = 'condition-1'
@@ -441,8 +565,9 @@ describe('EdgeConstructor', () => {
// Sentinel start should have edge to node in loop (it's a start node - no incoming from loop)
const sentinelStartNode = dag.nodes.get(sentinelStartId)!
expect(sentinelStartNode.outgoingEdges.size).toBe(1)
const startEdge = Array.from(sentinelStartNode.outgoingEdges.values())[0]
const startEdge = Array.from(sentinelStartNode.outgoingEdges.values()).find(
(edge) => edge.target === nodeInLoopId
)
expect(startEdge.target).toBe(nodeInLoopId)
// Node in loop should have edge to sentinel end (it's a terminal node - no outgoing to loop)
@@ -500,7 +625,10 @@ describe('EdgeConstructor', () => {
// Sentinel start should have edges to both nodes (both are start nodes)
const sentinelStartNode = dag.nodes.get(sentinelStartId)!
expect(sentinelStartNode.outgoingEdges.size).toBe(2)
const bodyStartEdges = Array.from(sentinelStartNode.outgoingEdges.values()).filter(
(edge) => edge.target === node1Id || edge.target === node2Id
)
expect(bodyStartEdges).toHaveLength(2)
// Both nodes should have edges to sentinel end (both are terminal nodes)
const node1 = dag.nodes.get(node1Id)!
@@ -702,7 +830,7 @@ describe('EdgeConstructor', () => {
const loop1StartNode = dag.nodes.get(loop1SentinelStart)!
const earlyExitEdges = Array.from(loop1StartNode.outgoingEdges.values()).filter(
(e) => e.target === loop2SentinelStart && e.sourceHandle === 'loop_exit'
(e) => e.target === loop1SentinelEnd && e.sourceHandle === 'loop_exit'
)
expect(earlyExitEdges.length).toBeGreaterThan(0)
})
@@ -864,7 +992,7 @@ describe('EdgeConstructor', () => {
const loopStartNode = dag.nodes.get(loopSentinelStart)!
const earlyExitEdges = Array.from(loopStartNode.outgoingEdges.values()).filter(
(e) => e.target === parallelSentinelStart && e.sourceHandle === 'loop_exit'
(e) => e.target === loopSentinelEnd && e.sourceHandle === 'loop_exit'
)
expect(earlyExitEdges.length).toBeGreaterThan(0)
})
@@ -925,6 +1053,15 @@ describe('EdgeConstructor', () => {
expect(edgesToRegular.length).toBe(1)
expect(edgesToRegular[0].sourceHandle).toBe('parallel_exit')
const edgesToParallelStart = Array.from(parallelEndNode.outgoingEdges.values()).filter(
(e) => e.target === parallelSentinelStart
)
expect(edgesToParallelStart.length).toBe(1)
expect(edgesToParallelStart[0].sourceHandle).toBe('parallel_continue')
const parallelStartNode = dag.nodes.get(parallelSentinelStart)!
expect(parallelStartNode.incomingEdges.has(parallelSentinelEnd)).toBe(false)
const regularBlockNode = dag.nodes.get(regularBlockId)!
expect(regularBlockNode.incomingEdges.has(parallelSentinelEnd)).toBe(true)
})
@@ -1356,31 +1493,32 @@ describe('EdgeConstructor', () => {
// Set up sentinel metadata
dag.nodes.get(outerSentinelStart)!.metadata = {
isSentinel: true,
isParallelSentinel: true,
sentinelType: 'start',
parallelId: outerParallelId,
subflowId: outerParallelId,
subflowType: 'parallel',
}
dag.nodes.get(outerSentinelEnd)!.metadata = {
isSentinel: true,
isParallelSentinel: true,
sentinelType: 'end',
parallelId: outerParallelId,
subflowId: outerParallelId,
subflowType: 'parallel',
}
dag.nodes.get(innerSentinelStart)!.metadata = {
isSentinel: true,
isParallelSentinel: true,
sentinelType: 'start',
parallelId: innerParallelId,
subflowId: innerParallelId,
subflowType: 'parallel',
}
dag.nodes.get(innerSentinelEnd)!.metadata = {
isSentinel: true,
isParallelSentinel: true,
sentinelType: 'end',
parallelId: innerParallelId,
subflowId: innerParallelId,
subflowType: 'parallel',
}
dag.nodes.get(funcTemplate)!.metadata = {
isParallelBranch: true,
parallelId: innerParallelId,
subflowId: innerParallelId,
subflowType: 'parallel',
branchIndex: 0,
branchTotal: 1,
originalBlockId: functionId,
@@ -1478,28 +1616,31 @@ describe('EdgeConstructor', () => {
dag.nodes.get(loopSentinelStart)!.metadata = {
isSentinel: true,
sentinelType: 'start',
loopId,
subflowId: loopId,
subflowType: 'loop',
}
dag.nodes.get(loopSentinelEnd)!.metadata = {
isSentinel: true,
sentinelType: 'end',
loopId,
subflowId: loopId,
subflowType: 'loop',
}
dag.nodes.get(parallelSentinelStart)!.metadata = {
isSentinel: true,
isParallelSentinel: true,
sentinelType: 'start',
parallelId: innerParallelId,
subflowId: innerParallelId,
subflowType: 'parallel',
}
dag.nodes.get(parallelSentinelEnd)!.metadata = {
isSentinel: true,
isParallelSentinel: true,
sentinelType: 'end',
parallelId: innerParallelId,
subflowId: innerParallelId,
subflowType: 'parallel',
}
dag.nodes.get(funcTemplate)!.metadata = {
isParallelBranch: true,
parallelId: innerParallelId,
subflowId: innerParallelId,
subflowType: 'parallel',
branchIndex: 0,
branchTotal: 1,
originalBlockId: functionId,
@@ -1592,25 +1733,27 @@ describe('EdgeConstructor', () => {
dag.nodes.get(outerSentinelStart)!.metadata = {
isSentinel: true,
isParallelSentinel: true,
sentinelType: 'start',
parallelId: outerParallelId,
subflowId: outerParallelId,
subflowType: 'parallel',
}
dag.nodes.get(outerSentinelEnd)!.metadata = {
isSentinel: true,
isParallelSentinel: true,
sentinelType: 'end',
parallelId: outerParallelId,
subflowId: outerParallelId,
subflowType: 'parallel',
}
dag.nodes.get(innerSentinelStart)!.metadata = {
isSentinel: true,
sentinelType: 'start',
loopId: innerLoopId,
subflowId: innerLoopId,
subflowType: 'loop',
}
dag.nodes.get(innerSentinelEnd)!.metadata = {
isSentinel: true,
sentinelType: 'end',
loopId: innerLoopId,
subflowId: innerLoopId,
subflowType: 'loop',
}
const innerLoop: SerializedLoop = {
+97 -21
View File
@@ -1,6 +1,7 @@
import { createLogger } from '@sim/logger'
import { toError } from '@sim/utils/errors'
import {
CONTROL_BACK_EDGE_HANDLES,
EDGE,
isConditionBlockType,
isRouterBlockType,
@@ -222,16 +223,28 @@ export class EdgeConstructor {
const sourceIsParallelBlock = parallelBlockIds.has(source)
const targetIsParallelBlock = parallelBlockIds.has(target)
let loopSentinelStartId: string | undefined
if (this.edgeStaysWithinSameParallel(originalSource, originalTarget, dag)) {
const sourceId = this.resolveSubflowToSentinelEnd(originalSource, dag)
const targetId = this.resolveSubflowToSentinelStart(originalTarget, dag)
const resolvedSourceHandle = this.resolveParallelChildSourceHandle(
originalSource,
dag,
sourceHandle
)
this.addEdge(dag, sourceId, targetId, resolvedSourceHandle, targetHandle)
this.addSubflowStartExitBypass(dag, originalSource)
continue
}
if (sourceIsLoopBlock) {
const sentinelEndId = buildSentinelEndId(originalSource)
loopSentinelStartId = buildSentinelStartId(originalSource)
const loopSentinelStartId = buildSentinelStartId(originalSource)
if (!dag.nodes.has(sentinelEndId) || !dag.nodes.has(loopSentinelStartId)) {
continue
}
source = sentinelEndId
sourceHandle = EDGE.LOOP_EXIT
this.addSubflowStartExitBypass(dag, originalSource)
}
if (targetIsLoopBlock) {
@@ -264,13 +277,12 @@ export class EdgeConstructor {
target = sentinelStartId
}
if (this.edgeCrossesLoopBoundary(originalSource, originalTarget, blocksInLoops, dag)) {
continue
if (sourceIsParallelBlock) {
this.addSubflowStartExitBypass(dag, originalSource)
}
const sourceLoopNodes = dag.loopConfigs.get(originalSource)?.nodes
if (loopSentinelStartId && !sourceLoopNodes?.includes(originalTarget)) {
this.addEdge(dag, loopSentinelStartId, target, EDGE.LOOP_EXIT, targetHandle)
if (this.edgeCrossesLoopBoundary(originalSource, originalTarget, blocksInLoops, dag)) {
continue
}
if (!this.isEdgeReachable(source, target, reachableBlocks, dag)) {
@@ -308,6 +320,8 @@ export class EdgeConstructor {
continue
}
this.addSubflowStartExitBypass(dag, loopId)
const { startNodes, terminalNodes } = this.findLoopBoundaryNodes(nodes, dag)
for (const startNodeId of startNodes) {
@@ -326,12 +340,15 @@ export class EdgeConstructor {
? EDGE.PARALLEL_EXIT
: EDGE.LOOP_EXIT
this.addEdge(dag, resolvedId, sentinelEndId, handle)
this.addSubflowStartExitBypass(dag, terminalNodeId)
} else {
this.addEdge(dag, resolvedId, sentinelEndId)
}
}
this.addEdge(dag, sentinelEndId, sentinelStartId, EDGE.LOOP_CONTINUE, undefined, true)
this.addEdge(dag, sentinelEndId, sentinelStartId, EDGE.LOOP_CONTINUE, undefined, {
registerIncoming: false,
})
}
}
@@ -348,6 +365,8 @@ export class EdgeConstructor {
continue
}
this.addSubflowStartExitBypass(dag, parallelId)
const { entryNodes, terminalNodes } = this.findParallelBoundaryNodes(nodes, dag)
for (const entryNodeId of entryNodes) {
@@ -360,12 +379,17 @@ export class EdgeConstructor {
for (const terminalNodeId of terminalNodes) {
const sourceId = this.resolveSubflowToSentinelEnd(terminalNodeId, dag)
if (dag.nodes.has(sourceId)) {
// Use the sourceHandle that matches the nested subflow's exit route.
// A nested loop sentinel-end outputs "loop_exit", not "parallel_exit".
const handle = dag.loopConfigs.has(terminalNodeId) ? EDGE.LOOP_EXIT : EDGE.PARALLEL_EXIT
const handle = this.resolveSubflowExitHandle(terminalNodeId, dag)
this.addEdge(dag, sourceId, sentinelEndId, handle)
if (handle) {
this.addSubflowStartExitBypass(dag, terminalNodeId)
}
}
}
this.addEdge(dag, sentinelEndId, sentinelStartId, EDGE.PARALLEL_CONTINUE, undefined, {
registerIncoming: false,
})
}
}
@@ -566,9 +590,7 @@ export class EdgeConstructor {
let hasOutgoingToLoop = false
for (const [, edge] of node.outgoingEdges) {
const isBackEdge =
edge.sourceHandle === EDGE.LOOP_CONTINUE || edge.sourceHandle === EDGE.LOOP_CONTINUE_ALT
if (isBackEdge) continue
if (this.isControlBackEdge(edge.sourceHandle)) continue
if (effectiveNodeSet.has(edge.target)) {
hasOutgoingToLoop = true
@@ -620,10 +642,7 @@ export class EdgeConstructor {
if (endNode) {
let hasOutgoingToParallel = false
for (const [, edge] of endNode.outgoingEdges) {
// Skip loop back-edges — they don't count as forward edges within the parallel
const isBackEdge =
edge.sourceHandle === EDGE.LOOP_CONTINUE || edge.sourceHandle === EDGE.LOOP_CONTINUE_ALT
if (isBackEdge) continue
if (this.isControlBackEdge(edge.sourceHandle)) continue
const originalTargetId = normalizeNodeId(edge.target)
if (nodesSet.has(originalTargetId)) {
@@ -667,6 +686,10 @@ export class EdgeConstructor {
return { startNode: templateNode, endNode: templateNode }
}
private isControlBackEdge(sourceHandle?: string): boolean {
return sourceHandle !== undefined && CONTROL_BACK_EDGE_HANDLES.has(sourceHandle)
}
private getParallelId(blockId: string, dag: DAG): string | null {
for (const [parallelId, parallelConfig] of dag.parallelConfigs) {
if (parallelConfig.nodes.includes(blockId)) {
@@ -676,13 +699,66 @@ export class EdgeConstructor {
return null
}
private edgeStaysWithinSameParallel(source: string, target: string, dag: DAG): boolean {
const sourceParallelId = this.getParallelId(source, dag)
const targetParallelId = this.getParallelId(target, dag)
return !!sourceParallelId && sourceParallelId === targetParallelId
}
private resolveParallelChildSourceHandle(
source: string,
dag: DAG,
sourceHandle?: string
): string | undefined {
if (dag.parallelConfigs.has(source)) {
return EDGE.PARALLEL_EXIT
}
if (dag.loopConfigs.has(source)) {
return EDGE.LOOP_EXIT
}
return sourceHandle
}
private resolveSubflowExitHandle(nodeId: string, dag: DAG): string | undefined {
if (dag.parallelConfigs.has(nodeId)) {
return EDGE.PARALLEL_EXIT
}
if (dag.loopConfigs.has(nodeId)) {
return EDGE.LOOP_EXIT
}
return undefined
}
private addSubflowStartExitBypass(dag: DAG, subflowId: string): void {
if (dag.parallelConfigs.has(subflowId)) {
const sourceId = buildParallelSentinelStartId(subflowId)
const targetId = buildParallelSentinelEndId(subflowId)
if (dag.nodes.has(sourceId) && dag.nodes.has(targetId)) {
this.addEdge(dag, sourceId, targetId, EDGE.PARALLEL_EXIT, undefined, {
registerIncoming: false,
})
}
return
}
if (dag.loopConfigs.has(subflowId)) {
const sourceId = buildSentinelStartId(subflowId)
const targetId = buildSentinelEndId(subflowId)
if (dag.nodes.has(sourceId) && dag.nodes.has(targetId)) {
this.addEdge(dag, sourceId, targetId, EDGE.LOOP_EXIT, undefined, {
registerIncoming: false,
})
}
}
}
private addEdge(
dag: DAG,
sourceId: string,
targetId: string,
sourceHandle?: string,
targetHandle?: string,
isLoopBackEdge = false
options: { registerIncoming?: boolean } = {}
): void {
const sourceNode = dag.nodes.get(sourceId)
const targetNode = dag.nodes.get(targetId)
@@ -698,10 +774,10 @@ export class EdgeConstructor {
target: targetId,
sourceHandle,
targetHandle,
isActive: isLoopBackEdge ? false : undefined,
})
if (!isLoopBackEdge) {
const { registerIncoming = true } = options
if (registerIncoming) {
targetNode.incomingEdges.add(sourceId)
}
}
+9 -38
View File
@@ -1,10 +1,8 @@
import { createLogger } from '@sim/logger'
import { BlockType, LOOP, type SentinelType } from '@/executor/constants'
import type { DAG, DAGNode } from '@/executor/dag/builder'
import { BlockType, LOOP } from '@/executor/constants'
import type { DAG } from '@/executor/dag/builder'
import { createSubflowSentinelNode } from '@/executor/dag/construction/sentinels'
import { buildSentinelEndId, buildSentinelStartId } from '@/executor/utils/subflow-utils'
const logger = createLogger('LoopConstructor')
export class LoopConstructor {
execute(dag: DAG, reachableBlocks: Set<string>): void {
for (const [loopId, loopConfig] of dag.loopConfigs) {
@@ -29,9 +27,10 @@ export class LoopConstructor {
dag.nodes.set(
startId,
this.createSentinelNode({
createSubflowSentinelNode({
id: startId,
loopId,
subflowId: loopId,
subflowType: 'loop',
sentinelType: LOOP.SENTINEL.START_TYPE,
blockType: BlockType.SENTINEL_START,
name: `${LOOP.SENTINEL.START_NAME_PREFIX} (${loopId})`,
@@ -40,42 +39,14 @@ export class LoopConstructor {
dag.nodes.set(
endId,
this.createSentinelNode({
createSubflowSentinelNode({
id: endId,
loopId,
subflowId: loopId,
subflowType: 'loop',
sentinelType: LOOP.SENTINEL.END_TYPE,
blockType: BlockType.SENTINEL_END,
name: `${LOOP.SENTINEL.END_NAME_PREFIX} (${loopId})`,
})
)
}
private createSentinelNode(config: {
id: string
loopId: string
sentinelType: SentinelType
blockType: BlockType
name: string
}): DAGNode {
return {
id: config.id,
block: {
id: config.id,
enabled: true,
metadata: {
id: config.blockType,
name: config.name,
loopId: config.loopId,
},
config: { params: {} },
} as any,
incomingEdges: new Set(),
outgoingEdges: new Map(),
metadata: {
isSentinel: true,
sentinelType: config.sentinelType,
loopId: config.loopId,
},
}
}
}
@@ -0,0 +1,46 @@
/**
* @vitest-environment node
*/
import { describe, expect, it } from 'vitest'
import { BlockType } from '@/executor/constants'
import type { DAG } from '@/executor/dag/builder'
import { NodeConstructor } from '@/executor/dag/construction/nodes'
import type { SerializedWorkflow } from '@/serializer/types'
describe('NodeConstructor', () => {
it('assigns nested loop nodes to the innermost loop metadata', () => {
const dag: DAG = {
nodes: new Map(),
loopConfigs: new Map([
['outer-loop', { id: 'outer-loop', nodes: ['inner-loop', 'task'], iterations: 1 }],
['inner-loop', { id: 'inner-loop', nodes: ['task'], iterations: 1 }],
]),
parallelConfigs: new Map(),
}
const workflow: SerializedWorkflow = {
version: '1',
blocks: [
{
id: 'task',
position: { x: 0, y: 0 },
config: { tool: '', params: {} },
inputs: {},
outputs: {},
metadata: { id: BlockType.FUNCTION, name: 'Task' },
enabled: true,
},
],
connections: [],
loops: {},
parallels: {},
}
new NodeConstructor().execute(workflow, dag, new Set(['task']))
expect(dag.nodes.get('task')?.metadata).toMatchObject({
isLoopNode: true,
subflowId: 'inner-loop',
subflowType: 'loop',
})
})
})
+14 -4
View File
@@ -104,7 +104,8 @@ export class NodeConstructor {
outgoingEdges: new Map(),
metadata: {
isParallelBranch: true,
parallelId,
subflowId: parallelId,
subflowType: 'parallel',
branchIndex: 0,
branchTotal: 1,
isPauseResponse: block.metadata?.id === BlockType.HUMAN_IN_THE_LOOP,
@@ -129,7 +130,7 @@ export class NodeConstructor {
outgoingEdges: new Map(),
metadata: {
isLoopNode,
loopId,
...(loopId && { subflowId: loopId, subflowType: 'loop' as const }),
isPauseResponse: isPauseBlock,
originalBlockId: block.id,
},
@@ -137,12 +138,21 @@ export class NodeConstructor {
}
private findLoopIdForBlock(blockId: string, dag: DAG): string | undefined {
const candidates: string[] = []
for (const [loopId, loopConfig] of dag.loopConfigs) {
if (loopConfig.nodes.includes(blockId)) {
return loopId
candidates.push(loopId)
}
}
return undefined
if (candidates.length <= 1) return candidates[0]
return candidates.find((candidateId) =>
candidates.every((otherId) => {
if (otherId === candidateId) return true
const candidateConfig = dag.loopConfigs.get(candidateId)
return !candidateConfig?.nodes.includes(otherId)
})
)
}
private findParallelForBlock(blockId: string, dag: DAG): string | null {
@@ -1,13 +1,11 @@
import { createLogger } from '@sim/logger'
import { BlockType, PARALLEL, type SentinelType } from '@/executor/constants'
import type { DAG, DAGNode } from '@/executor/dag/builder'
import { BlockType, PARALLEL } from '@/executor/constants'
import type { DAG } from '@/executor/dag/builder'
import { createSubflowSentinelNode } from '@/executor/dag/construction/sentinels'
import {
buildParallelSentinelEndId,
buildParallelSentinelStartId,
} from '@/executor/utils/subflow-utils'
const logger = createLogger('ParallelConstructor')
export class ParallelConstructor {
execute(dag: DAG, reachableBlocks: Set<string>): void {
for (const [parallelId, parallelConfig] of dag.parallelConfigs) {
@@ -32,9 +30,10 @@ export class ParallelConstructor {
dag.nodes.set(
startId,
this.createSentinelNode({
createSubflowSentinelNode({
id: startId,
parallelId,
subflowId: parallelId,
subflowType: 'parallel',
sentinelType: PARALLEL.SENTINEL.START_TYPE,
blockType: BlockType.SENTINEL_START,
name: `${PARALLEL.SENTINEL.START_NAME_PREFIX} (${parallelId})`,
@@ -43,43 +42,14 @@ export class ParallelConstructor {
dag.nodes.set(
endId,
this.createSentinelNode({
createSubflowSentinelNode({
id: endId,
parallelId,
subflowId: parallelId,
subflowType: 'parallel',
sentinelType: PARALLEL.SENTINEL.END_TYPE,
blockType: BlockType.SENTINEL_END,
name: `${PARALLEL.SENTINEL.END_NAME_PREFIX} (${parallelId})`,
})
)
}
private createSentinelNode(config: {
id: string
parallelId: string
sentinelType: SentinelType
blockType: BlockType
name: string
}): DAGNode {
return {
id: config.id,
block: {
id: config.id,
enabled: true,
metadata: {
id: config.blockType,
name: config.name,
parallelId: config.parallelId,
},
config: { params: {} },
} as any,
incomingEdges: new Set(),
outgoingEdges: new Map(),
metadata: {
isSentinel: true,
isParallelSentinel: true,
sentinelType: config.sentinelType,
parallelId: config.parallelId,
},
}
}
}
@@ -0,0 +1,38 @@
import type { BlockType, SentinelType } from '@/executor/constants'
import type { DAGNode } from '@/executor/dag/builder'
import type { SentinelSubflowType } from '@/executor/dag/types'
interface SubflowSentinelNodeConfig {
id: string
subflowId: string
subflowType: SentinelSubflowType
sentinelType: SentinelType
blockType: BlockType
name: string
}
export function createSubflowSentinelNode(config: SubflowSentinelNodeConfig): DAGNode {
return {
id: config.id,
block: {
id: config.id,
enabled: true,
position: { x: 0, y: 0 },
metadata: {
id: config.blockType,
name: config.name,
},
config: { tool: config.blockType, params: {} },
inputs: {},
outputs: {},
},
incomingEdges: new Set(),
outgoingEdges: new Map(),
metadata: {
isSentinel: true,
sentinelType: config.sentinelType,
subflowId: config.subflowId,
subflowType: config.subflowType,
},
}
}
+4 -4
View File
@@ -2,20 +2,20 @@ export interface DAGEdge {
target: string
sourceHandle?: string
targetHandle?: string
isActive?: boolean
}
export type SentinelSubflowType = 'loop' | 'parallel'
export interface NodeMetadata {
isParallelBranch?: boolean
isParallelSentinel?: boolean
parallelId?: string
branchIndex?: number
branchTotal?: number
distributionItem?: unknown
isLoopNode?: boolean
loopId?: string
isSentinel?: boolean
sentinelType?: 'start' | 'end'
subflowType?: SentinelSubflowType
subflowId?: string
isPauseResponse?: boolean
isResumeTrigger?: boolean
originalBlockId?: string
@@ -126,4 +126,257 @@ describe('BlockExecutor', () => {
totalCount: output.result.length,
})
})
it('persists stable outer-branch aliases for completed parallel branch outputs', async () => {
const block = createBlock()
const workflow: SerializedWorkflow = {
version: '1',
blocks: [block],
connections: [],
loops: {},
parallels: {},
}
const state = new ExecutionState()
const resolver = new VariableResolver(workflow, {}, state)
const output = { result: 'branch-2' }
const handler: BlockHandler = {
canHandle: () => true,
execute: async () => output,
}
const executor = new BlockExecutor(
[handler],
resolver,
{
workspaceId: 'workspace-1',
executionId: 'execution-1',
userId: 'user-1',
metadata: {
requestId: 'request-1',
executionId: 'execution-1',
workflowId: 'workflow-1',
workspaceId: 'workspace-1',
userId: 'user-1',
triggerType: 'manual',
useDraftState: false,
startTime: new Date().toISOString(),
},
},
state
)
const node = createNode(block)
node.id = 'function-block-1₍0₎'
node.metadata = {
isParallelBranch: true,
subflowId: 'parallel-1',
subflowType: 'parallel',
originalBlockId: block.id,
branchIndex: 2,
}
await executor.execute(createContext(state), node, block)
expect(state.getBlockOutput('function-block-1__obranch-2')).toEqual(output)
expect(state.getBlockOutput('function-block-1₍2₎')).toEqual(output)
expect(state.getBlockOutput('function-block-1₍0₎')).toEqual(output)
})
it('does not write global aliases for parallel branches inside cloned outer branches', async () => {
const block = createBlock()
const workflow: SerializedWorkflow = {
version: '1',
blocks: [block],
connections: [],
loops: {},
parallels: {},
}
const state = new ExecutionState()
const resolver = new VariableResolver(workflow, {}, state)
const output = { result: 'outer-2-inner-0' }
const handler: BlockHandler = {
canHandle: () => true,
execute: async () => output,
}
const executor = new BlockExecutor(
[handler],
resolver,
{
workspaceId: 'workspace-1',
executionId: 'execution-1',
userId: 'user-1',
metadata: {
requestId: 'request-1',
executionId: 'execution-1',
workflowId: 'workflow-1',
workspaceId: 'workspace-1',
userId: 'user-1',
triggerType: 'manual',
useDraftState: false,
startTime: new Date().toISOString(),
},
},
state
)
const node = createNode(block)
node.id = 'function-block-1__cloneabc__obranch-2₍0₎'
node.metadata = {
isParallelBranch: true,
subflowId: 'inner-parallel',
subflowType: 'parallel',
originalBlockId: block.id,
branchIndex: 0,
}
await executor.execute(createContext(state), node, block)
expect(state.getBlockOutput(node.id)).toEqual(output)
expect(state.getBlockOutput('function-block-1__obranch-0')).toBeUndefined()
expect(state.getBlockOutput('function-block-1₍0₎')).toBeUndefined()
})
it('does not let block completion callbacks overtake pending start callbacks', async () => {
const block = createBlock()
const workflow: SerializedWorkflow = {
version: '1',
blocks: [block],
connections: [],
loops: {},
parallels: {},
}
const state = new ExecutionState()
const resolver = new VariableResolver(workflow, {}, state)
const output = { result: 'done' }
const execute = vi.fn(async () => {
events.push('execute')
return output
})
const handler: BlockHandler = {
canHandle: () => true,
execute,
}
const events: string[] = []
let resolveStart!: () => void
const startGate = new Promise<void>((resolve) => {
resolveStart = resolve
})
const onBlockStart = vi.fn(async () => {
events.push('start-called')
await startGate
events.push('start-done')
})
const onBlockComplete = vi.fn(async () => {
events.push('complete')
})
const executor = new BlockExecutor(
[handler],
resolver,
{
workspaceId: 'workspace-1',
executionId: 'execution-1',
userId: 'user-1',
metadata: {
requestId: 'request-1',
executionId: 'execution-1',
workflowId: 'workflow-1',
workspaceId: 'workspace-1',
userId: 'user-1',
triggerType: 'manual',
useDraftState: false,
startTime: new Date().toISOString(),
},
onBlockStart,
onBlockComplete,
},
state
)
const execution = executor.execute(createContext(state), createNode(block), block)
expect(onBlockStart).toHaveBeenCalled()
expect(execute).not.toHaveBeenCalled()
expect(onBlockComplete).not.toHaveBeenCalled()
resolveStart()
await execution
await vi.waitFor(() => {
expect(onBlockComplete).toHaveBeenCalled()
})
expect(events).toEqual(['start-called', 'start-done', 'execute', 'complete'])
})
it('fires block completion callbacks for pausing blocks so clients receive pause output', async () => {
const block = {
...createBlock(),
id: 'hitl-block-1',
metadata: { id: BlockType.HUMAN_IN_THE_LOOP, name: 'Human in the Loop' },
config: { tool: BlockType.HUMAN_IN_THE_LOOP, params: {} },
}
const workflow: SerializedWorkflow = {
version: '1',
blocks: [block],
connections: [],
loops: {},
parallels: {},
}
const state = new ExecutionState()
const resolver = new VariableResolver(workflow, {}, state)
const output = {
response: { status: 'paused' },
_pauseMetadata: {
contextId: 'pause-context-1',
blockId: block.id,
response: { status: 'paused' },
timestamp: new Date().toISOString(),
pauseKind: 'human' as const,
},
}
const handler: BlockHandler = {
canHandle: () => true,
execute: async () => output,
}
const onBlockStart = vi.fn(async () => {})
const onBlockComplete = vi.fn(async () => {})
const executor = new BlockExecutor(
[handler],
resolver,
{
workspaceId: 'workspace-1',
executionId: 'execution-1',
userId: 'user-1',
metadata: {
requestId: 'request-1',
executionId: 'execution-1',
workflowId: 'workflow-1',
workspaceId: 'workspace-1',
userId: 'user-1',
triggerType: 'manual',
useDraftState: false,
startTime: new Date().toISOString(),
},
onBlockStart,
onBlockComplete,
},
state
)
await executor.execute(createContext(state), createNode(block), block)
expect(onBlockStart).toHaveBeenCalled()
expect(onBlockComplete).toHaveBeenCalledWith(
block.id,
'Human in the Loop',
BlockType.HUMAN_IN_THE_LOOP,
expect.objectContaining({
output: expect.objectContaining({
response: { status: 'paused' },
}),
}),
undefined,
undefined
)
expect(state.getBlockOutput(block.id)).toEqual(output)
})
})
+71 -24
View File
@@ -46,6 +46,11 @@ import {
} from '@/executor/utils/iteration-context'
import { isJSONString } from '@/executor/utils/json'
import { filterOutputForLog } from '@/executor/utils/output-filter'
import {
buildBranchNodeId,
buildOuterBranchScopedId,
extractOuterBranchIndex,
} from '@/executor/utils/subflow-utils'
import {
FUNCTION_BLOCK_CONTEXT_VARS_KEY,
FUNCTION_BLOCK_DISPLAY_CODE_KEY,
@@ -98,10 +103,12 @@ export class BlockExecutor {
const startTime = performance.now()
let blockLog: BlockLog | undefined
let blockStartPromise: Promise<void> | undefined
if (!isSentinel) {
blockLog = this.createBlockLog(ctx, node.id, block, node, startedAt)
ctx.blockLogs.push(blockLog)
this.fireBlockStartCallback(ctx, node, block, blockLog.executionOrder)
blockStartPromise = this.fireBlockStartCallback(ctx, node, block, blockLog.executionOrder)
await blockStartPromise
}
let resolvedInputs: Record<string, any> = {}
@@ -156,6 +163,7 @@ export class BlockExecutor {
ctx,
node,
block,
blockStartPromise,
startTime,
blockLog,
inputsForLog,
@@ -234,7 +242,7 @@ export class BlockExecutor {
}
const { childTraceSpans: _traces, ...outputForState } = normalizedOutput
this.state.setBlockOutput(node.id, outputForState as NormalizedBlockOutput, duration)
this.setNodeOutput(node, outputForState as NormalizedBlockOutput, duration)
if (!isSentinel && blockLog) {
const childWorkflowInstanceId =
@@ -245,6 +253,7 @@ export class BlockExecutor {
block,
})
this.fireBlockCompleteCallback(
blockStartPromise,
ctx,
node,
block,
@@ -265,6 +274,7 @@ export class BlockExecutor {
ctx,
node,
block,
blockStartPromise,
startTime,
blockLog,
inputsForLog,
@@ -278,8 +288,10 @@ export class BlockExecutor {
const metadata = node?.metadata ?? {}
return {
nodeId: node.id,
loopId: metadata.loopId,
parallelId: metadata.parallelId,
loopId: metadata.subflowType === 'loop' ? metadata.subflowId : undefined,
parallelId: metadata.subflowType === 'parallel' ? metadata.subflowId : undefined,
subflowId: metadata.subflowId,
subflowType: metadata.subflowType,
branchIndex: metadata.branchIndex,
branchTotal: metadata.branchTotal,
originalBlockId: metadata.originalBlockId,
@@ -287,6 +299,29 @@ export class BlockExecutor {
}
}
private setNodeOutput(node: DAGNode, output: NormalizedBlockOutput, duration = 0): void {
this.state.setBlockOutput(node.id, output, duration)
const originalBlockId = node.metadata.originalBlockId
const branchIndex = node.metadata.branchIndex
if (
node.metadata.isParallelBranch &&
originalBlockId &&
branchIndex !== undefined &&
extractOuterBranchIndex(node.id) === undefined
) {
const globalBranchNodeId = buildBranchNodeId(originalBlockId, branchIndex)
if (globalBranchNodeId !== node.id) {
this.state.setBlockOutput(globalBranchNodeId, output, duration)
}
this.state.setBlockOutput(
buildOuterBranchScopedId(originalBlockId, branchIndex),
output,
duration
)
}
}
private findHandler(block: SerializedBlock): BlockHandler | undefined {
return this.blockHandlers.find((h) => h.canHandle(block))
}
@@ -296,6 +331,7 @@ export class BlockExecutor {
ctx: ExecutionContext,
node: DAGNode,
block: SerializedBlock,
blockStartPromise: Promise<void> | undefined,
startTime: number,
blockLog: BlockLog | undefined,
inputsForLog: Record<string, any>,
@@ -322,7 +358,7 @@ export class BlockExecutor {
}
}
this.state.setBlockOutput(node.id, errorOutput, duration)
this.setNodeOutput(node, errorOutput, duration)
if (blockLog) {
blockLog.endedAt = endedAt
@@ -352,6 +388,7 @@ export class BlockExecutor {
: undefined
const displayOutput = filterOutputForLog(block.metadata?.id || '', errorOutput, { block })
this.fireBlockCompleteCallback(
blockStartPromise,
ctx,
node,
block,
@@ -412,12 +449,20 @@ export class BlockExecutor {
let iterationIndex: number | undefined
if (node?.metadata) {
if (node.metadata.branchIndex !== undefined && node.metadata.parallelId) {
if (
node.metadata.branchIndex !== undefined &&
node.metadata.subflowType === 'parallel' &&
node.metadata.subflowId
) {
blockName = `${blockName} (iteration ${node.metadata.branchIndex})`
iterationIndex = node.metadata.branchIndex
parallelId = node.metadata.parallelId
} else if (node.metadata.isLoopNode && node.metadata.loopId) {
loopId = node.metadata.loopId
parallelId = node.metadata.subflowId
} else if (
node.metadata.isLoopNode &&
node.metadata.subflowType === 'loop' &&
node.metadata.subflowId
) {
loopId = node.metadata.subflowId
const loopScope = ctx.loopExecutions?.get(loopId)
if (loopScope && loopScope.iteration !== undefined) {
blockName = `${blockName} (iteration ${loopScope.iteration})`
@@ -510,23 +555,23 @@ export class BlockExecutor {
}
/**
* Fires the `onBlockStart` progress callback without blocking block execution.
* Any error is logged and swallowed so callback I/O never stalls the critical path.
* Fires the `onBlockStart` progress callback before block execution continues.
* Returning the promise lets completion callbacks preserve lifecycle ordering.
*/
private fireBlockStartCallback(
ctx: ExecutionContext,
node: DAGNode,
block: SerializedBlock,
executionOrder: number
): void {
if (!this.contextExtensions.onBlockStart) return
): Promise<void> | undefined {
if (!this.contextExtensions.onBlockStart) return undefined
const blockId = node.metadata?.originalBlockId ?? node.id
const blockName = block.metadata?.name ?? blockId
const blockType = block.metadata?.id ?? DEFAULTS.BLOCK_TYPE
const iterationContext = getIterationContext(ctx, node?.metadata)
void this.contextExtensions
return this.contextExtensions
.onBlockStart(
blockId,
blockName,
@@ -546,10 +591,11 @@ export class BlockExecutor {
/**
* Fires the `onBlockComplete` progress callback without blocking subsequent blocks.
* The callback typically performs DB writes for progress markers — awaiting it would
* add latency between blocks and skew wall-clock timing in the trace view.
* Completion is chained behind the matching start callback so SSE/log consumers
* never observe `block:completed` before `block:started` for the same execution.
*/
private fireBlockCompleteCallback(
blockStartPromise: Promise<void> | undefined,
ctx: ExecutionContext,
node: DAGNode,
block: SerializedBlock,
@@ -568,8 +614,9 @@ export class BlockExecutor {
const blockType = block.metadata?.id ?? DEFAULTS.BLOCK_TYPE
const iterationContext = getIterationContext(ctx, node?.metadata)
void this.contextExtensions
.onBlockComplete(
void (async () => {
await blockStartPromise
await this.contextExtensions.onBlockComplete?.(
blockId,
blockName,
blockType,
@@ -585,13 +632,13 @@ export class BlockExecutor {
iterationContext,
ctx.childWorkflowContext
)
.catch((error) => {
this.execLogger.warn('Block completion callback failed', {
blockId,
blockType,
error: toError(error).message,
})
})().catch((error) => {
this.execLogger.warn('Block completion callback failed', {
blockId,
blockType,
error: toError(error).message,
})
})
}
private preparePauseResumeSelfReference(
+286 -17
View File
@@ -1,4 +1,5 @@
import { describe, expect, it } from 'vitest'
import { EDGE } from '@/executor/constants'
import type { DAG, DAGNode } from '@/executor/dag/builder'
import type { DAGEdge } from '@/executor/dag/types'
import type { SerializedBlock } from '@/serializer/types'
@@ -162,6 +163,33 @@ describe('EdgeManager', () => {
})
})
describe('Subflow control edges', () => {
it('does not activate loop exit edge without matching selected route', () => {
const loopStartId = 'loop-loop-1-sentinel-start'
const bodyId = 'body'
const afterLoopId = 'after-loop'
const loopStartNode = createMockNode(loopStartId, [
{ target: bodyId },
{ target: afterLoopId, sourceHandle: EDGE.LOOP_EXIT },
])
const bodyNode = createMockNode(bodyId, [], [loopStartId])
const afterLoopNode = createMockNode(afterLoopId, [], [loopStartId])
const dag = createMockDAG(
new Map<string, DAGNode>([
[loopStartId, loopStartNode],
[bodyId, bodyNode],
[afterLoopId, afterLoopNode],
])
)
const edgeManager = new EdgeManager(dag)
const readyNodes = edgeManager.processOutgoingEdges(loopStartNode, { sentinelStart: true })
expect(readyNodes).toContain(bodyId)
expect(readyNodes).not.toContain(afterLoopId)
})
})
describe('Multiple condition edges to same target', () => {
it('should not cascade-deactivate when multiple edges from same source go to same target', () => {
const conditionId = 'condition-1'
@@ -598,6 +626,34 @@ describe('EdgeManager', () => {
expect(readyNodes).toContain(function1Id)
})
it('does not clear deactivated edges from prefix-sharing source node IDs', () => {
const shortSourceId = 'a'
const longSourceId = 'a-b'
const targetId = 'target'
const shortSourceNode = createMockNode(shortSourceId)
const longSourceNode = createMockNode(longSourceId, [
{ target: targetId, sourceHandle: 'condition-if' },
])
const targetNode = createMockNode(targetId, [], [longSourceId])
const dag = createMockDAG(
new Map<string, DAGNode>([
[shortSourceId, shortSourceNode],
[longSourceId, longSourceNode],
[targetId, targetNode],
])
)
const edgeManager = new EdgeManager(dag)
edgeManager.processOutgoingEdges(longSourceNode, { selectedOption: 'else' })
expect(edgeManager.isNodeReady(targetNode)).toBe(true)
edgeManager.clearDeactivatedEdgesForNodes(new Set([shortSourceId]))
expect(edgeManager.isNodeReady(targetNode)).toBe(true)
})
/**
* Regression for the substring-match bug in clearDeactivatedEdgesForNodes.
*
@@ -788,6 +844,37 @@ describe('EdgeManager', () => {
})
})
describe('restoreDeactivatedEdges', () => {
it('restores activated target state used by convergent routing after resume', () => {
const edgeManager = new EdgeManager(createMockDAG(new Map()))
edgeManager.restoreDeactivatedEdges([], ['join'])
expect(edgeManager.getNodesWithActivatedEdge()).toEqual(['join'])
})
it('normalizes legacy deactivated edge keys on restore', () => {
const sourceNode = createMockNode('source-node', [
{ target: 'target-node', sourceHandle: 'condition-if' },
])
const targetNode = createMockNode('target-node', [], ['source-node'])
const edgeManager = new EdgeManager(
createMockDAG(
new Map<string, DAGNode>([
[sourceNode.id, sourceNode],
[targetNode.id, targetNode],
])
)
)
edgeManager.restoreDeactivatedEdges(['source-node-target-node-condition-if'])
expect(edgeManager.getDeactivatedEdges()).toEqual([
JSON.stringify(['source-node', 'target-node', 'condition-if']),
])
})
})
describe('Diamond pattern (convergent paths)', () => {
it('should handle diamond: condition splits then converges at merge point', () => {
const conditionId = 'condition-1'
@@ -1206,7 +1293,11 @@ describe('EdgeManager', () => {
const edgeManager = new EdgeManager(dag)
// Process without skipping backwards edges
const readyNodes = edgeManager.processOutgoingEdges(loopBodyNode, {}, false)
const readyNodes = edgeManager.processOutgoingEdges(
loopBodyNode,
{ selectedRoute: EDGE.LOOP_CONTINUE },
false
)
// Loop start should be activated
expect(readyNodes).toContain(loopStartId)
@@ -1302,6 +1393,107 @@ describe('EdgeManager', () => {
expect(readyNodes).not.toContain(target2Id)
expect(readyNodes).toHaveLength(0)
})
it('should handle no matching router route without queuing downstream sentinels', () => {
const routerId = 'router-1'
const routeTargetId = 'route-target'
const downstreamSentinelId = 'loop-downstream-sentinel-end'
const afterLoopId = 'after-loop'
const routerNode = createMockNode(routerId, [
{ target: routeTargetId, sourceHandle: 'router-route1' },
])
const routeTargetNode = createMockNode(
routeTargetId,
[{ target: downstreamSentinelId }],
[routerId]
)
const downstreamSentinelNode = createMockNode(
downstreamSentinelId,
[{ target: afterLoopId, sourceHandle: EDGE.LOOP_EXIT }],
[routeTargetId]
)
downstreamSentinelNode.metadata = {
isSentinel: true,
sentinelType: 'end',
subflowId: 'downstream',
subflowType: 'loop',
}
const afterLoopNode = createMockNode(afterLoopId, [], [downstreamSentinelId])
const nodes = new Map<string, DAGNode>([
[routerId, routerNode],
[routeTargetId, routeTargetNode],
[downstreamSentinelId, downstreamSentinelNode],
[afterLoopId, afterLoopNode],
])
const dag = createMockDAG(nodes)
const edgeManager = new EdgeManager(dag)
const readyNodes = edgeManager.processOutgoingEdges(routerNode, {
selectedRoute: 'missing-route',
})
expect(readyNodes).not.toContain(routeTargetId)
expect(readyNodes).not.toContain(downstreamSentinelId)
expect(readyNodes).toHaveLength(0)
})
it('should allow no matching router route to queue an enclosing subflow sentinel', () => {
const loopId = 'loop-1'
const routerId = 'router-1'
const routeTargetId = 'route-target'
const loopEndId = `loop-${loopId}-sentinel-end`
const loopStartId = `loop-${loopId}-sentinel-start`
const routerNode = createMockNode(
routerId,
[{ target: routeTargetId, sourceHandle: 'router-route1' }],
[loopStartId]
)
routerNode.metadata = {
subflowId: loopId,
subflowType: 'loop',
}
const routeTargetNode = createMockNode(routeTargetId, [{ target: loopEndId }], [routerId])
routeTargetNode.metadata = {
subflowId: loopId,
subflowType: 'loop',
}
const loopEndNode = createMockNode(
loopEndId,
[
{ target: loopStartId, sourceHandle: EDGE.LOOP_CONTINUE },
{ target: 'after-loop', sourceHandle: EDGE.LOOP_EXIT },
],
[routeTargetId]
)
loopEndNode.metadata = {
isSentinel: true,
sentinelType: 'end',
subflowId: loopId,
subflowType: 'loop',
}
const loopStartNode = createMockNode(loopStartId, [{ target: routerId }], [loopEndId])
const afterLoopNode = createMockNode('after-loop', [], [loopEndId])
const nodes = new Map<string, DAGNode>([
[routerId, routerNode],
[routeTargetId, routeTargetNode],
[loopEndId, loopEndNode],
[loopStartId, loopStartNode],
['after-loop', afterLoopNode],
])
const dag = createMockDAG(nodes)
const edgeManager = new EdgeManager(dag)
const readyNodes = edgeManager.processOutgoingEdges(routerNode, {
selectedRoute: 'missing-route',
})
expect(readyNodes).not.toContain(routeTargetId)
expect(readyNodes).toContain(loopEndId)
})
})
describe('Condition inside loop - loop control edges should not be cascade-deactivated', () => {
@@ -1430,6 +1622,35 @@ describe('EdgeManager', () => {
expect(readyAfterParallelEnd).toContain(afterParallelId)
})
it('should route parallel_continue back to parallel start without activating exit', () => {
const parallelStartId = 'parallel-start'
const parallelEndId = 'parallel-end'
const afterParallelId = 'after-parallel'
const parallelStartNode = createMockNode(parallelStartId)
const parallelEndNode = createMockNode(parallelEndId, [
{ target: parallelStartId, sourceHandle: 'parallel_continue' },
{ target: afterParallelId, sourceHandle: 'parallel_exit' },
])
const afterParallelNode = createMockNode(afterParallelId, [], [parallelEndId])
const nodes = new Map<string, DAGNode>([
[parallelStartId, parallelStartNode],
[parallelEndId, parallelEndNode],
[afterParallelId, afterParallelNode],
])
const dag = createMockDAG(nodes)
const edgeManager = new EdgeManager(dag)
const readyAfterParallelContinue = edgeManager.processOutgoingEdges(parallelEndNode, {
selectedRoute: 'parallel_continue',
})
expect(readyAfterParallelContinue).toContain(parallelStartId)
expect(readyAfterParallelContinue).not.toContain(afterParallelId)
})
it('should handle condition with null selectedOption inside loop (dead-end branch)', () => {
// When a condition selects a branch with no outgoing connection (dead-end),
// selectedOption is null - cascade deactivation should make sentinel_end ready
@@ -1832,20 +2053,25 @@ describe('EdgeManager', () => {
const func2Id = 'func2'
const sentinelStartNode = createMockNode(sentinelStartId, [{ target: func1Id }])
sentinelStartNode.metadata = { isSentinel: true, sentinelType: 'start', loopId }
sentinelStartNode.metadata = {
isSentinel: true,
sentinelType: 'start',
subflowId: loopId,
subflowType: 'loop',
}
const func1Node = createMockNode(func1Id, [{ target: conditionId }], [sentinelStartId])
func1Node.metadata = { loopId, isLoopNode: true }
func1Node.metadata = { subflowId: loopId, subflowType: 'loop', isLoopNode: true }
const conditionNode = createMockNode(
conditionId,
[{ target: func2Id, sourceHandle: 'condition-if' }],
[func1Id]
)
conditionNode.metadata = { loopId, isLoopNode: true }
conditionNode.metadata = { subflowId: loopId, subflowType: 'loop', isLoopNode: true }
const func2Node = createMockNode(func2Id, [{ target: sentinelEndId }], [conditionId])
func2Node.metadata = { loopId, isLoopNode: true }
func2Node.metadata = { subflowId: loopId, subflowType: 'loop', isLoopNode: true }
const sentinelEndNode = createMockNode(
sentinelEndId,
@@ -1855,7 +2081,12 @@ describe('EdgeManager', () => {
],
[func2Id]
)
sentinelEndNode.metadata = { isSentinel: true, sentinelType: 'end', loopId }
sentinelEndNode.metadata = {
isSentinel: true,
sentinelType: 'end',
subflowId: loopId,
subflowType: 'loop',
}
const afterLoopNode = createMockNode('after-loop', [], [sentinelEndId])
@@ -1963,31 +2194,41 @@ describe('EdgeManager', () => {
const funcId = 'func'
const sentinelStartNode = createMockNode(sentinelStartId, [{ target: condition1Id }])
sentinelStartNode.metadata = { isSentinel: true, sentinelType: 'start', loopId }
sentinelStartNode.metadata = {
isSentinel: true,
sentinelType: 'start',
subflowId: loopId,
subflowType: 'loop',
}
const condition1Node = createMockNode(
condition1Id,
[{ target: condition2Id, sourceHandle: 'condition-if' }],
[sentinelStartId]
)
condition1Node.metadata = { loopId, isLoopNode: true }
condition1Node.metadata = { subflowId: loopId, subflowType: 'loop', isLoopNode: true }
const condition2Node = createMockNode(
condition2Id,
[{ target: funcId, sourceHandle: 'condition-if' }],
[condition1Id]
)
condition2Node.metadata = { loopId, isLoopNode: true }
condition2Node.metadata = { subflowId: loopId, subflowType: 'loop', isLoopNode: true }
const funcNode = createMockNode(funcId, [{ target: sentinelEndId }], [condition2Id])
funcNode.metadata = { loopId, isLoopNode: true }
funcNode.metadata = { subflowId: loopId, subflowType: 'loop', isLoopNode: true }
const sentinelEndNode = createMockNode(
sentinelEndId,
[{ target: sentinelStartId, sourceHandle: 'loop_continue' }],
[funcId]
)
sentinelEndNode.metadata = { isSentinel: true, sentinelType: 'end', loopId }
sentinelEndNode.metadata = {
isSentinel: true,
sentinelType: 'end',
subflowId: loopId,
subflowType: 'loop',
}
const nodes = new Map<string, DAGNode>([
[sentinelStartId, sentinelStartNode],
@@ -2036,34 +2277,62 @@ describe('EdgeManager', () => {
const innerEndId = 'inner-end'
const outerStartNode = createMockNode(outerStartId, [{ target: conditionId }])
outerStartNode.metadata = { isSentinel: true, sentinelType: 'start', loopId: outerLoopId }
outerStartNode.metadata = {
isSentinel: true,
sentinelType: 'start',
subflowId: outerLoopId,
subflowType: 'loop',
}
const conditionNode = createMockNode(
conditionId,
[{ target: innerStartId, sourceHandle: 'condition-if' }],
[outerStartId]
)
conditionNode.metadata = { loopId: outerLoopId, isLoopNode: true }
conditionNode.metadata = {
subflowId: outerLoopId,
subflowType: 'loop',
isLoopNode: true,
}
const innerStartNode = createMockNode(innerStartId, [{ target: innerBodyId }], [conditionId])
innerStartNode.metadata = { isSentinel: true, sentinelType: 'start', loopId: innerLoopId }
innerStartNode.metadata = {
isSentinel: true,
sentinelType: 'start',
subflowId: innerLoopId,
subflowType: 'loop',
}
const innerBodyNode = createMockNode(innerBodyId, [{ target: innerEndId }], [innerStartId])
innerBodyNode.metadata = { loopId: innerLoopId, isLoopNode: true }
innerBodyNode.metadata = {
subflowId: innerLoopId,
subflowType: 'loop',
isLoopNode: true,
}
const innerEndNode = createMockNode(
innerEndId,
[{ target: outerEndId, sourceHandle: 'loop_exit' }],
[innerBodyId]
)
innerEndNode.metadata = { isSentinel: true, sentinelType: 'end', loopId: innerLoopId }
innerEndNode.metadata = {
isSentinel: true,
sentinelType: 'end',
subflowId: innerLoopId,
subflowType: 'loop',
}
const outerEndNode = createMockNode(
outerEndId,
[{ target: outerStartId, sourceHandle: 'loop_continue' }],
[innerEndId]
)
outerEndNode.metadata = { isSentinel: true, sentinelType: 'end', loopId: outerLoopId }
outerEndNode.metadata = {
isSentinel: true,
sentinelType: 'end',
subflowId: outerLoopId,
subflowType: 'loop',
}
const nodes = new Map<string, DAGNode>([
[outerStartId, outerStartNode],
+87 -30
View File
@@ -1,5 +1,5 @@
import { createLogger } from '@sim/logger'
import { EDGE } from '@/executor/constants'
import { CONTROL_BACK_EDGE_HANDLES, EDGE, SUBFLOW_CONTROL_EDGE_HANDLES } from '@/executor/constants'
import type { DAG, DAGNode } from '@/executor/dag/builder'
import type { DAGEdge } from '@/executor/dag/types'
import type { NormalizedBlockOutput } from '@/executor/types'
@@ -27,7 +27,7 @@ export class EdgeManager {
}
if (!this.shouldActivateEdge(edge, output)) {
if (!this.isLoopEdge(edge.sourceHandle)) {
if (!this.isSubflowControlEdge(edge.sourceHandle)) {
edgesToDeactivate.push({ target: edge.target, handle: edge.sourceHandle })
}
continue
@@ -126,6 +126,25 @@ export class EdgeManager {
this.nodesWithActivatedEdge.clear()
}
getDeactivatedEdges(): string[] {
return Array.from(this.deactivatedEdges)
}
getNodesWithActivatedEdge(): string[] {
return Array.from(this.nodesWithActivatedEdge)
}
restoreDeactivatedEdges(edgeKeys?: string[], activatedNodeIds?: string[]): void {
this.deactivatedEdges = new Set(
(edgeKeys ?? []).map((edgeKey) => this.normalizeSerializedEdgeKey(edgeKey))
)
this.nodesWithActivatedEdge = new Set(activatedNodeIds ?? [])
}
markNodeWithActivatedEdge(nodeId: string): void {
this.nodesWithActivatedEdge.add(nodeId)
}
/**
* Clear deactivated edges for a set of nodes (used when restoring loop state for next iteration).
*
@@ -134,15 +153,17 @@ export class EdgeManager {
* remain deactivated — otherwise `countActiveIncomingEdges` would count a source that will never
* fire again, stalling the loop on its next iteration.
*
* Edge-key format is `${sourceId}-${targetId}-${handle}`, so `startsWith("${nodeId}-")` uniquely
* matches "node is source". An `includes("-${nodeId}-")` check would also match "node is target"
* and is unsafe for the reset semantics.
* Deactivated edge keys encode the source separately so node IDs with shared prefixes
* cannot clear each other's deactivated edges.
*/
clearDeactivatedEdgesForNodes(nodeIds: Set<string>): void {
const edgesToRemove: string[] = []
for (const edgeKey of this.deactivatedEdges) {
const sourceId = this.parseEdgeKey(edgeKey)?.sourceId
if (!sourceId) continue
for (const nodeId of nodeIds) {
if (edgeKey.startsWith(`${nodeId}-`)) {
if (sourceId === nodeId) {
edgesToRemove.push(edgeKey)
break
}
@@ -170,37 +191,31 @@ export class EdgeManager {
const sentinel = this.dag.nodes.get(sentinelId)
if (!sentinel?.metadata.isSentinel) return false
const sourceLoopId = sourceNode.metadata.loopId
const sourceParallelId = sourceNode.metadata.parallelId
const sentinelLoopId = sentinel.metadata.loopId
const sentinelParallelId = sentinel.metadata.parallelId
const sourceSubflowType = sourceNode.metadata.subflowType
const sentinelSubflowType = sentinel.metadata.subflowType
const sourceSubflowId = sourceNode.metadata.subflowId
const sentinelSubflowId = sentinel.metadata.subflowId
if (sourceLoopId && sentinelLoopId && sourceLoopId === sentinelLoopId) return true
if (sourceParallelId && sentinelParallelId && sourceParallelId === sentinelParallelId)
if (
sourceSubflowType &&
sentinelSubflowType &&
sourceSubflowType === sentinelSubflowType &&
sourceSubflowId &&
sentinelSubflowId &&
sourceSubflowId === sentinelSubflowId
) {
return true
}
return false
}
private isLoopEdge(handle?: string): boolean {
return (
handle === EDGE.LOOP_CONTINUE ||
handle === EDGE.LOOP_CONTINUE_ALT ||
handle === EDGE.LOOP_EXIT
)
}
private isControlEdge(handle?: string): boolean {
return (
handle === EDGE.LOOP_CONTINUE ||
handle === EDGE.LOOP_CONTINUE_ALT ||
handle === EDGE.LOOP_EXIT ||
handle === EDGE.PARALLEL_EXIT
)
private isSubflowControlEdge(handle?: string): boolean {
return handle !== undefined && SUBFLOW_CONTROL_EDGE_HANDLES.has(handle)
}
private isBackwardsEdge(sourceHandle?: string): boolean {
return sourceHandle === EDGE.LOOP_CONTINUE || sourceHandle === EDGE.LOOP_CONTINUE_ALT
return sourceHandle !== undefined && CONTROL_BACK_EDGE_HANDLES.has(sourceHandle)
}
private isTerminalControlNode(nodeId: string): boolean {
@@ -208,7 +223,7 @@ export class EdgeManager {
if (!node || node.outgoingEdges.size === 0) return false
for (const [, edge] of node.outgoingEdges) {
if (!this.isControlEdge(edge.sourceHandle)) {
if (!this.isSubflowControlEdge(edge.sourceHandle)) {
return false
}
}
@@ -231,6 +246,10 @@ export class EdgeManager {
}
if (output.selectedRoute === EDGE.PARALLEL_CONTINUE) {
return handle === EDGE.PARALLEL_CONTINUE
}
if (this.isSubflowControlEdge(handle)) {
return false
}
@@ -350,6 +369,44 @@ export class EdgeManager {
}
private createEdgeKey(sourceId: string, targetId: string, sourceHandle?: string): string {
return `${sourceId}-${targetId}-${sourceHandle ?? EDGE.DEFAULT}`
return JSON.stringify([sourceId, targetId, sourceHandle ?? EDGE.DEFAULT])
}
private parseEdgeKey(
edgeKey: string
): { sourceId: string; targetId: string; handle: string } | null {
let parsed: unknown
try {
parsed = JSON.parse(edgeKey)
} catch {
return null
}
if (
Array.isArray(parsed) &&
parsed.length === 3 &&
typeof parsed[0] === 'string' &&
typeof parsed[1] === 'string' &&
typeof parsed[2] === 'string'
) {
return { sourceId: parsed[0], targetId: parsed[1], handle: parsed[2] }
}
return null
}
private normalizeSerializedEdgeKey(edgeKey: string): string {
if (this.parseEdgeKey(edgeKey)) {
return edgeKey
}
for (const [sourceId, sourceNode] of this.dag.nodes) {
for (const [, edge] of sourceNode.outgoingEdges) {
const legacyKey = `${sourceId}-${edge.target}-${edge.sourceHandle ?? EDGE.DEFAULT}`
if (legacyKey === edgeKey) {
return this.createEdgeKey(sourceId, edge.target, edge.sourceHandle)
}
}
}
return edgeKey
}
}
+129 -10
View File
@@ -28,6 +28,7 @@ vi.mock('@/lib/execution/cancellation', () => ({
}))
import { isExecutionCancelled, isRedisCancellationEnabled } from '@/lib/execution/cancellation'
import { EDGE } from '@/executor/constants'
import type { DAG, DAGNode } from '@/executor/dag/builder'
import type { EdgeManager } from '@/executor/execution/edge-manager'
import type { NodeExecutionOrchestrator } from '@/executor/orchestrators/node'
@@ -108,6 +109,9 @@ function createMockEdgeManager(
restoreIncomingEdge: vi.fn(),
clearDeactivatedEdges: vi.fn(),
clearDeactivatedEdgesForNodes: vi.fn(),
getDeactivatedEdges: vi.fn(() => []),
getNodesWithActivatedEdge: vi.fn(() => []),
markNodeWithActivatedEdge: vi.fn(),
} as unknown as MockEdgeManager
}
@@ -150,7 +154,9 @@ describe('ExecutionEngine', () => {
endNode.incomingEdges.add('start')
const dag = createMockDAG([startNode, endNode])
const context = createMockContext()
const context = createMockContext({
decisions: { router: new Map(), condition: new Map() },
})
const edgeManager = createMockEdgeManager((node) => {
if (node.id === 'start') return ['end']
return []
@@ -167,7 +173,9 @@ describe('ExecutionEngine', () => {
it('should mark execution as successful when completed without cancellation', async () => {
const startNode = createMockNode('start', 'starter')
const dag = createMockDAG([startNode])
const context = createMockContext()
const context = createMockContext({
decisions: { router: new Map(), condition: new Map() },
})
const edgeManager = createMockEdgeManager()
const nodeOrchestrator = createMockNodeOrchestrator()
@@ -193,12 +201,36 @@ describe('ExecutionEngine', () => {
const nodeOrchestrator = createMockNodeOrchestrator()
const engine = new ExecutionEngine(context, dag, edgeManager, nodeOrchestrator)
const result = await engine.run()
const result = await engine.run('hitl')
expect(result.success).toBe(true)
expect(nodeOrchestrator.executionCount).toBe(0)
})
it('marks resumed pause edge targets as activated before readiness checks', async () => {
const targetNode = createMockNode('join', 'function')
targetNode.incomingEdges.add('pause-block')
targetNode.incomingEdges.add('condition-block')
const dag = createMockDAG([targetNode])
const context = createMockContext({
metadata: {
executionId: 'test-execution',
startTime: new Date().toISOString(),
pendingBlocks: [],
remainingEdges: [{ source: 'pause-block', target: 'join' }],
} as any,
})
const edgeManager = createMockEdgeManager(() => [])
vi.mocked(edgeManager.isNodeReady).mockReturnValue(false)
const nodeOrchestrator = createMockNodeOrchestrator()
const engine = new ExecutionEngine(context, dag, edgeManager, nodeOrchestrator)
await engine.run()
expect(edgeManager.markNodeWithActivatedEdge).toHaveBeenCalledWith('join')
expect(nodeOrchestrator.executeNode).not.toHaveBeenCalled()
})
it('should execute all nodes in a multi-node workflow', async () => {
const nodes = [
createMockNode('start', 'starter'),
@@ -227,6 +259,73 @@ describe('ExecutionEngine', () => {
expect(result.success).toBe(true)
expect(nodeOrchestrator.executionCount).toBe(4)
})
it('records paused block completion before returning paused result', async () => {
const node = createMockNode('hitl', 'function')
const dag = createMockDAG([node])
const context = createMockContext({
decisions: { router: new Map(), condition: new Map() },
})
const edgeManager = createMockEdgeManager()
const nodeOrchestrator = createMockNodeOrchestrator()
const pauseOutput = {
response: { status: 'paused' },
_pauseMetadata: {
contextId: 'pause-1',
blockId: 'hitl',
response: { status: 'paused' },
timestamp: new Date().toISOString(),
pauseKind: 'hitl',
},
}
vi.mocked(nodeOrchestrator.executeNode).mockResolvedValue({
nodeId: 'hitl',
output: pauseOutput,
isFinalOutput: false,
})
const engine = new ExecutionEngine(context, dag, edgeManager, nodeOrchestrator)
const result = await engine.run('hitl')
expect(result.status).toBe('paused')
expect(nodeOrchestrator.handleNodeCompletion).toHaveBeenCalledWith(
context,
'hitl',
pauseOutput
)
})
it('does not stop run-until execution on parallel batch continuation', async () => {
const parallelEnd = createMockNode('parallel-end', 'parallel')
const nextNode = createMockNode('next', 'function')
parallelEnd.outgoingEdges.set('continue', {
target: 'next',
sourceHandle: EDGE.PARALLEL_CONTINUE,
})
nextNode.incomingEdges.add('parallel-end')
const dag = createMockDAG([parallelEnd, nextNode])
const context = createMockContext({
stopAfterBlockId: 'parallel-end',
decisions: { router: new Map(), condition: new Map() },
})
const edgeManager = createMockEdgeManager((node) =>
node.id === 'parallel-end' ? ['next'] : []
)
const nodeOrchestrator = createMockNodeOrchestrator()
vi.mocked(nodeOrchestrator.executeNode).mockImplementation(async (_ctx, nodeId) => ({
nodeId,
output:
nodeId === 'parallel-end'
? { selectedRoute: EDGE.PARALLEL_CONTINUE }
: { result: 'done' },
isFinalOutput: nodeId === 'next',
}))
const engine = new ExecutionEngine(context, dag, edgeManager, nodeOrchestrator)
await engine.run('parallel-end')
expect(nodeOrchestrator.executeNode).toHaveBeenCalledWith(context, 'next')
})
})
describe('Cancellation via AbortSignal', () => {
@@ -301,7 +400,7 @@ describe('ExecutionEngine', () => {
const duration = Date.now() - startTime
expect(result.status).toBe('cancelled')
expect(duration).toBeLessThan(100)
expect(duration).toBeLessThan(200)
})
it('should return cancelled status even if error thrown during cancellation', async () => {
@@ -473,13 +572,23 @@ describe('ExecutionEngine', () => {
const abortController = new AbortController()
const loopStartNode = createMockNode('loop-start', 'loop_sentinel')
loopStartNode.metadata = { isSentinel: true, sentinelType: 'start', loopId: 'loop1' }
loopStartNode.metadata = {
isSentinel: true,
sentinelType: 'start',
subflowId: 'loop1',
subflowType: 'loop',
}
const loopBodyNode = createMockNode('loop-body', 'function')
loopBodyNode.metadata = { isLoopNode: true, loopId: 'loop1' }
loopBodyNode.metadata = { isLoopNode: true, subflowId: 'loop1', subflowType: 'loop' }
const loopEndNode = createMockNode('loop-end', 'loop_sentinel')
loopEndNode.metadata = { isSentinel: true, sentinelType: 'end', loopId: 'loop1' }
loopEndNode.metadata = {
isSentinel: true,
sentinelType: 'end',
subflowId: 'loop1',
subflowType: 'loop',
}
loopStartNode.outgoingEdges.set('edge1', { target: 'loop-body' })
loopBodyNode.outgoingEdges.set('edge2', { target: 'loop-end' })
@@ -953,13 +1062,23 @@ describe('ExecutionEngine', () => {
it('should stop loop iteration when error occurs in loop body', async () => {
const loopStartNode = createMockNode('loop-start', 'loop_sentinel')
loopStartNode.metadata = { isSentinel: true, sentinelType: 'start', loopId: 'loop1' }
loopStartNode.metadata = {
isSentinel: true,
sentinelType: 'start',
subflowId: 'loop1',
subflowType: 'loop',
}
const loopBodyNode = createMockNode('loop-body', 'function')
loopBodyNode.metadata = { isLoopNode: true, loopId: 'loop1' }
loopBodyNode.metadata = { isLoopNode: true, subflowId: 'loop1', subflowType: 'loop' }
const loopEndNode = createMockNode('loop-end', 'loop_sentinel')
loopEndNode.metadata = { isSentinel: true, sentinelType: 'end', loopId: 'loop1' }
loopEndNode.metadata = {
isSentinel: true,
sentinelType: 'end',
subflowId: 'loop1',
subflowType: 'loop',
}
const afterLoopNode = createMockNode('after-loop', 'function')
+12 -12
View File
@@ -5,7 +5,7 @@ import {
isExecutionCancelled,
isRedisCancellationEnabled,
} from '@/lib/execution/cancellation'
import { BlockType } from '@/executor/constants'
import { BlockType, EDGE } from '@/executor/constants'
import type { DAG } from '@/executor/dag/builder'
import type { EdgeManager } from '@/executor/execution/edge-manager'
import { serializePauseSnapshot } from '@/executor/execution/snapshot-serializer'
@@ -290,6 +290,9 @@ export class ExecutionEngine {
if (targetNode) {
const hadEdge = targetNode.incomingEdges.has(edge.source)
targetNode.incomingEdges.delete(edge.source)
if (hadEdge) {
this.edgeManager.markNodeWithActivatedEdge(targetNode.id)
}
if (this.edgeManager.isNodeReady(targetNode)) {
this.execLogger.info('Node became ready after edge removal', { nodeId: targetNode.id })
@@ -399,6 +402,8 @@ export class ExecutionEngine {
}
if (output._pauseMetadata) {
await this.nodeOrchestrator.handleNodeCompletion(this.context, nodeId, output)
const pauseMetadata = output._pauseMetadata
this.pausedBlocks.set(pauseMetadata.contextId, pauseMetadata)
this.context.metadata.status = 'paused'
@@ -426,8 +431,9 @@ export class ExecutionEngine {
if (this.context.stopAfterBlockId === nodeId) {
// For loop/parallel sentinels, only stop if the subflow has fully exited (all iterations done)
// shouldContinue: true means more iterations, shouldExit: true means loop is done
const shouldContinueLoop = output.shouldContinue === true
if (!shouldContinueLoop) {
const shouldContinue =
output.shouldContinue === true || output.selectedRoute === EDGE.PARALLEL_CONTINUE
if (!shouldContinue) {
this.execLogger.info('Stopping execution after target block', { nodeId })
this.stoppedEarlyFlag = true
return
@@ -437,13 +443,6 @@ export class ExecutionEngine {
const readyNodes = this.edgeManager.processOutgoingEdges(node, output, false)
this.addMultipleToQueue(readyNodes)
if (this.context.pendingDynamicNodes && this.context.pendingDynamicNodes.length > 0) {
const dynamicNodes = this.context.pendingDynamicNodes
this.context.pendingDynamicNodes = []
this.execLogger.info('Adding dynamically expanded parallel nodes', { dynamicNodes })
this.addMultipleToQueue(dynamicNodes)
}
}
private buildPausedResult(startTime: number): ExecutionResult {
@@ -452,7 +451,7 @@ export class ExecutionEngine {
this.context.metadata.duration = endTime - startTime
this.context.metadata.status = 'paused'
const snapshotSeed = serializePauseSnapshot(this.context, [], this.dag)
const snapshotSeed = serializePauseSnapshot(this.context, [], this.dag, this.edgeManager)
const pausePoints: PausePoint[] = Array.from(this.pausedBlocks.values()).map((pause) => ({
contextId: pause.contextId,
blockId: pause.blockId,
@@ -484,7 +483,8 @@ export class ExecutionEngine {
}): SerializableExecutionState | undefined {
try {
const serializedSnapshot =
snapshotSeed?.snapshot ?? serializePauseSnapshot(this.context, [], this.dag).snapshot
snapshotSeed?.snapshot ??
serializePauseSnapshot(this.context, [], this.dag, this.edgeManager).snapshot
const parsedSnapshot = JSON.parse(serializedSnapshot) as {
state?: SerializableExecutionState
}
@@ -0,0 +1,332 @@
/**
* @vitest-environment node
*/
import { describe, expect, it, vi } from 'vitest'
import { BlockType } from '@/executor/constants'
import { DAGBuilder } from '@/executor/dag/builder'
import { DAGExecutor } from '@/executor/execution/executor'
import type { SerializableExecutionState } from '@/executor/execution/types'
import type { ExecutionContext, ExecutionResult } from '@/executor/types'
import { buildSentinelStartId } from '@/executor/utils/subflow-utils'
import type { SerializedBlock, SerializedWorkflow } from '@/serializer/types'
function createExecutor(): DAGExecutor {
return new DAGExecutor({
workflow: {
version: '1',
blocks: [],
connections: [],
},
})
}
function createBlock(id: string, metadataId: string): SerializedBlock {
return {
id,
position: { x: 0, y: 0 },
config: { tool: 'noop', params: {} },
inputs: {},
outputs: {},
metadata: { id: metadataId, name: id },
enabled: true,
}
}
describe('DAGExecutor restored cloned subflow registration', () => {
it('registers restored cloned subflows under their parent parallel branch', () => {
const executor = createExecutor() as unknown as {
registerRestoredClonedSubflows: (
parentMap: Map<
string,
{ parentId: string; parentType: 'loop' | 'parallel'; branchIndex?: number }
>,
clonedSubflows: Array<{
originalId: string
clonedId: string
outerBranchIndex: number
parentParallelId: string
}>
) => void
}
const parentMap = new Map<
string,
{ parentId: string; parentType: 'loop' | 'parallel'; branchIndex?: number }
>([['nested-loop', { parentId: 'parent-parallel', parentType: 'parallel' }]])
executor.registerRestoredClonedSubflows(parentMap, [
{
originalId: 'nested-loop',
clonedId: 'nested-loop__obranch-2',
outerBranchIndex: 2,
parentParallelId: 'parent-parallel',
},
])
expect(parentMap.get('nested-loop__obranch-2')).toEqual({
parentId: 'parent-parallel',
parentType: 'parallel',
branchIndex: 2,
})
})
it('preserves cloned nested parent relationships within the same restored branch', () => {
const executor = createExecutor() as unknown as {
registerRestoredClonedSubflows: (
parentMap: Map<
string,
{ parentId: string; parentType: 'loop' | 'parallel'; branchIndex?: number }
>,
clonedSubflows: Array<{
originalId: string
clonedId: string
outerBranchIndex: number
parentParallelId: string
}>
) => void
}
const parentMap = new Map<
string,
{ parentId: string; parentType: 'loop' | 'parallel'; branchIndex?: number }
>([
['middle-loop', { parentId: 'parent-parallel', parentType: 'parallel' }],
['inner-parallel', { parentId: 'middle-loop', parentType: 'loop' }],
])
executor.registerRestoredClonedSubflows(parentMap, [
{
originalId: 'middle-loop',
clonedId: 'middle-loop__obranch-2',
outerBranchIndex: 2,
parentParallelId: 'parent-parallel',
},
{
originalId: 'inner-parallel',
clonedId: 'inner-parallel__obranch-2',
outerBranchIndex: 2,
parentParallelId: 'parent-parallel',
},
])
expect(parentMap.get('inner-parallel__obranch-2')).toEqual({
parentId: 'middle-loop__obranch-2',
parentType: 'loop',
branchIndex: 0,
})
})
it('restores snapshot parallel batches with later global branch indexes', () => {
const parallelId = 'parallel-1'
const loopId = 'loop-1'
const taskId = 'task-1'
const workflow: SerializedWorkflow = {
version: '1',
blocks: [
createBlock('start', BlockType.STARTER),
createBlock(parallelId, BlockType.PARALLEL),
createBlock(loopId, BlockType.LOOP),
createBlock(taskId, BlockType.FUNCTION),
],
connections: [
{ source: 'start', target: parallelId },
{ source: parallelId, target: loopId, sourceHandle: 'parallel-start-source' },
{ source: loopId, target: taskId, sourceHandle: 'loop-start-source' },
],
loops: {
[loopId]: {
id: loopId,
nodes: [taskId],
iterations: 1,
loopType: 'for',
},
},
parallels: {
[parallelId]: {
id: parallelId,
nodes: [loopId],
count: 4,
parallelType: 'count',
},
},
}
const dag = new DAGBuilder().build(workflow)
const executor = new DAGExecutor({ workflow }) as unknown as {
restoreSnapshotParallelBatches: (
dag: ReturnType<DAGBuilder['build']>,
snapshotState?: SerializableExecutionState
) => Array<{
originalId: string
clonedId: string
outerBranchIndex: number
parentParallelId: string
}>
}
const restoredClones = executor.restoreSnapshotParallelBatches(dag, {
blockStates: {},
executedBlocks: [],
blockLogs: [],
decisions: { router: {}, condition: {} },
completedLoops: [],
activeExecutionPath: [],
parallelExecutions: {
[parallelId]: {
currentBatchStart: 2,
currentBatchSize: 1,
totalBranches: 4,
items: ['zero', 'one', 'two', 'three'],
},
},
})
expect(dag.nodes.has(buildSentinelStartId(`${loopId}__obranch-2`))).toBe(true)
expect(restoredClones).toContainEqual(
expect.objectContaining({
originalId: loopId,
clonedId: `${loopId}__obranch-2`,
outerBranchIndex: 2,
parentParallelId: parallelId,
})
)
})
})
describe('DAGExecutor run-from-block snapshot metadata', () => {
it('preserves reachable large value and file keys in run-from-block metadata', async () => {
const reachableLargeValue = {
__simLargeValueRef: true,
version: 1,
id: 'lv_ABCDEF123456',
kind: 'object',
size: 1024,
key: 'execution/ws/wf/exec/large-value-lv_ABCDEF123456.json',
}
const unreachableLargeValue = {
__simLargeValueRef: true,
version: 1,
id: 'lv_ZYXWVU654321',
kind: 'object',
size: 1024,
key: 'execution/ws/wf/exec/large-value-lv_ZYXWVU654321.json',
}
const reachableFile = {
id: 'file-1',
name: 'reachable.txt',
url: '/api/files/serve/reachable',
size: 10,
type: 'text/plain',
key: 'execution/ws/wf/exec/reachable.txt',
}
const unreachableFile = {
id: 'file-2',
name: 'unreachable.txt',
url: '/api/files/serve/unreachable',
size: 10,
type: 'text/plain',
key: 'execution/ws/wf/exec/unreachable.txt',
}
const workflow: SerializedWorkflow = {
version: '1',
blocks: [
createBlock('start', BlockType.STARTER),
createBlock('producer', BlockType.FUNCTION),
createBlock('consumer', BlockType.FUNCTION),
createBlock('unreachable', BlockType.FUNCTION),
],
connections: [
{ source: 'start', target: 'producer' },
{ source: 'producer', target: 'consumer' },
],
loops: {},
parallels: {},
}
const executor = new DAGExecutor({
workflow,
contextExtensions: {
workspaceId: 'ws',
executionId: 'exec',
largeValueKeys: ['existing-large-key'],
fileKeys: ['existing-file-key'],
},
}) as unknown as DAGExecutor & {
buildExecutionPipeline: (context: ExecutionContext) => { run: () => Promise<ExecutionResult> }
}
const run = vi.fn(async (): Promise<ExecutionResult> => {
return {
success: true,
output: { ok: true },
metadata: {} as ExecutionResult['metadata'],
}
})
executor.buildExecutionPipeline = vi.fn(() => ({ run }))
const sourceSnapshot: SerializableExecutionState = {
blockStates: {
producer: { output: { reachableLargeValue, reachableFile } },
consumer: { output: { previous: true } },
unreachable: { output: { unreachableLargeValue, unreachableFile } },
},
executedBlocks: ['producer', 'consumer', 'unreachable'],
blockLogs: [],
decisions: { router: {}, condition: {} },
completedLoops: [],
activeExecutionPath: [],
}
const result = await executor.executeFromBlock('wf', 'consumer', sourceSnapshot)
expect(result.metadata?.largeValueKeys).toEqual(['existing-large-key', reachableLargeValue.key])
expect(result.metadata?.fileKeys).toEqual(['existing-file-key', reachableFile.key])
expect(result.metadata?.largeValueKeys).not.toContain(unreachableLargeValue.key)
expect(result.metadata?.fileKeys).not.toContain(unreachableFile.key)
})
it('preserves reachable stable branch aliases in run-from-block snapshots', async () => {
const workflow: SerializedWorkflow = {
version: '1',
blocks: [
createBlock('start', BlockType.STARTER),
createBlock('producer', BlockType.FUNCTION),
createBlock('consumer', BlockType.FUNCTION),
],
connections: [
{ source: 'start', target: 'producer' },
{ source: 'producer', target: 'consumer' },
],
loops: {},
parallels: {},
}
let capturedContext: ExecutionContext | undefined
const executor = new DAGExecutor({ workflow }) as unknown as DAGExecutor & {
buildExecutionPipeline: (context: ExecutionContext) => { run: () => Promise<ExecutionResult> }
}
executor.buildExecutionPipeline = vi.fn((context: ExecutionContext) => {
capturedContext = context
return {
run: async (): Promise<ExecutionResult> => ({
success: true,
output: { ok: true },
metadata: {},
}),
}
})
const sourceSnapshot: SerializableExecutionState = {
blockStates: {
producer: { output: { result: 'latest-local-batch' } },
'producer__obranch-0': { output: { result: 'global-branch-0' } },
'unreachable__obranch-0': { output: { result: 'unreachable' } },
consumer: { output: { previous: true } },
},
executedBlocks: ['producer', 'producer__obranch-0', 'unreachable__obranch-0', 'consumer'],
blockLogs: [],
decisions: { router: {}, condition: {} },
completedLoops: [],
activeExecutionPath: [],
}
await executor.executeFromBlock('wf', 'consumer', sourceSnapshot)
expect(capturedContext?.blockStates.get('producer__obranch-0')?.output).toEqual({
result: 'global-branch-0',
})
expect(capturedContext?.blockStates.has('unreachable__obranch-0')).toBe(false)
})
})
+100 -15
View File
@@ -21,7 +21,7 @@ import { LoopOrchestrator } from '@/executor/orchestrators/loop'
import { NodeExecutionOrchestrator } from '@/executor/orchestrators/node'
import { ParallelOrchestrator } from '@/executor/orchestrators/parallel'
import type { BlockState, ExecutionContext, ExecutionResult } from '@/executor/types'
import { ParallelExpander } from '@/executor/utils/parallel-expansion'
import { type ClonedSubflowInfo, ParallelExpander } from '@/executor/utils/parallel-expansion'
import {
computeExecutionSets,
type RunFromBlockContext,
@@ -36,6 +36,8 @@ import {
import {
extractLoopIdFromSentinel,
extractParallelIdFromSentinel,
stripCloneSuffixes,
stripOuterBranchSuffix,
} from '@/executor/utils/subflow-utils'
import { VariableResolver } from '@/executor/variables/resolver'
import { navigatePathAsync } from '@/executor/variables/resolvers/reference-async.server'
@@ -44,6 +46,10 @@ import type { SubflowType } from '@/stores/workflows/workflow/types'
const logger = createLogger('DAGExecutor')
interface RestoredClonedSubflowInfo extends ClonedSubflowInfo {
parentParallelId: string
}
export interface DAGExecutorOptions {
workflow: SerializedWorkflow
envVarValues?: Record<string, string>
@@ -83,10 +89,14 @@ export class DAGExecutor {
triggerBlockId,
savedIncomingEdges,
})
this.restoreSnapshotParallelBatches(dag, this.contextExtensions.snapshotState)
const restoredClonedSubflows = this.restoreSnapshotParallelBatches(
dag,
this.contextExtensions.snapshotState
)
this.restoreSavedIncomingEdges(dag, savedIncomingEdges)
const { context, state } = this.createExecutionContext(workflowId, triggerBlockId)
context.subflowParentMap = this.buildSubflowParentMap(dag)
this.registerRestoredClonedSubflows(context.subflowParentMap, restoredClonedSubflows)
const engine = this.buildExecutionPipeline(context, dag, state)
return await engine.run(triggerBlockId)
@@ -148,13 +158,31 @@ export class DAGExecutor {
// This preserves sibling branch outputs that dirty blocks may reference
const filteredBlockStates: Record<string, any> = {}
for (const [blockId, state] of Object.entries(sourceSnapshot.blockStates)) {
if (reachableUpstreamSet.has(blockId) || reachableContainerIds.has(blockId)) {
const aliasBaseId = stripOuterBranchSuffix(blockId)
const isReachableOuterBranchAlias =
aliasBaseId !== blockId &&
Array.from(reachableUpstreamSet).some(
(reachableId) => stripCloneSuffixes(reachableId) === aliasBaseId
)
if (
reachableUpstreamSet.has(blockId) ||
reachableContainerIds.has(blockId) ||
isReachableOuterBranchAlias
) {
filteredBlockStates[blockId] = state
}
}
const filteredExecutedBlocks = sourceSnapshot.executedBlocks.filter(
(id) => reachableUpstreamSet.has(id) || reachableContainerIds.has(id)
)
const filteredExecutedBlocks = sourceSnapshot.executedBlocks.filter((id) => {
const aliasBaseId = stripOuterBranchSuffix(id)
const isReachableOuterBranchAlias =
aliasBaseId !== id &&
Array.from(reachableUpstreamSet).some(
(reachableId) => stripCloneSuffixes(reachableId) === aliasBaseId
)
return (
reachableUpstreamSet.has(id) || reachableContainerIds.has(id) || isReachableOuterBranchAlias
)
})
// Filter loop/parallel executions to only include reachable containers
const filteredLoopExecutions: Record<string, any> = {}
@@ -227,7 +255,7 @@ export class DAGExecutor {
mergeFileKeys(context, filteredFileKeys)
context.subflowParentMap = this.buildSubflowParentMap(dag)
const engine = this.buildExecutionPipeline(context, dag, state)
const engine = this.buildExecutionPipeline(context, dag, state, filteredSnapshot)
const result = await engine.run()
if (result.metadata) {
result.metadata.largeValueKeys = context.largeValueKeys
@@ -250,10 +278,11 @@ export class DAGExecutor {
private restoreSnapshotParallelBatches(
dag: DAG,
snapshotState?: SerializableExecutionState
): void {
if (!snapshotState?.parallelExecutions) return
): RestoredClonedSubflowInfo[] {
if (!snapshotState?.parallelExecutions) return []
const expander = new ParallelExpander()
const clonedSubflows: RestoredClonedSubflowInfo[] = []
for (const [parallelId, scope] of Object.entries(snapshotState.parallelExecutions)) {
const currentBatchSize = Number(scope.currentBatchSize ?? 0)
if (!Number.isFinite(currentBatchSize) || currentBatchSize <= 0) continue
@@ -264,14 +293,62 @@ export class DAGExecutor {
? scope.items.slice(currentBatchStart, currentBatchStart + currentBatchSize)
: undefined
expander.expandParallel(dag, parallelId, currentBatchSize, items, {
const restoredBatch = expander.expandParallel(dag, parallelId, currentBatchSize, items, {
branchIndexOffset: currentBatchStart,
totalBranches,
})
clonedSubflows.push(
...restoredBatch.clonedSubflows.map((clone) => ({
...clone,
parentParallelId: parallelId,
}))
)
}
return clonedSubflows
}
private registerRestoredClonedSubflows(
parentMap: Map<string, { parentId: string; parentType: SubflowType; branchIndex?: number }>,
clonedSubflows: RestoredClonedSubflowInfo[]
): void {
const branchCloneMaps = new Map<string, Map<number, Map<string, string>>>()
for (const clone of clonedSubflows) {
let parallelBranchMaps = branchCloneMaps.get(clone.parentParallelId)
if (!parallelBranchMaps) {
parallelBranchMaps = new Map()
branchCloneMaps.set(clone.parentParallelId, parallelBranchMaps)
}
let cloneMap = parallelBranchMaps.get(clone.outerBranchIndex)
if (!cloneMap) {
cloneMap = new Map()
parallelBranchMaps.set(clone.outerBranchIndex, cloneMap)
}
cloneMap.set(clone.originalId, clone.clonedId)
}
for (const clone of clonedSubflows) {
const originalEntry = parentMap.get(clone.originalId)
const cloneMap = branchCloneMaps.get(clone.parentParallelId)?.get(clone.outerBranchIndex)
const clonedParentId = originalEntry ? cloneMap?.get(originalEntry.parentId) : undefined
parentMap.set(clone.clonedId, {
parentId: clonedParentId ?? clone.parentParallelId,
parentType: clonedParentId && originalEntry ? originalEntry.parentType : 'parallel',
branchIndex: clonedParentId ? 0 : clone.outerBranchIndex,
})
}
}
private buildExecutionPipeline(context: ExecutionContext, dag: DAG, state: ExecutionState) {
private buildExecutionPipeline(
context: ExecutionContext,
dag: DAG,
state: ExecutionState,
snapshotState = this.contextExtensions.snapshotState
) {
const resolver = new VariableResolver(this.workflow, this.workflowVariables, state, {
navigatePathAsync,
})
@@ -289,7 +366,12 @@ export class DAGExecutor {
dag,
state,
resolver,
this.contextExtensions
this.contextExtensions,
edgeManager
)
edgeManager.restoreDeactivatedEdges(
snapshotState?.deactivatedEdges,
snapshotState?.nodesWithActivatedEdge
)
const nodeOrchestrator = new NodeExecutionOrchestrator(
dag,
@@ -460,8 +542,11 @@ export class DAGExecutor {
*/
private buildSubflowParentMap(
dag: DAG
): Map<string, { parentId: string; parentType: SubflowType }> {
const parentMap = new Map<string, { parentId: string; parentType: SubflowType }>()
): Map<string, { parentId: string; parentType: SubflowType; branchIndex?: number }> {
const parentMap = new Map<
string,
{ parentId: string; parentType: SubflowType; branchIndex?: number }
>()
// Scan loop configs: children can be loops or parallels
for (const [loopId, config] of dag.loopConfigs) {
@@ -476,7 +561,7 @@ export class DAGExecutor {
for (const [parallelId, config] of dag.parallelConfigs) {
for (const nodeId of config.nodes ?? []) {
if (dag.parallelConfigs.has(nodeId) || dag.loopConfigs.has(nodeId)) {
parentMap.set(nodeId, { parentId: parallelId, parentType: 'parallel' })
parentMap.set(nodeId, { parentId: parallelId, parentType: 'parallel', branchIndex: 0 })
}
}
}
@@ -2,6 +2,8 @@
* @vitest-environment node
*/
import { describe, expect, it, vi } from 'vitest'
import type { DAG, DAGNode } from '@/executor/dag/builder'
import { EdgeManager } from '@/executor/execution/edge-manager'
import { serializePauseSnapshot } from '@/executor/execution/snapshot-serializer'
import type { ExecutionContext } from '@/executor/types'
@@ -68,6 +70,57 @@ describe('serializePauseSnapshot', () => {
})
})
it('serializes deactivated edge state for resume', () => {
const context = createContext()
const sourceNode = {
id: 'condition',
block: {} as DAGNode['block'],
incomingEdges: new Set<string>(),
outgoingEdges: new Map([['if-edge', { target: 'target', sourceHandle: 'condition-if' }]]),
metadata: {},
}
const targetNode = {
id: 'target',
block: {} as DAGNode['block'],
incomingEdges: new Set(['condition']),
outgoingEdges: new Map(),
metadata: {},
}
const activeSourceNode = {
id: 'active-source',
block: {} as DAGNode['block'],
incomingEdges: new Set<string>(),
outgoingEdges: new Map([['active-edge', { target: 'active-target' }]]),
metadata: {},
}
const activeTargetNode = {
id: 'active-target',
block: {} as DAGNode['block'],
incomingEdges: new Set(['active-source']),
outgoingEdges: new Map(),
metadata: {},
}
const dag: DAG = {
nodes: new Map([
[sourceNode.id, sourceNode],
[targetNode.id, targetNode],
[activeSourceNode.id, activeSourceNode],
[activeTargetNode.id, activeTargetNode],
]),
loopConfigs: new Map(),
parallelConfigs: new Map(),
}
const edgeManager = new EdgeManager(dag)
edgeManager.processOutgoingEdges(sourceNode, { selectedOption: 'else' })
edgeManager.processOutgoingEdges(activeSourceNode, { result: true })
const snapshot = serializePauseSnapshot(context, ['next-block'], dag, edgeManager)
const serialized = JSON.parse(snapshot.snapshot)
expect(serialized.state.deactivatedEdges).toHaveLength(1)
expect(serialized.state.nodesWithActivatedEdge).toEqual(['active-target'])
})
it('rejects oversized snapshot values without full JSON serialization', () => {
const stringifySpy = vi.spyOn(JSON, 'stringify').mockImplementation(() => {
throw new Error('full stringify should not be used for compactness checks')
@@ -1,5 +1,6 @@
import { LARGE_VALUE_THRESHOLD_BYTES } from '@/lib/execution/payloads/large-value-ref'
import type { DAG } from '@/executor/dag/builder'
import type { EdgeManager } from '@/executor/execution/edge-manager'
import { ExecutionSnapshot } from '@/executor/execution/snapshot'
import type { ExecutionMetadata, SerializableExecutionState } from '@/executor/execution/types'
import type { ExecutionContext, SerializedSnapshot } from '@/executor/types'
@@ -184,7 +185,8 @@ function serializeParallelExecutions(
export function serializePauseSnapshot(
context: ExecutionContext,
triggerBlockIds: string[],
dag?: DAG
dag?: DAG,
edgeManager?: EdgeManager
): SerializedSnapshot {
const metadataFromContext = context.metadata as ExecutionMetadata | undefined
let useDraftState: boolean
@@ -220,6 +222,8 @@ export function serializePauseSnapshot(
activeExecutionPath: Array.from(context.activeExecutionPath),
pendingQueue: triggerBlockIds,
dagIncomingEdges,
deactivatedEdges: edgeManager?.getDeactivatedEdges(),
nodesWithActivatedEdge: edgeManager?.getNodesWithActivatedEdge(),
}
assertSnapshotValueIsCompact(context.workflowVariables, 'workflow variables')
+77
View File
@@ -0,0 +1,77 @@
/**
* @vitest-environment node
*/
import { describe, expect, it } from 'vitest'
import { ExecutionState } from '@/executor/execution/state'
describe('ExecutionState', () => {
it('returns exact suffixed cached node outputs', () => {
const state = new ExecutionState()
state.setBlockOutput('producer₍1₎', { value: 'branch-1' })
state.setBlockOutput('producer_loop1', { value: 'loop-1' })
expect(state.getBlockOutput('producer₍1₎')).toEqual({ value: 'branch-1' })
expect(state.getBlockOutput('producer_loop1')).toEqual({ value: 'loop-1' })
})
it('prefers branch-local cloned outputs when resolving original block references', () => {
const state = new ExecutionState()
state.setBlockOutput('producer', { value: 'branch-0' })
state.setBlockOutput('producer__cloneaaa__obranch-2', { value: 'branch-2' })
expect(state.getBlockOutput('producer', 'consumer__clonebbb__obranch-2')).toEqual({
value: 'branch-2',
})
})
it('keeps cloned parallel branch references scoped to the same branch index', () => {
const state = new ExecutionState()
state.setBlockOutput('producer__cloneaaa__obranch-2₍0₎', { value: 'branch-0' })
state.setBlockOutput('producer__cloneaaa__obranch-2₍1₎', { value: 'branch-1' })
expect(state.getBlockOutput('producer', 'consumer__clonebbb__obranch-2₍1₎')).toEqual({
value: 'branch-1',
})
})
it('does not fall back to another branch when cloned scoped output is missing', () => {
const state = new ExecutionState()
state.setBlockOutput('producer₍0₎', { value: 'wrong-branch' })
expect(state.getBlockOutput('producer', 'consumer__clonebbb__obranch-2')).toBeUndefined()
})
it('resolves regular sibling outputs from the same parent parallel branch', () => {
const state = new ExecutionState()
state.setBlockOutput('producer₍2₎', { value: 'parent-branch-2' })
expect(state.getBlockOutput('producer', 'consumer__clonebbb__obranch-2₍0₎')).toEqual({
value: 'parent-branch-2',
})
})
it('does not fall back to direct branch-zero output from cloned nodes', () => {
const state = new ExecutionState()
state.setBlockOutput('producer', { value: 'branch-0' })
expect(state.getBlockOutput('producer', 'consumer__clonebbb__obranch-2')).toBeUndefined()
})
it('resolves branch-zero sibling output deterministically for unsuffixed nested branch nodes', () => {
const state = new ExecutionState()
state.setBlockOutput('producer₍1₎', { value: 'branch-1' })
state.setBlockOutput('producer₍0₎', { value: 'branch-0' })
expect(state.getBlockOutput('producer', 'nested-condition')).toEqual({ value: 'branch-0' })
})
it('prefers stable branch-zero aliases when later batches reuse local branch ids', () => {
const state = new ExecutionState()
state.setBlockOutput('producer__obranch-0', { value: 'global-branch-0' })
state.setBlockOutput('producer₍0₎', { value: 'later-batch-local-0' })
expect(state.getBlockOutput('producer', 'after-parallel')).toEqual({
value: 'global-branch-0',
})
})
})
+80 -11
View File
@@ -1,9 +1,25 @@
import type { BlockStateController } from '@/executor/execution/types'
import type { BlockState, NormalizedBlockOutput } from '@/executor/types'
import {
buildOuterBranchScopedId,
extractOuterBranchIndex,
stripCloneSuffixes,
} from '@/executor/utils/subflow-utils'
const BRANCH_SUFFIX_PATTERN = /₍\d+₎/u
const LOOP_SUFFIX_PATTERN = /_loop\d+/
function normalizeLookupId(id: string): string {
return id.replace(/₍\d+₎/gu, '').replace(/_loop\d+/g, '')
}
function extractBranchSuffix(id: string): string {
return id.match(BRANCH_SUFFIX_PATTERN)?.[0] ?? ''
}
function extractLoopSuffix(id: string): string {
return id.match(LOOP_SUFFIX_PATTERN)?.[0] ?? ''
}
export interface LoopScope {
iteration: number
currentIterationOutputs: Map<string, NormalizedBlockOutput>
@@ -14,6 +30,7 @@ export interface LoopScope {
condition?: string
loopType?: 'for' | 'forEach' | 'while' | 'doWhile'
skipFirstConditionCheck?: boolean
skippedAtStart?: boolean
/** Error message if loop validation failed (e.g., exceeded max iterations) */
validationError?: string
}
@@ -51,23 +68,40 @@ export class ExecutionState implements BlockStateController {
}
getBlockOutput(blockId: string, currentNodeId?: string): NormalizedBlockOutput | undefined {
const normalizedId = normalizeLookupId(blockId)
if (normalizedId !== blockId) {
return this.blockStates.get(blockId)?.output
}
if (currentNodeId) {
const scopedOutput = this.getScopedBlockOutput(blockId, currentNodeId)
if (scopedOutput !== undefined) {
return scopedOutput
}
if (extractOuterBranchIndex(currentNodeId) !== undefined) {
return undefined
}
}
const direct = this.blockStates.get(blockId)?.output
if (direct !== undefined) {
return direct
}
const normalizedId = normalizeLookupId(blockId)
if (normalizedId !== blockId) {
return undefined
}
if (currentNodeId && extractBranchSuffix(currentNodeId) === '') {
const stableBranchZeroOutput = this.blockStates.get(
buildOuterBranchScopedId(blockId, 0)
)?.output
if (stableBranchZeroOutput !== undefined) {
return stableBranchZeroOutput
}
if (currentNodeId) {
const currentSuffix = currentNodeId.replace(normalizedId, '').match(/₍\d+₎/g)?.[0] ?? ''
const loopSuffix = currentNodeId.match(/_loop\d+/)?.[0] ?? ''
const withSuffix = `${blockId}${currentSuffix}${loopSuffix}`
const suffixedOutput = this.blockStates.get(withSuffix)?.output
if (suffixedOutput !== undefined) {
return suffixedOutput
const branchZeroOutput = this.blockStates.get(
`${blockId}₍0₎${extractLoopSuffix(currentNodeId)}`
)?.output
if (branchZeroOutput !== undefined) {
return branchZeroOutput
}
}
@@ -80,6 +114,41 @@ export class ExecutionState implements BlockStateController {
return undefined
}
private getScopedBlockOutput(
blockId: string,
currentNodeId: string
): NormalizedBlockOutput | undefined {
const currentBranchSuffix = extractBranchSuffix(currentNodeId)
const loopSuffix = extractLoopSuffix(currentNodeId)
const currentOuterBranchIndex = extractOuterBranchIndex(currentNodeId)
if (currentOuterBranchIndex !== undefined) {
for (const [storedId, state] of this.blockStates.entries()) {
if (stripCloneSuffixes(storedId) !== blockId) continue
if (extractOuterBranchIndex(storedId) !== currentOuterBranchIndex) continue
if (extractBranchSuffix(storedId) !== currentBranchSuffix) continue
if (extractLoopSuffix(storedId) !== loopSuffix) continue
return state.output
}
const siblingBranchOutput = this.blockStates.get(
`${blockId}₍${currentOuterBranchIndex}₎`
)?.output
if (siblingBranchOutput !== undefined) {
return siblingBranchOutput
}
} else {
const withSuffix = `${blockId}${currentBranchSuffix}${loopSuffix}`
const suffixedOutput = this.blockStates.get(withSuffix)?.output
if (suffixedOutput !== undefined) {
return suffixedOutput
}
}
return undefined
}
setBlockOutput(blockId: string, output: NormalizedBlockOutput, executionTime = 0): void {
this.blockStates.set(blockId, { output, executed: true, executionTime })
this.executedBlocks.add(blockId)
+5 -1
View File
@@ -61,6 +61,8 @@ export interface SerializableExecutionState {
remainingEdges?: Edge[]
resumeTerminalNoop?: boolean
dagIncomingEdges?: Record<string, string[]>
deactivatedEdges?: string[]
nodesWithActivatedEdge?: string[]
completedPauseContexts?: string[]
}
@@ -99,9 +101,11 @@ export interface IterationContext {
export interface WorkflowNodeMetadata
extends Pick<
NodeMetadata,
'loopId' | 'parallelId' | 'branchIndex' | 'branchTotal' | 'originalBlockId' | 'isLoopNode'
'subflowType' | 'subflowId' | 'branchIndex' | 'branchTotal' | 'originalBlockId' | 'isLoopNode'
> {
nodeId: string
loopId?: string
parallelId?: string
executionOrder?: number
}
+5 -3
View File
@@ -1,10 +1,12 @@
import { PARALLEL } from '@/executor/constants'
import type { NodeMetadata } from '@/executor/dag/types'
import type { ExecutionContext, LoopPauseScope, ParallelPauseScope } from '@/executor/types'
interface NodeMetadataLike
extends Pick<NodeMetadata, 'loopId' | 'parallelId' | 'branchIndex' | 'branchTotal'> {
interface NodeMetadataLike {
nodeId: string
loopId?: string
parallelId?: string
branchIndex?: number
branchTotal?: number
}
export function generatePauseContextId(
+262 -8
View File
@@ -5,20 +5,57 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'
import { clearLargeValueCacheForTests } from '@/lib/execution/payloads/cache'
import { isLargeArrayManifest } from '@/lib/execution/payloads/large-array-manifest-metadata'
import { EDGE } from '@/executor/constants'
import type { DAG, DAGNode } from '@/executor/dag/builder'
import type { EdgeManager } from '@/executor/execution/edge-manager'
import type { BlockStateController } from '@/executor/execution/types'
import { LoopOrchestrator } from '@/executor/orchestrators/loop'
import type { ExecutionContext } from '@/executor/types'
const { mockUploadFile } = vi.hoisted(() => ({
const { mockExecuteInIsolatedVM, mockUploadFile } = vi.hoisted(() => ({
mockExecuteInIsolatedVM: vi.fn(),
mockUploadFile: vi.fn(),
}))
vi.mock('@/lib/execution/isolated-vm', () => ({
executeInIsolatedVM: mockExecuteInIsolatedVM,
}))
vi.mock('@/lib/uploads', () => ({
StorageService: {
uploadFile: mockUploadFile,
},
}))
function createContext(scope: Record<string, unknown>): ExecutionContext {
function createNode(id: string): DAGNode {
return {
id,
block: {
id,
position: { x: 0, y: 0 },
enabled: true,
metadata: { id: 'function', name: id },
config: { params: {} },
inputs: {},
outputs: {},
},
incomingEdges: new Set(),
outgoingEdges: new Map(),
metadata: {},
}
}
function createState(): BlockStateController {
return {
getBlockOutput: vi.fn(),
hasExecuted: vi.fn(() => false),
setBlockOutput: vi.fn(),
setBlockState: vi.fn(),
deleteBlockState: vi.fn(),
unmarkExecuted: vi.fn(),
}
}
function createContext(scope: Record<string, unknown> = {}, loopId = 'loop-1'): ExecutionContext {
return {
workflowId: 'workflow-1',
workspaceId: 'workspace-1',
@@ -33,27 +70,244 @@ function createContext(scope: Record<string, unknown>): ExecutionContext {
decisions: { router: new Map(), condition: new Map() },
completedLoops: new Set(),
activeExecutionPath: new Set(),
loopExecutions: new Map([['loop-1', scope as any]]),
loopExecutions: new Map([[loopId, scope as any]]),
} as ExecutionContext
}
function createOrchestrator(loopConfigs = new Map<string, any>()) {
const setBlockOutput = vi.fn()
const state = createState()
const orchestrator = new LoopOrchestrator(
{ loopConfigs, parallelConfigs: new Map(), nodes: new Map() } as any,
{ setBlockOutput, unmarkExecuted: vi.fn() } as any,
state,
{ resolveSingleReference: vi.fn() } as any
)
return { orchestrator, setBlockOutput }
return { orchestrator, setBlockOutput: vi.mocked(state.setBlockOutput) }
}
describe('LoopOrchestrator', () => {
beforeEach(() => {
vi.clearAllMocks()
clearLargeValueCacheForTests()
mockExecuteInIsolatedVM.mockResolvedValue({ result: true })
mockUploadFile.mockImplementation(async ({ customKey }) => ({ key: customKey }))
})
it('does not restore parallel_continue back edges for nested parallels', () => {
const loopId = 'loop-1'
const parallelId = 'parallel-1'
const loopStartId = `loop-${loopId}-sentinel-start`
const loopEndId = `loop-${loopId}-sentinel-end`
const parallelStartId = `parallel-${parallelId}-sentinel-start`
const parallelEndId = `parallel-${parallelId}-sentinel-end`
const loopStart = createNode(loopStartId)
const loopEnd = createNode(loopEndId)
const parallelStart = createNode(parallelStartId)
const parallelEnd = createNode(parallelEndId)
loopStart.outgoingEdges.set(`${loopStartId}->${parallelStartId}`, { target: parallelStartId })
loopStart.outgoingEdges.set(`${loopStartId}->${loopEndId}-exit`, {
target: loopEndId,
sourceHandle: EDGE.LOOP_EXIT,
})
parallelStart.outgoingEdges.set(`${parallelStartId}->${parallelEndId}-exit`, {
target: parallelEndId,
sourceHandle: EDGE.PARALLEL_EXIT,
})
parallelEnd.outgoingEdges.set(`${parallelEndId}->${parallelStartId}-continue`, {
target: parallelStartId,
sourceHandle: EDGE.PARALLEL_CONTINUE,
})
parallelEnd.outgoingEdges.set(`${parallelEndId}->${loopEndId}-exit`, {
target: loopEndId,
sourceHandle: EDGE.PARALLEL_EXIT,
})
const dag: DAG = {
nodes: new Map([
[loopStartId, loopStart],
[loopEndId, loopEnd],
[parallelStartId, parallelStart],
[parallelEndId, parallelEnd],
]),
loopConfigs: new Map([[loopId, { id: loopId, nodes: [parallelId], loopType: 'for' }]]),
parallelConfigs: new Map([
[parallelId, { id: parallelId, nodes: [], parallelType: 'count' }],
]),
}
const edgeManager = {
clearDeactivatedEdgesForNodes: vi.fn(),
} as unknown as EdgeManager
const orchestrator = new LoopOrchestrator(dag, createState(), null as any, {}, edgeManager)
orchestrator.restoreLoopEdges(loopId)
expect(parallelStart.incomingEdges.has(loopStartId)).toBe(true)
expect(parallelStart.incomingEdges.has(parallelEndId)).toBe(false)
expect(loopEnd.incomingEdges.has(loopStartId)).toBe(false)
expect(parallelEnd.incomingEdges.has(parallelStartId)).toBe(false)
expect(loopEnd.incomingEdges.has(parallelEndId)).toBe(true)
})
it('resolves forEach collections with the loop start sentinel scope', async () => {
const loopId = 'loop-1'
const dag: DAG = {
nodes: new Map(),
loopConfigs: new Map([
[
loopId,
{
id: loopId,
nodes: ['task-1'],
loopType: 'forEach',
forEachItems: '<Producer.items>',
},
],
]),
parallelConfigs: new Map(),
}
const resolver = {
resolveSingleReference: vi.fn().mockResolvedValue(['item-1']),
}
const orchestrator = new LoopOrchestrator(dag, createState(), resolver as any, {}, {
clearDeactivatedEdgesForNodes: vi.fn(),
} as unknown as EdgeManager)
const ctx = createContext()
const scope = await orchestrator.initializeLoopScope(ctx, loopId)
expect(resolver.resolveSingleReference).toHaveBeenCalledWith(
expect.any(Object),
'loop-loop-1-sentinel-start',
'<Producer.items>',
undefined,
{ allowLargeValueRefs: true }
)
expect(scope.maxIterations).toBe(1)
})
it('exits immediately when a loop was skipped at start', async () => {
const loopId = 'loop-1'
const state = createState()
const dag: DAG = {
nodes: new Map(),
loopConfigs: new Map([[loopId, { id: loopId, nodes: ['task-1'], loopType: 'while' }]]),
parallelConfigs: new Map(),
}
const resolver = {
resolveSingleReference: vi.fn().mockResolvedValue(1),
}
const orchestrator = new LoopOrchestrator(dag, state, resolver as any, {}, {
clearDeactivatedEdgesForNodes: vi.fn(),
} as unknown as EdgeManager)
const ctx = createContext(
{
iteration: 0,
currentIterationOutputs: new Map(),
allIterationOutputs: [],
loopType: 'while',
condition: '<loop.index> > 0',
skippedAtStart: true,
},
loopId
)
const result = await orchestrator.evaluateLoopContinuation(ctx, loopId)
expect(result).toMatchObject({
shouldContinue: false,
shouldExit: true,
selectedRoute: EDGE.LOOP_EXIT,
aggregatedResults: [],
})
expect(resolver.resolveSingleReference).not.toHaveBeenCalled()
expect(state.setBlockOutput).toHaveBeenCalledWith(loopId, { results: [] }, 0)
})
it('marks empty forEach loops as skipped at the initial condition check', async () => {
const { orchestrator, setBlockOutput } = createOrchestrator()
const scope = {
iteration: 0,
currentIterationOutputs: new Map(),
allIterationOutputs: [],
loopType: 'forEach',
items: [],
maxIterations: 0,
condition: '<loop.index> < 0',
} as { skippedAtStart?: boolean } & Record<string, unknown>
const ctx = createContext(scope)
const shouldExecute = await orchestrator.evaluateInitialCondition(ctx, 'loop-1')
expect(shouldExecute).toBe(false)
expect(scope.skippedAtStart).toBe(true)
expect(setBlockOutput).not.toHaveBeenCalled()
const result = await orchestrator.evaluateLoopContinuation(ctx, 'loop-1')
expect(result).toMatchObject({
shouldContinue: false,
shouldExit: true,
selectedRoute: EDGE.LOOP_EXIT,
aggregatedResults: [],
})
expect(scope.skippedAtStart).toBe(false)
expect(setBlockOutput).toHaveBeenCalledWith('loop-1', { results: [] }, 0)
})
it.each([
['for loop with zero iterations', { loopType: 'for', maxIterations: 0 }],
['while loop with no condition', { loopType: 'while' }],
])('marks %s as skipped at the initial condition check', async (_name, overrides) => {
const { orchestrator, setBlockOutput } = createOrchestrator()
const scope = {
iteration: 0,
currentIterationOutputs: new Map(),
allIterationOutputs: [],
...overrides,
} as { skippedAtStart?: boolean } & Record<string, unknown>
const ctx = createContext(scope)
const shouldExecute = await orchestrator.evaluateInitialCondition(ctx, 'loop-1')
expect(shouldExecute).toBe(false)
expect(scope.skippedAtStart).toBe(true)
expect(setBlockOutput).not.toHaveBeenCalled()
await orchestrator.evaluateLoopContinuation(ctx, 'loop-1')
expect(scope.skippedAtStart).toBe(false)
expect(setBlockOutput).toHaveBeenCalledWith('loop-1', { results: [] }, 0)
})
it('marks while loops with false initial conditions as skipped at start', async () => {
const state = createState()
const resolver = { resolveSingleReference: vi.fn().mockResolvedValue(false) }
mockExecuteInIsolatedVM.mockResolvedValueOnce({ result: false })
const orchestrator = new LoopOrchestrator(
{ loopConfigs: new Map(), parallelConfigs: new Map(), nodes: new Map() },
state,
resolver as any
)
const scope = {
iteration: 0,
currentIterationOutputs: new Map(),
allIterationOutputs: [],
loopType: 'while',
condition: '<condition.output>',
} as { skippedAtStart?: boolean } & Record<string, unknown>
const ctx = createContext(scope)
const shouldExecute = await orchestrator.evaluateInitialCondition(ctx, 'loop-1')
expect(shouldExecute).toBe(false)
expect(scope.skippedAtStart).toBe(true)
expect(state.setBlockOutput).not.toHaveBeenCalled()
expect(mockExecuteInIsolatedVM).toHaveBeenCalledWith(
expect.objectContaining({
code: 'return Boolean(false)',
})
)
})
it('exits doWhile loops when the configured iteration cap is reached', async () => {
const { orchestrator } = createOrchestrator()
const ctx = createContext({
@@ -89,7 +343,7 @@ describe('LoopOrchestrator', () => {
],
])
)
const ctx = createContext({})
const ctx = createContext()
const scope = await orchestrator.initializeLoopScope(ctx, 'loop-1')
@@ -111,7 +365,7 @@ describe('LoopOrchestrator', () => {
],
])
)
const ctx = createContext({})
const ctx = createContext()
const scope = await orchestrator.initializeLoopScope(ctx, 'loop-1')
+55 -16
View File
@@ -5,7 +5,13 @@ import { isExecutionCancelled, isRedisCancellationEnabled } from '@/lib/executio
import { executeInIsolatedVM } from '@/lib/execution/isolated-vm'
import { compactSubflowResults } from '@/lib/execution/payloads/serializer'
import { isLikelyReferenceSegment } from '@/lib/workflows/sanitization/references'
import { buildLoopIndexCondition, DEFAULTS, EDGE, PARALLEL } from '@/executor/constants'
import {
buildLoopIndexCondition,
CONTROL_BACK_EDGE_HANDLES,
DEFAULTS,
EDGE,
PARALLEL,
} from '@/executor/constants'
import type { DAG } from '@/executor/dag/builder'
import type { EdgeManager } from '@/executor/execution/edge-manager'
import type { LoopScope } from '@/executor/execution/state'
@@ -19,9 +25,10 @@ import {
buildParallelSentinelStartId,
buildSentinelEndId,
buildSentinelStartId,
emitEmptySubflowEvents,
emitSubflowSuccessEvents,
extractBaseBlockId,
extractLoopIdFromSentinel,
extractParallelIdFromSentinel,
} from '@/executor/utils/subflow-utils'
import { resolveArrayInputAsync } from '@/executor/utils/subflow-utils.server'
import type { VariableResolver } from '@/executor/variables/resolver'
@@ -134,7 +141,12 @@ export class LoopOrchestrator {
}
let items: any[]
try {
items = await resolveArrayInputAsync(ctx, loopConfig.forEachItems, this.resolver)
items = await resolveArrayInputAsync(
ctx,
loopConfig.forEachItems,
this.resolver,
buildSentinelStartId(loopId)
)
} catch (error) {
const errorMessage = `ForEach loop resolution failed: ${toError(error).message}`
logger.error(errorMessage, { loopId, forEachItems: loopConfig.forEachItems })
@@ -243,6 +255,11 @@ export class LoopOrchestrator {
return await this.createExitResult(ctx, loopId, scope)
}
if (scope.skippedAtStart) {
scope.skippedAtStart = false
return await this.createExitResult(ctx, loopId, scope)
}
const iterationResults: NormalizedBlockOutput[] = []
for (const blockOutput of scope.currentIterationOutputs.values()) {
iterationResults.push(blockOutput)
@@ -579,11 +596,10 @@ export class LoopOrchestrator {
for (const [, edge] of potentialSourceNode.outgoingEdges) {
if (edge.target === nodeId) {
const isBackwardEdge =
edge.sourceHandle === EDGE.LOOP_CONTINUE ||
edge.sourceHandle === EDGE.LOOP_CONTINUE_ALT
if (!isBackwardEdge) {
if (
!this.isSubflowStartExitBypassEdge(potentialSourceId, nodeId, edge.sourceHandle) &&
(edge.sourceHandle === undefined || !CONTROL_BACK_EDGE_HANDLES.has(edge.sourceHandle))
) {
nodeToRestore.incomingEdges.add(potentialSourceId)
}
}
@@ -592,6 +608,32 @@ export class LoopOrchestrator {
}
}
private isSubflowStartExitBypassEdge(
sourceId: string,
targetId: string,
sourceHandle?: string
): boolean {
if (sourceHandle === EDGE.LOOP_EXIT) {
const loopId = extractLoopIdFromSentinel(sourceId)
return (
!!loopId &&
sourceId === buildSentinelStartId(loopId) &&
targetId === buildSentinelEndId(loopId)
)
}
if (sourceHandle === EDGE.PARALLEL_EXIT) {
const parallelId = extractParallelIdFromSentinel(sourceId)
return (
!!parallelId &&
sourceId === buildParallelSentinelStartId(parallelId) &&
targetId === buildParallelSentinelEndId(parallelId)
)
}
return false
}
getLoopScope(ctx: ExecutionContext, loopId: string): LoopScope | undefined {
return ctx.loopExecutions?.get(loopId)
}
@@ -615,8 +657,7 @@ export class LoopOrchestrator {
if (scope.loopType === 'forEach') {
if (!scope.items || scope.items.length === 0) {
logger.info('ForEach loop has empty collection, skipping loop body', { loopId })
this.state.setBlockOutput(loopId, { results: [] }, DEFAULTS.EXECUTION_TIME)
await emitEmptySubflowEvents(ctx, loopId, 'loop', this.contextExtensions)
scope.skippedAtStart = true
return false
}
return true
@@ -625,8 +666,7 @@ export class LoopOrchestrator {
if (scope.loopType === 'for') {
if (scope.maxIterations === 0) {
logger.info('For loop has 0 iterations, skipping loop body', { loopId })
this.state.setBlockOutput(loopId, { results: [] }, DEFAULTS.EXECUTION_TIME)
await emitEmptySubflowEvents(ctx, loopId, 'loop', this.contextExtensions)
scope.skippedAtStart = true
return false
}
return true
@@ -639,8 +679,7 @@ export class LoopOrchestrator {
if (scope.loopType === 'while') {
if (!scope.condition) {
logger.warn('No condition defined for while loop', { loopId })
this.state.setBlockOutput(loopId, { results: [] }, DEFAULTS.EXECUTION_TIME)
await emitEmptySubflowEvents(ctx, loopId, 'loop', this.contextExtensions)
scope.skippedAtStart = true
return false
}
@@ -652,8 +691,8 @@ export class LoopOrchestrator {
})
if (!result) {
this.state.setBlockOutput(loopId, { results: [] }, DEFAULTS.EXECUTION_TIME)
await emitEmptySubflowEvents(ctx, loopId, 'loop', this.contextExtensions)
logger.info('While loop initial condition is false, skipping loop body', { loopId })
scope.skippedAtStart = true
}
return result
@@ -0,0 +1,464 @@
/**
* @vitest-environment node
*/
import { describe, expect, it, vi } from 'vitest'
import { EDGE } from '@/executor/constants'
import type { DAG, DAGNode } from '@/executor/dag/builder'
import type { BlockExecutor } from '@/executor/execution/block-executor'
import type { BlockStateController } from '@/executor/execution/types'
import type { LoopOrchestrator } from '@/executor/orchestrators/loop'
import { NodeExecutionOrchestrator } from '@/executor/orchestrators/node'
import type { ParallelOrchestrator } from '@/executor/orchestrators/parallel'
import type { ExecutionContext } from '@/executor/types'
function createContext(): ExecutionContext {
return {
workflowId: 'workflow-1',
workspaceId: 'workspace-1',
executionId: 'execution-1',
userId: 'user-1',
blockStates: new Map(),
executedBlocks: new Set(),
blockLogs: [],
metadata: { duration: 0 },
environmentVariables: {},
decisions: {
router: new Map(),
condition: new Map(),
},
completedLoops: new Set(),
activeExecutionPath: new Set(),
workflow: {
version: '1',
blocks: [],
connections: [],
loops: {},
parallels: {},
},
}
}
function createState(): BlockStateController {
return {
getBlockOutput: vi.fn(),
hasExecuted: vi.fn(() => false),
setBlockOutput: vi.fn(),
setBlockState: vi.fn(),
deleteBlockState: vi.fn(),
unmarkExecuted: vi.fn(),
}
}
function createSentinelNode(id: string, sentinelType: 'start' | 'end'): DAGNode {
return {
id,
block: {
id,
position: { x: 0, y: 0 },
enabled: true,
metadata: { id: 'parallel', name: id },
config: { params: {} },
inputs: {},
outputs: {},
},
incomingEdges: new Set(),
outgoingEdges: new Map(),
metadata: {
isSentinel: true,
sentinelType,
subflowId: 'parallel-1',
subflowType: 'parallel',
},
}
}
function createOrchestrator(
dag: DAG,
state: BlockStateController,
parallelOrchestrator: Partial<ParallelOrchestrator>,
loopOrchestratorOverrides: Partial<LoopOrchestrator> = {}
): NodeExecutionOrchestrator {
const blockExecutor = { execute: vi.fn() } as unknown as BlockExecutor
const loopOrchestrator = {
getLoopScope: vi.fn(),
initializeLoopScope: vi.fn(),
evaluateInitialCondition: vi.fn(),
evaluateLoopContinuation: vi.fn(),
clearLoopExecutionState: vi.fn(),
restoreLoopEdges: vi.fn(),
storeLoopNodeOutput: vi.fn(),
...loopOrchestratorOverrides,
} as unknown as LoopOrchestrator
return new NodeExecutionOrchestrator(
dag,
state,
blockExecutor,
loopOrchestrator,
parallelOrchestrator as ParallelOrchestrator
)
}
describe('NodeExecutionOrchestrator parallel sentinel batching', () => {
it('returns loop_exit from a loop start sentinel when the initial condition is false', async () => {
const startNode = {
...createSentinelNode('loop-loop-1-sentinel-start', 'start'),
metadata: {
isSentinel: true,
sentinelType: 'start' as const,
subflowId: 'loop-1',
subflowType: 'loop' as const,
},
}
const dag: DAG = {
nodes: new Map([[startNode.id, startNode]]),
loopConfigs: new Map(),
parallelConfigs: new Map(),
}
const state = createState()
const loopOrchestrator = {
getLoopScope: vi.fn(() => ({})),
evaluateInitialCondition: vi.fn().mockResolvedValue(false),
}
const orchestrator = createOrchestrator(dag, state, {}, loopOrchestrator)
const result = await orchestrator.executeNode(createContext(), startNode.id)
expect(result.output).toMatchObject({
shouldExit: true,
selectedRoute: EDGE.LOOP_EXIT,
})
})
it('prepares the current batch when executing a parallel start sentinel', async () => {
const startNode = createSentinelNode('parallel-parallel-1-sentinel-start', 'start')
const dag: DAG = {
nodes: new Map([[startNode.id, startNode]]),
loopConfigs: new Map(),
parallelConfigs: new Map(),
}
const state = createState()
const parallelOrchestrator = {
getParallelScope: vi.fn(() => ({ parallelId: 'parallel-1', totalBranches: 2 })),
initializeParallelScope: vi.fn(),
prepareCurrentBatch: vi.fn(),
}
const orchestrator = createOrchestrator(dag, state, parallelOrchestrator)
const result = await orchestrator.executeNode(createContext(), startNode.id)
expect(result.output).toEqual({ sentinelStart: true })
expect(parallelOrchestrator.prepareCurrentBatch).toHaveBeenCalledWith(
expect.any(Object),
'parallel-1'
)
})
it('returns parallel_exit from an empty parallel start sentinel without preparing a batch', async () => {
const startNode = createSentinelNode('parallel-parallel-1-sentinel-start', 'start')
const dag: DAG = {
nodes: new Map([[startNode.id, startNode]]),
loopConfigs: new Map(),
parallelConfigs: new Map(),
}
const state = createState()
const parallelOrchestrator = {
getParallelScope: vi.fn(() => ({
parallelId: 'parallel-1',
totalBranches: 0,
isEmpty: true,
})),
initializeParallelScope: vi.fn(),
prepareCurrentBatch: vi.fn(),
}
const orchestrator = createOrchestrator(dag, state, parallelOrchestrator)
const result = await orchestrator.executeNode(createContext(), startNode.id)
expect(result.output).toMatchObject({
shouldExit: true,
selectedRoute: EDGE.PARALLEL_EXIT,
})
expect(parallelOrchestrator.prepareCurrentBatch).not.toHaveBeenCalled()
})
it('prepares a batch continuation when parallel end selects parallel_continue', async () => {
const endNode = createSentinelNode('parallel-parallel-1-sentinel-end', 'end')
const dag: DAG = {
nodes: new Map([[endNode.id, endNode]]),
loopConfigs: new Map(),
parallelConfigs: new Map(),
}
const state = createState()
const parallelOrchestrator = {
getParallelScope: vi.fn(),
prepareForBatchContinuation: vi.fn(),
}
const orchestrator = createOrchestrator(dag, state, parallelOrchestrator)
await orchestrator.handleNodeCompletion(createContext(), endNode.id, {
selectedRoute: EDGE.PARALLEL_CONTINUE,
})
expect(state.setBlockOutput).toHaveBeenCalledWith(endNode.id, {
selectedRoute: EDGE.PARALLEL_CONTINUE,
})
expect(parallelOrchestrator.prepareForBatchContinuation).toHaveBeenCalledWith('parallel-1')
})
it('marks terminal parallel exit output as final when only the continue back edge remains', async () => {
const endNode = createSentinelNode('parallel-parallel-1-sentinel-end', 'end')
endNode.outgoingEdges.set('continue', {
target: 'parallel-parallel-1-sentinel-start',
sourceHandle: EDGE.PARALLEL_CONTINUE,
})
const dag: DAG = {
nodes: new Map([[endNode.id, endNode]]),
loopConfigs: new Map(),
parallelConfigs: new Map(),
}
const parallelOrchestrator = {
getParallelScope: vi.fn(() => ({ parallelId: 'parallel-1', totalBranches: 1 })),
aggregateParallelResults: vi.fn().mockResolvedValue({
allBranchesComplete: true,
results: [['result']],
totalBranches: 1,
}),
}
const orchestrator = createOrchestrator(dag, createState(), parallelOrchestrator)
const result = await orchestrator.executeNode(createContext(), endNode.id)
expect(result.isFinalOutput).toBe(true)
expect(result.output).toMatchObject({
results: [['result']],
selectedRoute: EDGE.PARALLEL_EXIT,
})
})
it('does not mark a continuing parallel batch as final output', async () => {
const endNode = createSentinelNode('parallel-parallel-1-sentinel-end', 'end')
endNode.outgoingEdges.set('continue', {
target: 'parallel-parallel-1-sentinel-start',
sourceHandle: EDGE.PARALLEL_CONTINUE,
})
const dag: DAG = {
nodes: new Map([[endNode.id, endNode]]),
loopConfigs: new Map(),
parallelConfigs: new Map(),
}
const parallelOrchestrator = {
getParallelScope: vi.fn(() => ({ parallelId: 'parallel-1', totalBranches: 3 })),
aggregateParallelResults: vi.fn().mockResolvedValue({
allBranchesComplete: false,
totalBranches: 3,
}),
}
const orchestrator = createOrchestrator(dag, createState(), parallelOrchestrator)
const result = await orchestrator.executeNode(createContext(), endNode.id)
expect(result.isFinalOutput).toBe(false)
expect(result.output).toMatchObject({
selectedRoute: EDGE.PARALLEL_CONTINUE,
})
})
it('records completed nested subflow sentinels as parent parallel branch output', async () => {
const endNode = {
...createSentinelNode('loop-nested-loop-sentinel-end', 'end'),
metadata: {
isSentinel: true,
sentinelType: 'end' as const,
subflowId: 'nested-loop',
subflowType: 'loop' as const,
},
}
const dag: DAG = {
nodes: new Map([[endNode.id, endNode]]),
loopConfigs: new Map(),
parallelConfigs: new Map(),
}
const state = createState()
const parallelOrchestrator = {
getParallelScope: vi.fn(),
handleParallelBranchCompletion: vi.fn(),
prepareForBatchContinuation: vi.fn(),
}
const orchestrator = createOrchestrator(dag, state, parallelOrchestrator)
const ctx = createContext()
ctx.subflowParentMap = new Map([
['nested-loop', { parentId: 'parent-parallel', parentType: 'parallel', branchIndex: 3 }],
])
await orchestrator.handleNodeCompletion(ctx, endNode.id, {
results: ['loop-result'],
shouldExit: true,
selectedRoute: EDGE.LOOP_EXIT,
})
expect(parallelOrchestrator.handleParallelBranchCompletion).toHaveBeenCalledWith(
ctx,
'parent-parallel',
endNode.id,
{ results: ['loop-result'] },
3
)
})
it('does not record continuing nested parallel batches as parent parallel branch output', async () => {
const endNode = {
...createSentinelNode('parallel-nested-parallel-sentinel-end', 'end'),
metadata: {
isSentinel: true,
sentinelType: 'end' as const,
subflowId: 'nested-parallel',
subflowType: 'parallel' as const,
},
}
const dag: DAG = {
nodes: new Map([[endNode.id, endNode]]),
loopConfigs: new Map(),
parallelConfigs: new Map(),
}
const state = createState()
const parallelOrchestrator = {
getParallelScope: vi.fn(),
handleParallelBranchCompletion: vi.fn(),
prepareForBatchContinuation: vi.fn(),
}
const orchestrator = createOrchestrator(dag, state, parallelOrchestrator)
const ctx = createContext()
ctx.subflowParentMap = new Map([
['nested-parallel', { parentId: 'parent-parallel', parentType: 'parallel', branchIndex: 3 }],
])
await orchestrator.handleNodeCompletion(ctx, endNode.id, {
sentinelEnd: true,
selectedRoute: EDGE.PARALLEL_CONTINUE,
})
expect(parallelOrchestrator.handleParallelBranchCompletion).not.toHaveBeenCalled()
})
it('writes stable outer-branch output aliases for completed parallel branch nodes', async () => {
const branchNode: DAGNode = {
id: 'worker₍0₎',
block: {
id: 'worker',
position: { x: 0, y: 0 },
enabled: true,
metadata: { id: 'function', name: 'Worker' },
config: { params: {} },
inputs: {},
outputs: {},
},
incomingEdges: new Set(),
outgoingEdges: new Map(),
metadata: {
isParallelBranch: true,
subflowId: 'parallel-1',
subflowType: 'parallel',
originalBlockId: 'worker',
branchIndex: 2,
},
}
const dag: DAG = {
nodes: new Map([[branchNode.id, branchNode]]),
loopConfigs: new Map(),
parallelConfigs: new Map(),
}
const state = createState()
const parallelOrchestrator = {
getParallelScope: vi.fn(() => ({ parallelId: 'parallel-1', totalBranches: 3 })),
initializeParallelScope: vi.fn(),
handleParallelBranchCompletion: vi.fn(),
}
const orchestrator = createOrchestrator(dag, state, parallelOrchestrator)
const output = { result: 'branch-2' }
const ctx = createContext()
await orchestrator.handleNodeCompletion(ctx, branchNode.id, output)
expect(parallelOrchestrator.handleParallelBranchCompletion).toHaveBeenCalledWith(
ctx,
'parallel-1',
branchNode.id,
output
)
expect(state.setBlockOutput).toHaveBeenCalledWith('worker__obranch-2', output)
expect(state.setBlockOutput).toHaveBeenCalledWith(branchNode.id, output)
})
it('records completed nested subflow sentinels as parent loop iteration output', async () => {
const endNode = {
...createSentinelNode('parallel-nested-parallel-sentinel-end', 'end'),
metadata: {
isSentinel: true,
sentinelType: 'end' as const,
subflowId: 'nested-parallel',
subflowType: 'parallel' as const,
},
}
const dag: DAG = {
nodes: new Map([[endNode.id, endNode]]),
loopConfigs: new Map(),
parallelConfigs: new Map(),
}
const state = createState()
const loopOrchestrator = {
storeLoopNodeOutput: vi.fn(),
}
const orchestrator = createOrchestrator(dag, state, {}, loopOrchestrator)
const ctx = createContext()
ctx.subflowParentMap = new Map([
['nested-parallel', { parentId: 'parent-loop', parentType: 'loop' }],
])
await orchestrator.handleNodeCompletion(ctx, endNode.id, {
results: ['parallel-result'],
sentinelEnd: true,
selectedRoute: EDGE.PARALLEL_EXIT,
})
expect(loopOrchestrator.storeLoopNodeOutput).toHaveBeenCalledWith(
ctx,
'parent-loop',
'nested-parallel',
{ results: ['parallel-result'] }
)
})
it('does not record continuing nested loop iterations as parent loop output', async () => {
const endNode = {
...createSentinelNode('loop-nested-loop-sentinel-end', 'end'),
metadata: {
isSentinel: true,
sentinelType: 'end' as const,
subflowId: 'nested-loop',
subflowType: 'loop' as const,
},
}
const dag: DAG = {
nodes: new Map([[endNode.id, endNode]]),
loopConfigs: new Map(),
parallelConfigs: new Map(),
}
const state = createState()
const loopOrchestrator = {
storeLoopNodeOutput: vi.fn(),
}
const orchestrator = createOrchestrator(dag, state, {}, loopOrchestrator)
const ctx = createContext()
ctx.subflowParentMap = new Map([
['nested-loop', { parentId: 'parent-loop', parentType: 'loop' }],
])
await orchestrator.handleNodeCompletion(ctx, endNode.id, {
shouldContinue: true,
selectedRoute: EDGE.LOOP_CONTINUE,
})
expect(loopOrchestrator.storeLoopNodeOutput).not.toHaveBeenCalled()
})
})
+120 -43
View File
@@ -7,7 +7,11 @@ import type { BlockStateController } from '@/executor/execution/types'
import type { LoopOrchestrator } from '@/executor/orchestrators/loop'
import type { ParallelOrchestrator } from '@/executor/orchestrators/parallel'
import type { ExecutionContext, NormalizedBlockOutput } from '@/executor/types'
import { extractBaseBlockId } from '@/executor/utils/subflow-utils'
import {
buildOuterBranchScopedId,
extractBaseBlockId,
extractOuterBranchIndex,
} from '@/executor/utils/subflow-utils'
const logger = createLogger('NodeExecutionOrchestrator')
@@ -25,6 +29,10 @@ function getResultCount(value: unknown): number {
return Array.isArray(value) ? value.length : 0
}
function getSubflowResultOutput(output: NormalizedBlockOutput): NormalizedBlockOutput {
return { results: output.results ?? [] }
}
export interface NodeExecutionResult {
nodeId: string
output: NormalizedBlockOutput
@@ -66,19 +74,20 @@ export class NodeExecutionOrchestrator {
}
}
const loopId = node.metadata.loopId
const loopId = node.metadata.subflowType === 'loop' ? node.metadata.subflowId : undefined
if (loopId && !this.loopOrchestrator.getLoopScope(ctx, loopId)) {
await this.loopOrchestrator.initializeLoopScope(ctx, loopId)
}
const parallelId = node.metadata.parallelId
const parallelId =
node.metadata.subflowType === 'parallel' ? node.metadata.subflowId : undefined
if (parallelId && !this.parallelOrchestrator.getParallelScope(ctx, parallelId)) {
await this.parallelOrchestrator.initializeParallelScope(ctx, parallelId)
}
if (node.metadata.isSentinel) {
const output = await this.handleSentinel(ctx, node)
const isFinalOutput = node.outgoingEdges.size === 0
const isFinalOutput = this.isFinalSentinelOutput(node, output)
return {
nodeId,
output,
@@ -95,42 +104,57 @@ export class NodeExecutionOrchestrator {
}
}
private isFinalSentinelOutput(node: DAGNode, output: NormalizedBlockOutput): boolean {
const selectedRoute = output.selectedRoute
if (selectedRoute === EDGE.LOOP_CONTINUE || selectedRoute === EDGE.PARALLEL_CONTINUE) {
return false
}
if (selectedRoute === EDGE.LOOP_EXIT || selectedRoute === EDGE.PARALLEL_EXIT) {
return !Array.from(node.outgoingEdges.values()).some(
(edge) => edge.sourceHandle === selectedRoute
)
}
return node.outgoingEdges.size === 0
}
private async handleSentinel(
ctx: ExecutionContext,
node: DAGNode
): Promise<NormalizedBlockOutput> {
const sentinelType = node.metadata.sentinelType
const loopId = node.metadata.loopId
const parallelId = node.metadata.parallelId
const isParallelSentinel = node.metadata.isParallelSentinel
const subflowType = node.metadata.subflowType
const subflowId = node.metadata.subflowId
if (isParallelSentinel) {
return await this.handleParallelSentinel(ctx, node, sentinelType, parallelId)
if (!subflowType || !subflowId) {
logger.warn('Sentinel missing subflow metadata', { nodeId: node.id, sentinelType })
return {}
}
if (subflowType === 'parallel') {
return await this.handleParallelSentinel(ctx, node, sentinelType, subflowId)
}
switch (sentinelType) {
case 'start': {
if (loopId) {
const shouldExecute = await this.loopOrchestrator.evaluateInitialCondition(ctx, loopId)
if (!shouldExecute) {
logger.info('Loop initial condition false, skipping loop body', { loopId })
return {
sentinelStart: true,
shouldExit: true,
selectedRoute: EDGE.LOOP_EXIT,
}
const shouldExecute = await this.loopOrchestrator.evaluateInitialCondition(ctx, subflowId)
if (!shouldExecute) {
logger.info('Loop initial condition false, skipping loop body', { loopId: subflowId })
return {
sentinelStart: true,
shouldExit: true,
selectedRoute: EDGE.LOOP_EXIT,
}
}
return { sentinelStart: true }
}
case 'end': {
if (!loopId) {
logger.warn('Sentinel end called without loopId')
return { shouldExit: true, selectedRoute: EDGE.LOOP_EXIT }
}
const continuationResult = await this.loopOrchestrator.evaluateLoopContinuation(ctx, loopId)
const continuationResult = await this.loopOrchestrator.evaluateLoopContinuation(
ctx,
subflowId
)
if (continuationResult.shouldContinue) {
return {
@@ -161,13 +185,8 @@ export class NodeExecutionOrchestrator {
ctx: ExecutionContext,
node: DAGNode,
sentinelType: string | undefined,
parallelId: string | undefined
parallelId: string
): Promise<NormalizedBlockOutput> {
if (!parallelId) {
logger.warn('Parallel sentinel called without parallelId')
return {}
}
if (sentinelType === 'start') {
if (!this.parallelOrchestrator.getParallelScope(ctx, parallelId)) {
const parallelConfig = this.dag.parallelConfigs.get(parallelId)
@@ -186,6 +205,7 @@ export class NodeExecutionOrchestrator {
}
}
this.parallelOrchestrator.prepareCurrentBatch(ctx, parallelId)
return { sentinelStart: true }
}
@@ -222,18 +242,21 @@ export class NodeExecutionOrchestrator {
return
}
const loopId = node.metadata.loopId
const loopId = node.metadata.subflowType === 'loop' ? node.metadata.subflowId : undefined
const isParallelBranch = node.metadata.isParallelBranch
const isSentinel = node.metadata.isSentinel
if (isSentinel) {
this.handleRegularNodeCompletion(ctx, node, output)
this.handleParentSubflowCompletion(ctx, node, output)
} else if (loopId) {
this.handleLoopNodeCompletion(ctx, node, output, loopId)
} else if (isParallelBranch) {
const parallelId = this.findParallelIdForNode(node.id)
const parallelId =
node.metadata.subflowType === 'parallel' ? node.metadata.subflowId : undefined
if (parallelId) {
await this.handleParallelNodeCompletion(ctx, node, output, parallelId)
} else {
logger.warn('Parallel branch missing subflow metadata', { nodeId: node.id })
this.handleRegularNodeCompletion(ctx, node, output)
}
} else {
@@ -262,9 +285,60 @@ export class NodeExecutionOrchestrator {
await this.parallelOrchestrator.initializeParallelScope(ctx, parallelId)
}
this.parallelOrchestrator.handleParallelBranchCompletion(ctx, parallelId, node.id, output)
const branchIndex = node.metadata.branchIndex
if (branchIndex !== undefined && extractOuterBranchIndex(node.id) === undefined) {
const originalBlockId = node.metadata.originalBlockId ?? extractBaseBlockId(node.id)
this.state.setBlockOutput(buildOuterBranchScopedId(originalBlockId, branchIndex), output)
}
this.state.setBlockOutput(node.id, output)
}
private handleParentSubflowCompletion(
ctx: ExecutionContext,
node: DAGNode,
output: NormalizedBlockOutput
): void {
if (node.metadata.sentinelType !== 'end' || !node.metadata.subflowId) {
return
}
if (
output.selectedRoute === EDGE.LOOP_CONTINUE ||
output.selectedRoute === EDGE.LOOP_CONTINUE_ALT ||
output.selectedRoute === EDGE.PARALLEL_CONTINUE
) {
return
}
const subflowId = node.metadata.subflowId
const parentEntry = ctx.subflowParentMap?.get(subflowId)
if (!parentEntry) {
return
}
if (parentEntry.parentType === 'parallel') {
if (parentEntry.branchIndex === undefined) {
return
}
this.parallelOrchestrator.handleParallelBranchCompletion(
ctx,
parentEntry.parentId,
node.id,
getSubflowResultOutput(output),
parentEntry.branchIndex
)
return
}
this.loopOrchestrator.storeLoopNodeOutput(
ctx,
parentEntry.parentId,
subflowId,
getSubflowResultOutput(output)
)
}
private handleRegularNodeCompletion(
ctx: ExecutionContext,
node: DAGNode,
@@ -274,27 +348,30 @@ export class NodeExecutionOrchestrator {
if (
node.metadata.isSentinel &&
node.metadata.subflowType === 'loop' &&
node.metadata.sentinelType === 'end' &&
output.selectedRoute === 'loop_continue'
) {
const loopId = node.metadata.loopId
if (loopId) {
this.loopOrchestrator.clearLoopExecutionState(loopId, ctx)
this.loopOrchestrator.restoreLoopEdges(loopId)
const loopId = node.metadata.subflowId
if (!loopId) {
logger.warn('Loop sentinel missing subflow metadata', { nodeId: node.id })
return
}
this.loopOrchestrator.clearLoopExecutionState(loopId, ctx)
this.loopOrchestrator.restoreLoopEdges(loopId)
}
if (
node.metadata.isParallelSentinel &&
node.metadata.subflowType === 'parallel' &&
node.metadata.sentinelType === 'end' &&
output.selectedRoute === EDGE.PARALLEL_CONTINUE
) {
this.state.deleteBlockState(node.id)
const parallelId = node.metadata.subflowId
if (!parallelId) {
logger.warn('Parallel sentinel missing subflow metadata', { nodeId: node.id })
return
}
this.parallelOrchestrator.prepareForBatchContinuation(parallelId)
}
}
private findParallelIdForNode(nodeId: string): string | undefined {
const baseId = extractBaseBlockId(nodeId)
return this.parallelOrchestrator.findParallelIdForNode(baseId)
}
}
+347 -46
View File
@@ -2,11 +2,18 @@
* @vitest-environment node
*/
import { beforeEach, describe, expect, it, vi } from 'vitest'
import type { DAG } from '@/executor/dag/builder'
import { DEFAULTS } from '@/executor/constants'
import type { DAG, DAGNode } from '@/executor/dag/builder'
import type { BlockStateWriter, ContextExtensions } from '@/executor/execution/types'
import { ParallelOrchestrator } from '@/executor/orchestrators/parallel'
import type { ExecutionContext } from '@/executor/types'
import { buildBranchNodeId } from '@/executor/utils/subflow-utils'
import {
buildBranchNodeId,
buildParallelSentinelEndId,
buildParallelSentinelStartId,
buildSentinelEndId,
buildSentinelStartId,
} from '@/executor/utils/subflow-utils'
const { mockCompactSubflowResults } = vi.hoisted(() => ({
mockCompactSubflowResults: vi.fn(async (results: unknown) => results),
@@ -34,6 +41,24 @@ function createDag(): DAG {
}
}
function createDagNode(id: string, metadata: DAGNode['metadata'] = {}): DAGNode {
return {
id,
block: {
id,
position: { x: 0, y: 0 },
config: { tool: '', params: {} },
inputs: {},
outputs: {},
metadata: { id: 'function', name: id },
enabled: true,
},
incomingEdges: new Set(),
outgoingEdges: new Map(),
metadata,
}
}
function createState(): BlockStateWriter {
return {
setBlockOutput: vi.fn(),
@@ -43,6 +68,12 @@ function createState(): BlockStateWriter {
}
}
function createEdgeManager() {
return {
clearDeactivatedEdgesForNodes: vi.fn(),
}
}
function createContext(overrides: Partial<ExecutionContext> = {}): ExecutionContext {
return {
workflowId: 'workflow-1',
@@ -87,14 +118,8 @@ describe('ParallelOrchestrator', () => {
mockCompactSubflowResults.mockImplementation(async (results: unknown) => results)
})
it('awaits empty-subflow lifecycle callbacks before returning the empty scope', async () => {
let releaseStart: (() => void) | undefined
const onBlockStart = vi.fn(
() =>
new Promise<void>((resolve) => {
releaseStart = resolve
})
)
it('defers empty-subflow lifecycle callbacks to the sentinel end path', async () => {
const onBlockStart = vi.fn()
const onBlockComplete = vi.fn()
const contextExtensions: ContextExtensions = {
onBlockStart,
@@ -108,19 +133,14 @@ describe('ParallelOrchestrator', () => {
)
const ctx = createContext()
const initializePromise = orchestrator.initializeParallelScope(ctx, 'parallel-1')
await vi.waitFor(() => expect(onBlockStart).toHaveBeenCalledTimes(1))
const scope = await orchestrator.initializeParallelScope(ctx, 'parallel-1')
expect(onBlockStart).not.toHaveBeenCalled()
expect(onBlockComplete).not.toHaveBeenCalled()
releaseStart?.()
const scope = await initializePromise
expect(onBlockComplete).toHaveBeenCalledTimes(1)
expect(scope.isEmpty).toBe(true)
})
it('swallows helper callback failures on empty parallel paths', async () => {
it('returns an empty scope without emitting start-side lifecycle callbacks', async () => {
const contextExtensions: ContextExtensions = {
onBlockStart: vi.fn().mockRejectedValue(new Error('start failed')),
onBlockComplete: vi.fn().mockRejectedValue(new Error('complete failed')),
@@ -138,6 +158,36 @@ describe('ParallelOrchestrator', () => {
parallelId: 'parallel-1',
isEmpty: true,
})
expect(contextExtensions.onBlockStart).not.toHaveBeenCalled()
expect(contextExtensions.onBlockComplete).not.toHaveBeenCalled()
})
it('resolves collection distributions with the parallel start sentinel scope', async () => {
const dag = createDag()
const parallelConfig = dag.parallelConfigs.get('parallel-1')!
parallelConfig.distribution = '<Producer.items>'
const resolver = {
resolveSingleReference: vi.fn().mockResolvedValue(['item-1', 'item-2']),
}
const orchestrator = new ParallelOrchestrator(
dag,
createState(),
resolver as any,
{},
undefined,
createEdgeManager() as any
)
const scope = await orchestrator.initializeParallelScope(createContext(), 'parallel-1')
expect(resolver.resolveSingleReference).toHaveBeenCalledWith(
expect.any(Object),
'parallel-parallel-1-sentinel-start',
'<Producer.items>',
undefined,
{ allowLargeValueRefs: true }
)
expect(scope.totalBranches).toBe(2)
})
it('records resumed later-batch outputs under restored global branch indexes', () => {
@@ -184,6 +234,116 @@ describe('ParallelOrchestrator', () => {
expect(scope?.branchOutputs.has(0)).toBe(false)
})
it('clamps batch size and caps current batch to total branch count', async () => {
const dag = createDag()
const parallelConfig = dag.parallelConfigs.get('parallel-1')!
parallelConfig.parallelType = 'count'
parallelConfig.count = 9
parallelConfig.batchSize = 0
const orchestrator = new ParallelOrchestrator(dag, createState(), null, {})
const zeroBatchScope = await orchestrator.initializeParallelScope(createContext(), 'parallel-1')
expect(zeroBatchScope.batchSize).toBe(1)
expect(zeroBatchScope.currentBatchSize).toBe(1)
parallelConfig.batchSize = 50
const oversizedBatchScope = await orchestrator.initializeParallelScope(
createContext(),
'parallel-1'
)
expect(oversizedBatchScope.currentBatchSize).toBe(9)
})
it.each([
['oversized numeric batch size', 999, DEFAULTS.MAX_PARALLEL_BRANCHES],
['negative batch size', -1, 1],
['undefined batch size', undefined, DEFAULTS.MAX_PARALLEL_BRANCHES],
['nonnumeric batch size', 'not-a-number', DEFAULTS.MAX_PARALLEL_BRANCHES],
])('normalizes %s', async (_name, batchSize, expectedBatchSize) => {
const dag = createDag()
const parallelConfig = dag.parallelConfigs.get('parallel-1')!
parallelConfig.parallelType = 'count'
parallelConfig.count = DEFAULTS.MAX_PARALLEL_BRANCHES + 10
parallelConfig.batchSize = batchSize as never
const orchestrator = new ParallelOrchestrator(dag, createState(), null, {})
const scope = await orchestrator.initializeParallelScope(createContext(), 'parallel-1')
expect(scope.batchSize).toBe(expectedBatchSize)
expect(scope.currentBatchSize).toBe(expectedBatchSize)
})
it('advances batch state at sentinel end and prepares the next batch at sentinel start', async () => {
const dag = createDag()
const templateBranchId = buildBranchNodeId('task-1', 0)
const secondBranchId = buildBranchNodeId('task-1', 1)
dag.nodes.set(templateBranchId, {
id: templateBranchId,
block: {
id: 'task-1',
position: { x: 0, y: 0 },
config: { tool: '', params: {} },
inputs: {},
outputs: {},
metadata: { id: 'function', name: 'Task 1' },
enabled: true,
},
incomingEdges: new Set(),
outgoingEdges: new Map(),
metadata: {
subflowId: 'parallel-1',
subflowType: 'parallel',
isParallelBranch: true,
branchIndex: 0,
},
})
const state = createState()
const edgeManager = createEdgeManager()
const orchestrator = new ParallelOrchestrator(dag, state, null, {}, edgeManager)
const scope = {
parallelId: 'parallel-1',
totalBranches: 4,
batchSize: 2,
currentBatchStart: 0,
currentBatchSize: 2,
accumulatedOutputs: new Map<number, any[]>(),
branchOutputs: new Map<number, any[]>([
[0, [{ output: 'branch-0' }]],
[1, [{ output: 'branch-1' }]],
]),
}
const ctx = createContext({
parallelExecutions: new Map([['parallel-1', scope]]),
})
const result = await orchestrator.aggregateParallelResults(ctx, 'parallel-1')
expect(result.allBranchesComplete).toBe(false)
expect(scope.currentBatchStart).toBe(2)
expect(scope.currentBatchSize).toBe(2)
expect(ctx.parallelBlockMapping?.size ?? 0).toBe(0)
orchestrator.prepareCurrentBatch(ctx, 'parallel-1')
expect(ctx.parallelBlockMapping?.get(templateBranchId)).toMatchObject({
originalBlockId: 'task-1',
parallelId: 'parallel-1',
iterationIndex: 2,
})
expect(ctx.parallelBlockMapping?.get(secondBranchId)).toMatchObject({
originalBlockId: 'task-1',
parallelId: 'parallel-1',
iterationIndex: 3,
})
expect(state.deleteBlockState).toHaveBeenCalledWith(templateBranchId)
expect(state.deleteBlockState).toHaveBeenCalledWith(secondBranchId)
expect(edgeManager.clearDeactivatedEdgesForNodes).toHaveBeenCalledWith(
new Set([templateBranchId, secondBranchId])
)
})
it('resets only incoming batch branch state when scheduling later batches', async () => {
const dag = createDag()
const incomingBranchId = buildBranchNodeId('task-1', 0)
@@ -201,7 +361,12 @@ describe('ParallelOrchestrator', () => {
},
incomingEdges: new Set(),
outgoingEdges: new Set(),
metadata: { parallelId: 'parallel-1', isParallelBranch: true, branchIndex: 0 },
metadata: {
subflowId: 'parallel-1',
subflowType: 'parallel',
isParallelBranch: true,
branchIndex: 0,
},
})
dag.nodes.set(previousBranchId, {
id: previousBranchId,
@@ -216,36 +381,34 @@ describe('ParallelOrchestrator', () => {
},
incomingEdges: new Set(),
outgoingEdges: new Set(),
metadata: { parallelId: 'parallel-1', isParallelBranch: true, branchIndex: 1 },
metadata: {
subflowId: 'parallel-1',
subflowType: 'parallel',
isParallelBranch: true,
branchIndex: 1,
},
})
const state = createState()
const orchestrator = new ParallelOrchestrator(dag, state, null, {})
await (
orchestrator as unknown as {
scheduleNextBatch(
ctx: ExecutionContext,
scope: NonNullable<ExecutionContext['parallelExecutions']> extends Map<
string,
infer Scope
>
? Scope
: never,
nextBatchStart: number
): Promise<void>
}
).scheduleNextBatch(
createContext(),
{
parallelId: 'parallel-1',
totalBranches: 3,
batchSize: 1,
currentBatchStart: 0,
currentBatchSize: 2,
accumulatedOutputs: new Map([[1, [{ output: 'previous' }]]]),
branchOutputs: new Map(),
},
2
orchestrator.prepareCurrentBatch(
createContext({
parallelExecutions: new Map([
[
'parallel-1',
{
parallelId: 'parallel-1',
totalBranches: 3,
batchSize: 1,
currentBatchStart: 2,
currentBatchSize: 1,
accumulatedOutputs: new Map([[1, [{ output: 'previous' }]]]),
branchOutputs: new Map(),
},
],
]),
}),
'parallel-1'
)
expect(state.deleteBlockState).toHaveBeenCalledWith(incomingBranchId)
@@ -254,6 +417,139 @@ describe('ParallelOrchestrator', () => {
expect(state.unmarkExecuted).not.toHaveBeenCalledWith(previousBranchId)
})
it('marks expanded branch nodes dirty when running from a dirty parallel container', () => {
const dag = createDag()
const templateBranchId = buildBranchNodeId('task-1', 0)
const secondBranchId = buildBranchNodeId('task-1', 1)
dag.nodes.set(templateBranchId, {
id: templateBranchId,
block: {
id: 'task-1',
position: { x: 0, y: 0 },
config: { tool: '', params: {} },
inputs: {},
outputs: {},
metadata: { id: 'function', name: 'Task 1' },
enabled: true,
},
incomingEdges: new Set(),
outgoingEdges: new Map(),
metadata: {
subflowId: 'parallel-1',
subflowType: 'parallel',
isParallelBranch: true,
branchIndex: 0,
},
})
const dirtySet = new Set(['parallel-1'])
const orchestrator = new ParallelOrchestrator(dag, createState(), null, {})
orchestrator.prepareCurrentBatch(
createContext({
runFromBlockContext: { startBlockId: 'parallel-1', dirtySet },
parallelExecutions: new Map([
[
'parallel-1',
{
parallelId: 'parallel-1',
totalBranches: 2,
batchSize: 2,
currentBatchStart: 0,
currentBatchSize: 2,
branchOutputs: new Map(),
},
],
]),
}),
'parallel-1'
)
expect(dirtySet.has(templateBranchId)).toBe(true)
expect(dirtySet.has(secondBranchId)).toBe(true)
})
it('marks cloned nested loop body nodes dirty for non-zero branches', () => {
const dag = createDag()
const parallelId = 'parallel-1'
const loopId = 'loop-1'
const taskId = 'task-1'
const parallelStartId = buildParallelSentinelStartId(parallelId)
const parallelEndId = buildParallelSentinelEndId(parallelId)
const loopStartId = buildSentinelStartId(loopId)
const loopEndId = buildSentinelEndId(loopId)
dag.parallelConfigs.set(parallelId, {
id: parallelId,
nodes: [loopId],
count: 2,
parallelType: 'count',
})
dag.loopConfigs.set(loopId, {
id: loopId,
nodes: [taskId],
loopType: 'for',
iterations: 1,
})
dag.nodes.set(parallelStartId, createDagNode(parallelStartId))
dag.nodes.set(parallelEndId, createDagNode(parallelEndId))
dag.nodes.set(
loopStartId,
createDagNode(loopStartId, {
isSentinel: true,
sentinelType: 'start',
subflowId: loopId,
subflowType: 'loop',
})
)
dag.nodes.set(
taskId,
createDagNode(taskId, {
isLoopNode: true,
subflowId: loopId,
subflowType: 'loop',
originalBlockId: taskId,
})
)
dag.nodes.set(
loopEndId,
createDagNode(loopEndId, {
isSentinel: true,
sentinelType: 'end',
subflowId: loopId,
subflowType: 'loop',
})
)
dag.nodes.get(loopStartId)!.outgoingEdges.set(`${loopStartId}->${taskId}`, { target: taskId })
dag.nodes.get(taskId)!.incomingEdges.add(loopStartId)
dag.nodes.get(taskId)!.outgoingEdges.set(`${taskId}->${loopEndId}`, { target: loopEndId })
dag.nodes.get(loopEndId)!.incomingEdges.add(taskId)
const dirtySet = new Set([parallelId])
const orchestrator = new ParallelOrchestrator(dag, createState(), null, {})
orchestrator.prepareCurrentBatch(
createContext({
runFromBlockContext: { startBlockId: parallelId, dirtySet },
parallelExecutions: new Map([
[
parallelId,
{
parallelId,
totalBranches: 2,
batchSize: 2,
currentBatchStart: 0,
currentBatchSize: 2,
branchOutputs: new Map(),
},
],
]),
}),
parallelId
)
expect([...dirtySet]).toContain(taskId)
expect([...dirtySet].some((nodeId) => nodeId.startsWith(`${taskId}__clone`))).toBe(true)
})
it('compacts accumulated outputs before scheduling later batches', async () => {
const dag = createDag()
const templateBranchId = buildBranchNodeId('task-1', 0)
@@ -270,7 +566,12 @@ describe('ParallelOrchestrator', () => {
},
incomingEdges: new Set(),
outgoingEdges: new Set(),
metadata: { parallelId: 'parallel-1', isParallelBranch: true, branchIndex: 0 },
metadata: {
subflowId: 'parallel-1',
subflowType: 'parallel',
isParallelBranch: true,
branchIndex: 0,
},
})
const orchestrator = new ParallelOrchestrator(dag, createState(), null, {})
const previousOutputs = [{ output: 'previous' }]
+148 -70
View File
@@ -3,15 +3,20 @@ import { toError } from '@sim/utils/errors'
import { compactSubflowResults } from '@/lib/execution/payloads/serializer'
import { DEFAULTS } from '@/executor/constants'
import type { DAG } from '@/executor/dag/builder'
import type { EdgeManager } from '@/executor/execution/edge-manager'
import type { ParallelScope } from '@/executor/execution/state'
import type { BlockStateWriter, ContextExtensions } from '@/executor/execution/types'
import type { ExecutionContext, NormalizedBlockOutput } from '@/executor/types'
import type { ParallelConfigWithNodes } from '@/executor/types/parallel'
import { type ClonedSubflowInfo, ParallelExpander } from '@/executor/utils/parallel-expansion'
import {
addSubflowErrorLog,
emitEmptySubflowEvents,
buildBranchNodeId,
buildParallelSentinelEndId,
buildParallelSentinelStartId,
buildSentinelEndId,
buildSentinelStartId,
emitSubflowSuccessEvents,
extractBaseBlockId,
extractBranchIndex,
} from '@/executor/utils/subflow-utils'
import { resolveArrayInputAsync } from '@/executor/utils/subflow-utils.server'
@@ -42,7 +47,8 @@ export class ParallelOrchestrator {
private dag: DAG,
private state: BlockStateWriter,
private resolver: VariableResolver | null = null,
private contextExtensions: ContextExtensions | null = null
private contextExtensions: ContextExtensions | null = null,
private edgeManager: Pick<EdgeManager, 'clearDeactivatedEdgesForNodes'> | null = null
) {}
async initializeParallelScope(ctx: ExecutionContext, parallelId: string): Promise<ParallelScope> {
@@ -96,10 +102,6 @@ export class ParallelOrchestrator {
}
ctx.parallelExecutions.set(parallelId, scope)
this.state.setBlockOutput(parallelId, { results: [] })
await emitEmptySubflowEvents(ctx, parallelId, 'parallel', this.contextExtensions)
logger.info('Parallel scope initialized with empty distribution, skipping body', {
parallelId,
branchCount: 0,
@@ -110,18 +112,6 @@ export class ParallelOrchestrator {
const batchSize = this.resolveBatchSize(parallelConfig.batchSize)
const currentBatchSize = Math.min(batchSize, branchCount)
const batchItems = items?.slice(0, currentBatchSize)
const { entryNodes, clonedSubflows, allBranchNodes } = this.expander.expandParallel(
this.dag,
parallelId,
currentBatchSize,
batchItems,
{ branchIndexOffset: 0, totalBranches: branchCount }
)
this.registerClonedSubflows(ctx, parallelId, clonedSubflows)
this.registerBranchMappings(ctx, parallelId, allBranchNodes)
const scope: ParallelScope = {
parallelId,
totalBranches: branchCount,
@@ -138,26 +128,60 @@ export class ParallelOrchestrator {
}
ctx.parallelExecutions.set(parallelId, scope)
const newEntryNodes = entryNodes.filter((nodeId) => !nodeId.endsWith('__branch-0'))
if (newEntryNodes.length > 0) {
if (!ctx.pendingDynamicNodes) {
ctx.pendingDynamicNodes = []
}
ctx.pendingDynamicNodes.push(...newEntryNodes)
}
logger.info('Parallel scope initialized', {
parallelId,
branchCount,
batchSize,
currentBatchSize,
entryNodeCount: entryNodes.length,
newEntryNodes: newEntryNodes.length,
})
return scope
}
prepareCurrentBatch(ctx: ExecutionContext, parallelId: string): void {
const scope = ctx.parallelExecutions?.get(parallelId)
if (!scope || scope.isEmpty) {
return
}
const currentBatchStart = scope.currentBatchStart ?? 0
const currentBatchSize =
scope.currentBatchSize ??
Math.min(scope.batchSize ?? DEFAULT_PARALLEL_BATCH_SIZE, scope.totalBranches)
if (currentBatchSize <= 0) {
return
}
const batchItems = scope.items?.slice(currentBatchStart, currentBatchStart + currentBatchSize)
const { clonedSubflows, allBranchNodes, entryNodes, terminalNodes } =
this.expander.expandParallel(this.dag, parallelId, currentBatchSize, batchItems, {
branchIndexOffset: currentBatchStart,
totalBranches: scope.totalBranches,
})
this.markRunFromBlockBatchDirty(
ctx,
parallelId,
allBranchNodes,
entryNodes,
terminalNodes,
clonedSubflows
)
this.registerClonedSubflows(ctx, parallelId, clonedSubflows)
this.registerBranchMappings(ctx, parallelId, allBranchNodes)
this.resetBatchExecutionState(allBranchNodes)
this.edgeManager?.clearDeactivatedEdgesForNodes(new Set(allBranchNodes))
logger.info('Prepared parallel batch', {
parallelId,
currentBatchStart,
currentBatchSize,
totalBranches: scope.totalBranches,
branchNodeCount: allBranchNodes.length,
})
}
private async resolveBranchCount(
ctx: ExecutionContext,
config: SerializedParallel,
@@ -219,7 +243,12 @@ export class ParallelOrchestrator {
'Parallel collection distribution is empty. Provide an array or a reference that resolves to a collection.'
)
}
return resolveArrayInputAsync(ctx, config.distribution, this.resolver)
return resolveArrayInputAsync(
ctx,
config.distribution,
this.resolver,
buildParallelSentinelStartId(config.id)
)
}
private resolveBatchSize(batchSize: unknown): number {
@@ -278,6 +307,78 @@ export class ParallelOrchestrator {
}
}
private markRunFromBlockBatchDirty(
ctx: ExecutionContext,
parallelId: string,
allBranchNodes: string[],
entryNodes: string[],
terminalNodes: string[],
clonedSubflows: ClonedSubflowInfo[]
): void {
const dirtySet = ctx.runFromBlockContext?.dirtySet
if (!dirtySet) return
const parallelStartId = buildParallelSentinelStartId(parallelId)
const parallelEndId = buildParallelSentinelEndId(parallelId)
if (
!dirtySet.has(parallelId) &&
!dirtySet.has(parallelStartId) &&
!dirtySet.has(parallelEndId)
) {
return
}
for (const nodeId of allBranchNodes) dirtySet.add(nodeId)
for (const nodeId of entryNodes) dirtySet.add(nodeId)
for (const nodeId of terminalNodes) dirtySet.add(nodeId)
const config = this.dag.parallelConfigs.get(parallelId)
for (const nodeId of config?.nodes ?? []) {
if (this.dag.parallelConfigs.has(nodeId) || this.dag.loopConfigs.has(nodeId)) {
this.collectSubflowNodeIds(nodeId, dirtySet)
}
}
for (const clone of clonedSubflows) {
dirtySet.add(clone.clonedId)
this.collectSubflowNodeIds(clone.clonedId, dirtySet)
}
}
private collectSubflowNodeIds(
subflowId: string,
nodeIds: Set<string>,
visited = new Set<string>()
): void {
if (visited.has(subflowId)) return
visited.add(subflowId)
if (this.dag.parallelConfigs.has(subflowId)) {
nodeIds.add(buildParallelSentinelStartId(subflowId))
nodeIds.add(buildParallelSentinelEndId(subflowId))
for (const childId of this.dag.parallelConfigs.get(subflowId)?.nodes ?? []) {
if (this.dag.parallelConfigs.has(childId) || this.dag.loopConfigs.has(childId)) {
this.collectSubflowNodeIds(childId, nodeIds, visited)
} else {
nodeIds.add(buildBranchNodeId(childId, 0))
}
}
return
}
if (this.dag.loopConfigs.has(subflowId)) {
nodeIds.add(buildSentinelStartId(subflowId))
nodeIds.add(buildSentinelEndId(subflowId))
for (const childId of this.dag.loopConfigs.get(subflowId)?.nodes ?? []) {
if (this.dag.parallelConfigs.has(childId) || this.dag.loopConfigs.has(childId)) {
this.collectSubflowNodeIds(childId, nodeIds, visited)
} else {
nodeIds.add(childId)
}
}
}
}
/**
* Stores a node's output in the branch outputs for later aggregation.
* Aggregation is triggered by the sentinel-end node via the edge mechanism,
@@ -288,7 +389,8 @@ export class ParallelOrchestrator {
ctx: ExecutionContext,
parallelId: string,
nodeId: string,
output: NormalizedBlockOutput
output: NormalizedBlockOutput,
branchIndexOverride?: number
): void {
const scope = ctx.parallelExecutions?.get(parallelId)
if (!scope) {
@@ -298,9 +400,10 @@ export class ParallelOrchestrator {
const mappedBranch = ctx.parallelBlockMapping?.get(nodeId)
const branchIndex =
mappedBranch?.parallelId === parallelId
branchIndexOverride ??
(mappedBranch?.parallelId === parallelId
? mappedBranch.iterationIndex
: (this.dag.nodes.get(nodeId)?.metadata.branchIndex ?? extractBranchIndex(nodeId))
: (this.dag.nodes.get(nodeId)?.metadata.branchIndex ?? extractBranchIndex(nodeId)))
if (branchIndex === null) {
logger.warn('Could not extract branch index from node ID', { nodeId })
return
@@ -360,7 +463,7 @@ export class ParallelOrchestrator {
accumulatedOutputs.set(branchIdx, compactedAccumulated[position])
})
}
await this.scheduleNextBatch(ctx, scope, nextBatchStart)
this.advanceToNextBatch(scope, nextBatchStart)
return {
allBranchesComplete: false,
completedBranches: accumulatedOutputs.size,
@@ -400,37 +503,15 @@ export class ParallelOrchestrator {
}
}
private async scheduleNextBatch(
ctx: ExecutionContext,
scope: ParallelScope,
nextBatchStart: number
): Promise<void> {
private advanceToNextBatch(scope: ParallelScope, nextBatchStart: number): void {
const batchSize = scope.batchSize ?? DEFAULT_PARALLEL_BATCH_SIZE
const remaining = scope.totalBranches - nextBatchStart
const currentBatchSize = Math.min(batchSize, remaining)
const batchItems = scope.items?.slice(nextBatchStart, nextBatchStart + currentBatchSize)
const { entryNodes, clonedSubflows, allBranchNodes } = this.expander.expandParallel(
this.dag,
scope.parallelId,
currentBatchSize,
batchItems,
{ branchIndexOffset: nextBatchStart, totalBranches: scope.totalBranches }
)
this.registerClonedSubflows(ctx, scope.parallelId, clonedSubflows)
this.registerBranchMappings(ctx, scope.parallelId, allBranchNodes)
this.resetBatchExecutionState(allBranchNodes)
scope.currentBatchStart = nextBatchStart
scope.currentBatchSize = currentBatchSize
if (!ctx.pendingDynamicNodes) {
ctx.pendingDynamicNodes = []
}
ctx.pendingDynamicNodes.push(...entryNodes)
logger.info('Scheduled next parallel batch', {
logger.info('Advanced to next parallel batch', {
parallelId: scope.parallelId,
nextBatchStart,
currentBatchSize,
@@ -438,6 +519,12 @@ export class ParallelOrchestrator {
})
}
prepareForBatchContinuation(parallelId: string): void {
this.state.unmarkExecuted(buildParallelSentinelStartId(parallelId))
this.state.unmarkExecuted(buildParallelSentinelEndId(parallelId))
this.state.deleteBlockState(buildParallelSentinelEndId(parallelId))
}
private resetBatchExecutionState(branchNodeIds: string[]): void {
for (const nodeId of branchNodeIds) {
const node = this.dag.nodes.get(nodeId)
@@ -470,7 +557,7 @@ export class ParallelOrchestrator {
}
ctx.parallelBlockMapping.set(nodeId, {
originalBlockId: node?.metadata.originalBlockId ?? nodeId,
originalBlockId: node?.metadata.originalBlockId ?? extractBaseBlockId(nodeId),
parallelId,
iterationIndex: branchIndex,
})
@@ -488,7 +575,8 @@ export class ParallelOrchestrator {
return null
}
const parallelId = node.metadata.parallelId
const parallelId =
node.metadata.subflowType === 'parallel' ? node.metadata.subflowId : undefined
if (!parallelId) {
return null
}
@@ -504,14 +592,4 @@ export class ParallelOrchestrator {
getParallelScope(ctx: ExecutionContext, parallelId: string): ParallelScope | undefined {
return ctx.parallelExecutions?.get(parallelId)
}
findParallelIdForNode(baseNodeId: string): string | undefined {
for (const [parallelId, config] of this.dag.parallelConfigs) {
const parallelConfig = config as ParallelConfigWithNodes
if (parallelConfig.nodes?.includes(baseNodeId)) {
return parallelId
}
}
return undefined
}
}
+3 -3
View File
@@ -341,6 +341,7 @@ export interface ExecutionContext {
items?: any[]
condition?: string
skipFirstConditionCheck?: boolean
skippedAtStart?: boolean
loopType?: 'for' | 'forEach' | 'while' | 'doWhile'
}
>
@@ -357,6 +358,8 @@ export interface ExecutionContext {
branchOutputs: Map<number, any[]>
parallelType?: 'count' | 'collection'
items?: any[]
validationError?: string
isEmpty?: boolean
}
>
@@ -416,9 +419,6 @@ export interface ExecutionContext {
*/
abortSignal?: AbortSignal
// Dynamically added nodes that need to be scheduled (e.g., from parallel expansion)
pendingDynamicNodes?: string[]
/**
* When true, UserFile objects in block outputs will be hydrated with base64 content
* before being stored in execution state. This ensures base64 is available for
@@ -50,7 +50,8 @@ describe('getIterationContext', () => {
const metadata: IterationNodeMetadata = {
branchIndex: 1,
branchTotal: 3,
parallelId: 'p1',
subflowId: 'p1',
subflowType: 'parallel',
}
const result = getIterationContext(ctx, metadata)
expect(result).toEqual({
@@ -77,7 +78,8 @@ describe('getIterationContext', () => {
})
const metadata: IterationNodeMetadata = {
isLoopNode: true,
loopId: 'l1',
subflowId: 'l1',
subflowType: 'loop',
}
const result = getIterationContext(ctx, metadata)
expect(result).toEqual({
@@ -124,7 +126,7 @@ describe('buildUnifiedParentIterations', () => {
it('resolves parallel-in-parallel parent chain', () => {
const ctx = makeCtx({
subflowParentMap: new Map([
['inner-p__obranch-2', { parentId: 'outer-p', parentType: 'parallel' }],
['inner-p__obranch-2', { parentId: 'outer-p', parentType: 'parallel', branchIndex: 2 }],
]),
parallelExecutions: new Map([
[
@@ -151,7 +153,7 @@ describe('buildUnifiedParentIterations', () => {
it('resolves loop-in-parallel (cross-type nesting)', () => {
const ctx = makeCtx({
subflowParentMap: new Map([
['loop-1__obranch-1', { parentId: 'parallel-1', parentType: 'parallel' }],
['loop-1__obranch-1', { parentId: 'parallel-1', parentType: 'parallel', branchIndex: 1 }],
]),
parallelExecutions: new Map([
[
@@ -205,8 +207,8 @@ describe('buildUnifiedParentIterations', () => {
const ctx = makeCtx({
subflowParentMap: new Map([
['inner-p', { parentId: 'mid-loop', parentType: 'loop' }],
['mid-loop', { parentId: 'outer-p', parentType: 'parallel' }],
['mid-loop__obranch-2', { parentId: 'outer-p', parentType: 'parallel' }],
['mid-loop', { parentId: 'outer-p', parentType: 'parallel', branchIndex: 0 }],
['mid-loop__obranch-2', { parentId: 'outer-p', parentType: 'parallel', branchIndex: 2 }],
]),
loopExecutions: new Map([
[
@@ -251,8 +253,8 @@ describe('buildUnifiedParentIterations', () => {
// P1 → P2 → P3, with P2__obranch-1 and P3__clone0__obranch-1
const ctx = makeCtx({
subflowParentMap: new Map([
['P2', { parentId: 'P1', parentType: 'parallel' }],
['P3', { parentId: 'P2', parentType: 'parallel' }],
['P2', { parentId: 'P1', parentType: 'parallel', branchIndex: 0 }],
['P3', { parentId: 'P2', parentType: 'parallel', branchIndex: 0 }],
['P2__obranch-1', { parentId: 'P1', parentType: 'parallel', branchIndex: 1 }],
[
'P3__clone0__obranch-1',
@@ -340,7 +342,7 @@ describe('buildUnifiedParentIterations', () => {
it('includes parent iterations in getIterationContext for loop-in-parallel', () => {
const ctx = makeCtx({
subflowParentMap: new Map([
['loop-1__obranch-2', { parentId: 'parallel-1', parentType: 'parallel' }],
['loop-1__obranch-2', { parentId: 'parallel-1', parentType: 'parallel', branchIndex: 2 }],
]),
parallelExecutions: new Map([
[
@@ -367,7 +369,8 @@ describe('buildUnifiedParentIterations', () => {
const metadata: IterationNodeMetadata = {
isLoopNode: true,
loopId: 'loop-1__obranch-2',
subflowId: 'loop-1__obranch-2',
subflowType: 'loop',
}
const result = getIterationContext(ctx, metadata)
expect(result).toEqual({
@@ -415,7 +418,8 @@ describe('buildUnifiedParentIterations', () => {
const metadata: IterationNodeMetadata = {
branchIndex: 1,
branchTotal: 3,
parallelId: 'parallel-1',
subflowId: 'parallel-1',
subflowType: 'parallel',
}
const result = getIterationContext(ctx, metadata)
expect(result).toEqual({
@@ -451,7 +455,7 @@ describe('buildContainerIterationContext', () => {
it('resolves loop nested inside parallel', () => {
const ctx = makeCtx({
subflowParentMap: new Map([
['loop-1__obranch-2', { parentId: 'parallel-1', parentType: 'parallel' }],
['loop-1__obranch-2', { parentId: 'parallel-1', parentType: 'parallel', branchIndex: 2 }],
]),
parallelExecutions: new Map([
[
@@ -499,7 +503,9 @@ describe('buildContainerIterationContext', () => {
it('returns undefined when parent scope is missing', () => {
const ctx = makeCtx({
subflowParentMap: new Map([['loop-1', { parentId: 'parallel-1', parentType: 'parallel' }]]),
subflowParentMap: new Map([
['loop-1', { parentId: 'parallel-1', parentType: 'parallel', branchIndex: 0 }],
]),
parallelExecutions: new Map(),
})
expect(buildContainerIterationContext(ctx, 'loop-1')).toBeUndefined()
@@ -537,7 +543,7 @@ describe('buildContainerIterationContext', () => {
it('uses branch index 0 for non-cloned container in parallel parent', () => {
const ctx = makeCtx({
subflowParentMap: new Map([
['inner-loop', { parentId: 'outer-parallel', parentType: 'parallel' }],
['inner-loop', { parentId: 'outer-parallel', parentType: 'parallel', branchIndex: 0 }],
]),
parallelExecutions: new Map([
[
+22 -36
View File
@@ -2,7 +2,7 @@ import { DEFAULTS } from '@/executor/constants'
import type { NodeMetadata } from '@/executor/dag/types'
import type { IterationContext, ParentIteration } from '@/executor/execution/types'
import type { ExecutionContext } from '@/executor/types'
import { extractOuterBranchIndex, findEffectiveContainerId } from '@/executor/utils/subflow-utils'
import { findEffectiveContainerId } from '@/executor/utils/subflow-utils'
/** Maximum ancestor depth to prevent runaway traversal in deeply nested subflows. */
const MAX_PARENT_DEPTH = DEFAULTS.MAX_NESTING_DEPTH
@@ -13,7 +13,7 @@ const MAX_PARENT_DEPTH = DEFAULTS.MAX_NESTING_DEPTH
*/
export type IterationNodeMetadata = Pick<
NodeMetadata,
'loopId' | 'parallelId' | 'branchIndex' | 'branchTotal' | 'isLoopNode'
'subflowType' | 'subflowId' | 'branchIndex' | 'branchTotal' | 'isLoopNode'
>
/**
@@ -28,27 +28,27 @@ export function getIterationContext(
if (!metadata) return undefined
if (metadata.branchIndex !== undefined && metadata.branchTotal !== undefined) {
const parentIterations = metadata.parallelId
? buildUnifiedParentIterations(ctx, metadata.parallelId)
: []
const parallelId = metadata.subflowType === 'parallel' ? metadata.subflowId : undefined
const parentIterations = parallelId ? buildUnifiedParentIterations(ctx, parallelId) : []
return {
iterationCurrent: metadata.branchIndex,
iterationTotal: metadata.branchTotal,
iterationType: 'parallel',
iterationContainerId: metadata.parallelId,
iterationContainerId: parallelId,
...(parentIterations.length > 0 && { parentIterations }),
}
}
if (metadata.isLoopNode && metadata.loopId) {
const loopScope = ctx.loopExecutions?.get(metadata.loopId)
const loopId = metadata.subflowType === 'loop' ? metadata.subflowId : undefined
if (metadata.isLoopNode && loopId) {
const loopScope = ctx.loopExecutions?.get(loopId)
if (loopScope && loopScope.iteration !== undefined) {
const parentIterations = buildUnifiedParentIterations(ctx, metadata.loopId)
const parentIterations = buildUnifiedParentIterations(ctx, loopId)
return {
iterationCurrent: loopScope.iteration,
iterationTotal: loopScope.maxIterations,
iterationType: 'loop',
iterationContainerId: metadata.loopId,
iterationContainerId: loopId,
...(parentIterations.length > 0 && { parentIterations }),
}
}
@@ -70,23 +70,15 @@ export function buildContainerIterationContext(
if (!parentEntry) return undefined
if (parentEntry.parentType === 'parallel') {
// Use stored parentId directly when branchIndex is available (set during expansion),
// otherwise fall back to findEffectiveContainerId for backward compatibility.
const hasBranchIndex = parentEntry.branchIndex !== undefined
const effectiveParentId = hasBranchIndex
? parentEntry.parentId
: ctx.parallelExecutions
? findEffectiveContainerId(parentEntry.parentId, containerId, ctx.parallelExecutions)
: parentEntry.parentId
const parentScope = ctx.parallelExecutions?.get(effectiveParentId)
if (parentScope) {
if (parentEntry.branchIndex !== undefined) {
const parentScope = ctx.parallelExecutions?.get(parentEntry.parentId)
if (!parentScope) return undefined
return {
iterationCurrent: hasBranchIndex
? parentEntry.branchIndex!
: (extractOuterBranchIndex(containerId) ?? 0),
iterationCurrent: parentEntry.branchIndex,
iterationTotal: parentScope.totalBranches,
iterationType: 'parallel',
iterationContainerId: effectiveParentId,
iterationContainerId: parentEntry.parentId,
}
}
} else if (parentEntry.parentType === 'loop') {
@@ -150,21 +142,15 @@ export function buildUnifiedParentIterations(
})
}
} else {
// Use stored parentId directly when branchIndex is available (set during expansion),
// otherwise fall back to findEffectiveContainerId for backward compatibility.
const hasBranchIndex = entry.branchIndex !== undefined
const effectiveParentId = hasBranchIndex
? parentId
: ctx.parallelExecutions
? findEffectiveContainerId(parentId, currentId, ctx.parallelExecutions)
: parentId
if (entry.branchIndex === undefined) {
currentId = parentId
continue
}
const effectiveParentId = parentId
const parentScope = ctx.parallelExecutions?.get(effectiveParentId)
if (parentScope) {
const outerBranchIndex = hasBranchIndex
? entry.branchIndex!
: (extractOuterBranchIndex(currentId) ?? 0)
parents.unshift({
iterationCurrent: outerBranchIndex,
iterationCurrent: entry.branchIndex,
iterationTotal: parentScope.totalBranches,
iterationType: 'parallel',
iterationContainerId: effectiveParentId,
@@ -10,6 +10,7 @@ import {
buildBranchNodeId,
buildParallelSentinelEndId,
buildParallelSentinelStartId,
buildSentinelStartId,
stripCloneSuffixes,
} from '@/executor/utils/subflow-utils'
import type { SerializedBlock, SerializedWorkflow } from '@/serializer/types'
@@ -27,6 +28,55 @@ function createBlock(id: string, metadataId: string): SerializedBlock {
}
describe('Nested parallel expansion + edge resolution', () => {
it('waits for every branch terminal before queuing the parallel end sentinel', () => {
const parallelId = 'parallel-1'
const taskId = 'task-1'
const workflow: SerializedWorkflow = {
version: '1',
blocks: [
createBlock('start', BlockType.STARTER),
createBlock(parallelId, BlockType.PARALLEL),
createBlock(taskId, BlockType.FUNCTION),
],
connections: [
{ source: 'start', target: parallelId },
{
source: parallelId,
target: taskId,
sourceHandle: 'parallel-start-source',
},
],
loops: {},
parallels: {
[parallelId]: {
id: parallelId,
nodes: [taskId],
count: 2,
parallelType: 'count',
},
},
}
const dag = new DAGBuilder().build(workflow)
const expander = new ParallelExpander()
const { terminalNodes } = expander.expandParallel(dag, parallelId, 2)
const edgeManager = new EdgeManager(dag)
const parallelEndId = buildParallelSentinelEndId(parallelId)
expect(terminalNodes).toEqual([buildBranchNodeId(taskId, 0), buildBranchNodeId(taskId, 1)])
expect(dag.nodes.get(parallelEndId)?.incomingEdges).toEqual(new Set(terminalNodes))
const firstBranchReady = edgeManager.processOutgoingEdges(dag.nodes.get(terminalNodes[0])!, {
value: 'first',
})
expect(firstBranchReady).not.toContain(parallelEndId)
const secondBranchReady = edgeManager.processOutgoingEdges(dag.nodes.get(terminalNodes[1])!, {
value: 'second',
})
expect(secondBranchReady).toContain(parallelEndId)
})
it('outer parallel expansion clones inner subflow per branch and edge manager resolves correctly', () => {
const outerParallelId = 'outer-parallel'
const innerParallelId = 'inner-parallel'
@@ -105,6 +155,10 @@ describe('Nested parallel expansion + edge resolution', () => {
const clonedInnerEndId = buildParallelSentinelEndId(clonedInnerParallelId)
expect(outerStartTargets).toContain(clonedInnerStartId) // branch 1
expect(dag.nodes.get(clonedInnerStartId)?.metadata).toMatchObject({
subflowId: clonedInnerParallelId,
subflowType: 'parallel',
})
// Verify cloned parallel config was registered
expect(dag.parallelConfigs.has(clonedInnerParallelId)).toBe(true)
@@ -207,6 +261,78 @@ describe('Nested parallel expansion + edge resolution', () => {
expect(readyAfterClonedInnerEnd).toContain(outerEndId)
})
it('preserves regular-to-nested subflow dependencies across expanded branches', () => {
const outerParallelId = 'outer-parallel'
const innerParallelId = 'inner-parallel'
const prepareId = 'prepare'
const finishId = 'finish'
const innerTaskId = 'inner-task'
const workflow: SerializedWorkflow = {
version: '1',
blocks: [
createBlock('start', BlockType.STARTER),
createBlock(outerParallelId, BlockType.PARALLEL),
createBlock(prepareId, BlockType.FUNCTION),
createBlock(innerParallelId, BlockType.PARALLEL),
createBlock(innerTaskId, BlockType.FUNCTION),
createBlock(finishId, BlockType.FUNCTION),
],
connections: [
{ source: 'start', target: outerParallelId },
{ source: outerParallelId, target: prepareId, sourceHandle: 'parallel-start-source' },
{ source: prepareId, target: innerParallelId },
{ source: innerParallelId, target: finishId },
{ source: innerParallelId, target: innerTaskId, sourceHandle: 'parallel-start-source' },
],
loops: {},
parallels: {
[innerParallelId]: {
id: innerParallelId,
nodes: [innerTaskId],
count: 1,
parallelType: 'count',
},
[outerParallelId]: {
id: outerParallelId,
nodes: [prepareId, innerParallelId, finishId],
count: 2,
parallelType: 'count',
},
},
}
const dag = new DAGBuilder().build(workflow)
const expander = new ParallelExpander()
const result = expander.expandParallel(dag, outerParallelId, 2)
const clonedInnerId = result.clonedSubflows[0].clonedId
const prepareBranchOne = dag.nodes.get(buildBranchNodeId(prepareId, 1))!
const clonedInnerStart = buildParallelSentinelStartId(clonedInnerId)
const clonedInnerEnd = buildParallelSentinelEndId(clonedInnerId)
const finishBranchOne = buildBranchNodeId(finishId, 1)
expect(
Array.from(prepareBranchOne.outgoingEdges.values()).map((edge) => edge.target)
).toContain(clonedInnerStart)
expect(
Array.from(dag.nodes.get(clonedInnerEnd)!.outgoingEdges.values()).map((edge) => edge.target)
).toContain(finishBranchOne)
expect(
Array.from(dag.nodes.get(clonedInnerEnd)!.outgoingEdges.values()).map((edge) => edge.target)
).not.toContain(buildBranchNodeId(finishId, 0))
const outerStartTargets = Array.from(
dag.nodes.get(buildParallelSentinelStartId(outerParallelId))!.outgoingEdges.values()
).map((edge) => edge.target)
expect(outerStartTargets).toEqual([buildBranchNodeId(prepareId, 0), prepareBranchOne.id])
const outerEndIncoming = dag.nodes.get(
buildParallelSentinelEndId(outerParallelId)
)!.incomingEdges
expect(outerEndIncoming.has(buildBranchNodeId(finishId, 0))).toBe(true)
expect(outerEndIncoming.has(finishBranchOne)).toBe(true)
})
it('uses global branch indexes for nested subflow clones in later batches', () => {
const outerParallelId = 'outer-parallel'
const innerParallelId = 'inner-parallel'
@@ -266,6 +392,241 @@ describe('Nested parallel expansion + edge resolution', () => {
])
})
it('clears stale regular-to-nested edges when local branch nodes are reused in later batches', () => {
const outerParallelId = 'outer-parallel'
const innerParallelId = 'inner-parallel'
const prepareId = 'prepare'
const innerTaskId = 'inner-task'
const workflow: SerializedWorkflow = {
version: '1',
blocks: [
createBlock('start', BlockType.STARTER),
createBlock(outerParallelId, BlockType.PARALLEL),
createBlock(prepareId, BlockType.FUNCTION),
createBlock(innerParallelId, BlockType.PARALLEL),
createBlock(innerTaskId, BlockType.FUNCTION),
],
connections: [
{ source: 'start', target: outerParallelId },
{ source: outerParallelId, target: prepareId, sourceHandle: 'parallel-start-source' },
{ source: prepareId, target: innerParallelId },
{ source: innerParallelId, target: innerTaskId, sourceHandle: 'parallel-start-source' },
],
loops: {},
parallels: {
[innerParallelId]: {
id: innerParallelId,
nodes: [innerTaskId],
count: 1,
parallelType: 'count',
},
[outerParallelId]: {
id: outerParallelId,
nodes: [prepareId, innerParallelId],
count: 5,
parallelType: 'count',
},
},
}
const dag = new DAGBuilder().build(workflow)
const expander = new ParallelExpander()
expander.expandParallel(dag, outerParallelId, 2, undefined, {
branchIndexOffset: 0,
totalBranches: 5,
})
expander.expandParallel(dag, outerParallelId, 2, undefined, {
branchIndexOffset: 2,
totalBranches: 5,
})
expander.expandParallel(dag, outerParallelId, 1, undefined, {
branchIndexOffset: 4,
totalBranches: 5,
})
const prepareBranchZero = dag.nodes.get(buildBranchNodeId(prepareId, 0))!
const originalInnerStart = buildParallelSentinelStartId(innerParallelId)
const staleClonedInnerStart = buildParallelSentinelStartId(`${innerParallelId}__obranch-2`)
const clonedInnerStart = buildParallelSentinelStartId(`${innerParallelId}__obranch-4`)
const prepareTargets = Array.from(prepareBranchZero.outgoingEdges.values()).map(
(edge) => edge.target
)
expect(prepareTargets).toEqual([clonedInnerStart])
expect(dag.nodes.get(originalInnerStart)!.incomingEdges.has(prepareBranchZero.id)).toBe(false)
expect(dag.nodes.get(staleClonedInnerStart)!.incomingEdges.has(prepareBranchZero.id)).toBe(
false
)
expect(dag.nodes.get(clonedInnerStart)!.incomingEdges.has(prepareBranchZero.id)).toBe(true)
})
it('preserves internal edge topology when expanding cloned nested parallels', () => {
const outerParallelId = 'outer-parallel'
const innerParallelId = 'inner-parallel'
const firstTaskId = 'first-task'
const secondTaskId = 'second-task'
const workflow: SerializedWorkflow = {
version: '1',
blocks: [
createBlock('start', BlockType.STARTER),
createBlock(outerParallelId, BlockType.PARALLEL),
createBlock(innerParallelId, BlockType.PARALLEL),
createBlock(firstTaskId, BlockType.FUNCTION),
createBlock(secondTaskId, BlockType.FUNCTION),
],
connections: [
{ source: 'start', target: outerParallelId },
{ source: outerParallelId, target: innerParallelId, sourceHandle: 'parallel-start-source' },
{ source: innerParallelId, target: firstTaskId, sourceHandle: 'parallel-start-source' },
{ source: firstTaskId, target: secondTaskId },
],
loops: {},
parallels: {
[innerParallelId]: {
id: innerParallelId,
nodes: [firstTaskId, secondTaskId],
count: 2,
parallelType: 'count',
},
[outerParallelId]: {
id: outerParallelId,
nodes: [innerParallelId],
count: 3,
parallelType: 'count',
},
},
}
const dag = new DAGBuilder().build(workflow)
const expander = new ParallelExpander()
const outerResult = expander.expandParallel(dag, outerParallelId, 1, undefined, {
branchIndexOffset: 2,
totalBranches: 3,
})
const clonedInnerId = outerResult.clonedSubflows[0].clonedId
const clonedInnerConfig = dag.parallelConfigs.get(clonedInnerId)!
const [clonedFirstTaskId, clonedSecondTaskId] = clonedInnerConfig.nodes
const innerResult = expander.expandParallel(dag, clonedInnerId, 2)
const firstTaskBranchOneId = buildBranchNodeId(clonedFirstTaskId, 1)
const secondTaskBranchOneId = buildBranchNodeId(clonedSecondTaskId, 1)
const firstTaskBranchOne = dag.nodes.get(firstTaskBranchOneId)!
const secondTaskBranchOne = dag.nodes.get(secondTaskBranchOneId)!
expect(innerResult.entryNodes).toEqual([
buildBranchNodeId(clonedFirstTaskId, 0),
firstTaskBranchOneId,
])
expect(
Array.from(firstTaskBranchOne.outgoingEdges.values()).map((edge) => edge.target)
).toContain(secondTaskBranchOneId)
expect(secondTaskBranchOne.incomingEdges.has(firstTaskBranchOneId)).toBe(true)
expect(innerResult.entryNodes).not.toContain(secondTaskBranchOneId)
})
it('clears stale sentinel-end incoming edges when a later batch is smaller', () => {
const parallelId = 'parallel-1'
const functionId = 'func-1'
const workflow: SerializedWorkflow = {
version: '1',
blocks: [
createBlock('start', BlockType.STARTER),
createBlock(parallelId, BlockType.PARALLEL),
createBlock(functionId, BlockType.FUNCTION),
],
connections: [
{ source: 'start', target: parallelId },
{ source: parallelId, target: functionId, sourceHandle: 'parallel-start-source' },
],
loops: {},
parallels: {
[parallelId]: {
id: parallelId,
nodes: [functionId],
count: 4,
parallelType: 'count',
},
},
}
const builder = new DAGBuilder()
const dag = builder.build(workflow)
const expander = new ParallelExpander()
const sentinelEnd = dag.nodes.get(buildParallelSentinelEndId(parallelId))!
expander.expandParallel(dag, parallelId, 3, undefined, {
branchIndexOffset: 0,
totalBranches: 4,
})
expect(sentinelEnd.incomingEdges).toEqual(
new Set([
buildBranchNodeId(functionId, 0),
buildBranchNodeId(functionId, 1),
buildBranchNodeId(functionId, 2),
])
)
expander.expandParallel(dag, parallelId, 1, undefined, {
branchIndexOffset: 3,
totalBranches: 4,
})
expect(sentinelEnd.incomingEdges).toEqual(new Set([buildBranchNodeId(functionId, 0)]))
})
it('updates unified subflow metadata on cloned nested loop sentinels', () => {
const parallelId = 'parallel-1'
const loopId = 'loop-1'
const functionId = 'func-1'
const workflow: SerializedWorkflow = {
version: '1',
blocks: [
createBlock('start', BlockType.STARTER),
createBlock(parallelId, BlockType.PARALLEL),
createBlock(loopId, BlockType.LOOP),
createBlock(functionId, BlockType.FUNCTION),
],
connections: [
{ source: 'start', target: parallelId },
{ source: parallelId, target: loopId, sourceHandle: 'parallel-start-source' },
{ source: loopId, target: functionId, sourceHandle: 'loop-start-source' },
],
loops: {
[loopId]: {
id: loopId,
nodes: [functionId],
iterations: 1,
loopType: 'for',
},
},
parallels: {
[parallelId]: {
id: parallelId,
nodes: [loopId],
count: 2,
parallelType: 'count',
},
},
}
const builder = new DAGBuilder()
const dag = builder.build(workflow)
const expander = new ParallelExpander()
const result = expander.expandParallel(dag, parallelId, 2)
const clonedLoopId = result.clonedSubflows.find(
(clone) => clone.originalId === loopId
)?.clonedId
expect(clonedLoopId).toBe(`${loopId}__obranch-1`)
expect(dag.nodes.get(buildSentinelStartId(clonedLoopId!))?.metadata).toMatchObject({
subflowId: clonedLoopId,
subflowType: 'loop',
})
})
it('3-level nesting: pre-expansion clone IDs do not collide with runtime expansion', () => {
const p1 = 'p1'
const p2 = 'p2'
@@ -312,6 +673,11 @@ describe('Nested parallel expansion + edge resolution', () => {
expect(p3Clone).toBeDefined()
expect(p3Clone.clonedId).toMatch(/^p3__clone[0-9a-f]{24}__obranch-1$/)
expect(stripCloneSuffixes(p3Clone.clonedId)).toBe('p3')
expect(
Array.from(
dag.nodes.get(buildParallelSentinelEndId(p3Clone.clonedId))!.outgoingEdges.values()
).map((edge) => edge.target)
).toContain(buildParallelSentinelEndId(p2Clone.clonedId))
// Step 2: Expand P2 (original, branch 0 of P1) — this creates P3__obranch-1 at runtime
const p2Result = expander.expandParallel(dag, p2, 2)
+217 -110
View File
@@ -1,8 +1,7 @@
import { createLogger } from '@sim/logger'
import { sha256Hex } from '@sim/security/hash'
import { EDGE } from '@/executor/constants'
import { CONTROL_BACK_EDGE_HANDLES, EDGE } from '@/executor/constants'
import type { DAG, DAGNode } from '@/executor/dag/builder'
import type { SerializedBlock } from '@/serializer/types'
import {
buildBranchNodeId,
buildClonedSubflowId,
@@ -12,7 +11,11 @@ import {
buildSentinelStartId,
extractBaseBlockId,
isLoopSentinelNodeId,
} from './subflow-utils'
isParallelSentinelNodeId,
normalizeNodeId,
stripOuterBranchSuffix,
} from '@/executor/utils/subflow-utils'
import type { SerializedBlock } from '@/serializer/types'
const logger = createLogger('ParallelExpansion')
@@ -61,7 +64,6 @@ export class ParallelExpander {
}
}
const regularSet = new Set(regularBlocks)
const allBranchNodes: string[] = []
const branchIndexOffset = options.branchIndexOffset ?? 0
const branchTotal = options.totalBranches ?? branchCount
@@ -102,38 +104,19 @@ export class ParallelExpander {
}
}
this.wireInternalEdges(dag, regularBlocks, regularSet, branchCount)
const { entryNodes, terminalNodes } =
regularBlocks.length > 0
? this.identifyBoundaryNodes(dag, regularBlocks, regularSet, branchCount)
: { entryNodes: [] as string[], terminalNodes: [] as string[] }
// Clone nested subflow graphs per outer branch so each branch runs independently.
// Branch 0 uses the original sentinel/template nodes; branches 1..N get full clones.
const clonedSubflows: ClonedSubflowInfo[] = []
for (const subflowId of nestedSubflows) {
const isParallel = dag.parallelConfigs.has(subflowId)
const startId = isParallel
? buildParallelSentinelStartId(subflowId)
: buildSentinelStartId(subflowId)
const endId = isParallel
? buildParallelSentinelEndId(subflowId)
: buildSentinelEndId(subflowId)
for (let i = 0; i < branchCount; i++) {
const globalBranchIndex = branchIndexOffset + i
if (globalBranchIndex === 0) {
if (dag.nodes.has(startId)) entryNodes.push(startId)
if (dag.nodes.has(endId)) terminalNodes.push(endId)
continue
}
const cloned = this.cloneNestedSubflow(dag, subflowId, globalBranchIndex, clonedSubflows)
entryNodes.push(cloned.startId)
terminalNodes.push(cloned.endId)
clonedSubflows.push({
clonedId: cloned.clonedId,
originalId: subflowId,
@@ -142,6 +125,20 @@ export class ParallelExpander {
}
}
this.wireInternalEdges(
dag,
blocksInParallel,
new Set(blocksInParallel),
branchCount,
branchIndexOffset
)
const { entryNodes, terminalNodes } = this.identifyBoundaryNodes(
dag,
blocksInParallel,
branchCount,
branchIndexOffset
)
this.wireSentinelEdges(dag, parallelId, entryNodes, terminalNodes, branchCount)
logger.info('Parallel expanded', {
@@ -201,68 +198,95 @@ export class ParallelExpander {
dag: DAG,
blocksInParallel: string[],
blocksSet: Set<string>,
branchCount: number
branchCount: number,
branchIndexOffset: number
): void {
for (const blockId of blocksInParallel) {
const templateId = buildBranchNodeId(blockId, 0)
const templateNode = dag.nodes.get(templateId)
if (!templateNode) continue
const topology = this.collectInternalEdgeTopology(dag, blocksInParallel, blocksSet)
const cleanedSourceNodes = new Set<string>()
for (const [, edge] of templateNode.outgoingEdges) {
const baseTargetId = extractBaseBlockId(edge.target)
if (!blocksSet.has(baseTargetId)) continue
for (const edge of topology) {
for (let i = 0; i < branchCount; i++) {
const globalBranchIndex = branchIndexOffset + i
const sourceNodeId = this.resolveBranchChildBoundaryId(
dag,
edge.sourceBlockId,
i,
globalBranchIndex,
'end'
)
const targetNodeId = this.resolveBranchChildBoundaryId(
dag,
edge.targetBlockId,
i,
globalBranchIndex,
'start'
)
const sourceNode = dag.nodes.get(sourceNodeId)
const targetNode = dag.nodes.get(targetNodeId)
// Include branch 0 so per-batch re-expansion restores the template's
// incoming-edge bookkeeping that earlier batches consumed during
// edge processing. Without this, identifyBoundaryNodes mis-classifies
// chained children as entry nodes after the first batch.
for (let i = 0; i < branchCount; i++) {
const sourceNodeId = buildBranchNodeId(blockId, i)
const targetNodeId = buildBranchNodeId(baseTargetId, i)
const sourceNode = dag.nodes.get(sourceNodeId)
const targetNode = dag.nodes.get(targetNodeId)
if (!sourceNode || !targetNode) continue
if (!sourceNode || !targetNode) continue
const edgeId = edge.sourceHandle
? `${sourceNodeId}→${targetNodeId}-${edge.sourceHandle}`
: `${sourceNodeId}→${targetNodeId}`
sourceNode.outgoingEdges.set(edgeId, {
target: targetNodeId,
sourceHandle: edge.sourceHandle,
targetHandle: edge.targetHandle,
})
targetNode.incomingEdges.add(sourceNodeId)
if (!cleanedSourceNodes.has(sourceNodeId)) {
this.clearStaleInternalEdges(dag, sourceNodeId, sourceNode, blocksSet)
cleanedSourceNodes.add(sourceNodeId)
}
const edgeId = edge.sourceHandle
? `${sourceNodeId}→${targetNodeId}-${edge.sourceHandle}`
: `${sourceNodeId}→${targetNodeId}`
sourceNode.outgoingEdges.set(edgeId, {
target: targetNodeId,
sourceHandle: edge.sourceHandle,
targetHandle: edge.targetHandle,
})
targetNode.incomingEdges.add(sourceNodeId)
}
}
}
private clearStaleInternalEdges(
dag: DAG,
sourceNodeId: string,
sourceNode: DAGNode,
blocksSet: Set<string>
): void {
for (const [edgeId, edge] of Array.from(sourceNode.outgoingEdges.entries())) {
if (!blocksSet.has(stripOuterBranchSuffix(normalizeNodeId(edge.target)))) continue
sourceNode.outgoingEdges.delete(edgeId)
dag.nodes.get(edge.target)?.incomingEdges.delete(sourceNodeId)
}
}
private identifyBoundaryNodes(
dag: DAG,
blocksInParallel: string[],
blocksSet: Set<string>,
branchCount: number
branchCount: number,
branchIndexOffset: number
): { entryNodes: string[]; terminalNodes: string[] } {
const entryNodes: string[] = []
const terminalNodes: string[] = []
const topology = this.collectInternalEdgeTopology(
dag,
blocksInParallel,
new Set(blocksInParallel)
)
const blocksWithInternalIncoming = new Set(topology.map((edge) => edge.targetBlockId))
const blocksWithInternalOutgoing = new Set(topology.map((edge) => edge.sourceBlockId))
for (const blockId of blocksInParallel) {
const templateId = buildBranchNodeId(blockId, 0)
const templateNode = dag.nodes.get(templateId)
if (!templateNode) continue
const hasInternalIncoming = this.hasInternalIncomingEdge(templateNode, blocksSet)
const hasInternalOutgoing = this.hasInternalOutgoingEdge(templateNode, blocksSet)
for (let i = 0; i < branchCount; i++) {
const branchNodeId = buildBranchNodeId(blockId, i)
if (!hasInternalIncoming) {
entryNodes.push(branchNodeId)
const globalBranchIndex = branchIndexOffset + i
if (!blocksWithInternalIncoming.has(blockId)) {
entryNodes.push(
this.resolveBranchChildBoundaryId(dag, blockId, i, globalBranchIndex, 'start')
)
}
if (!hasInternalOutgoing) {
terminalNodes.push(branchNodeId)
if (!blocksWithInternalOutgoing.has(blockId)) {
terminalNodes.push(
this.resolveBranchChildBoundaryId(dag, blockId, i, globalBranchIndex, 'end')
)
}
}
}
@@ -270,24 +294,82 @@ export class ParallelExpander {
return { entryNodes, terminalNodes }
}
private hasInternalIncomingEdge(node: DAGNode, blocksSet: Set<string>): boolean {
for (const incomingId of node.incomingEdges) {
const baseId = extractBaseBlockId(incomingId)
if (blocksSet.has(baseId)) {
return true
private collectInternalEdgeTopology(
dag: DAG,
blocksInParallel: string[],
blocksSet: Set<string>
): Array<{
sourceBlockId: string
targetBlockId: string
sourceHandle?: string
targetHandle?: string
}> {
const topology: Array<{
sourceBlockId: string
targetBlockId: string
sourceHandle?: string
targetHandle?: string
}> = []
for (const blockId of blocksInParallel) {
const sourceNodeId = this.resolveOriginalChildBoundaryId(dag, blockId, 'end')
const sourceNode = dag.nodes.get(sourceNodeId)
if (!sourceNode) continue
for (const [, edge] of sourceNode.outgoingEdges) {
if (edge.sourceHandle && CONTROL_BACK_EDGE_HANDLES.has(edge.sourceHandle)) continue
const targetBoundaryId = extractBaseBlockId(normalizeNodeId(edge.target))
const targetBlockId = blocksSet.has(targetBoundaryId)
? targetBoundaryId
: stripOuterBranchSuffix(targetBoundaryId)
if (!blocksSet.has(targetBlockId)) continue
topology.push({
sourceBlockId: blockId,
targetBlockId,
sourceHandle: edge.sourceHandle,
targetHandle: edge.targetHandle,
})
}
}
return false
return topology
}
private hasInternalOutgoingEdge(node: DAGNode, blocksSet: Set<string>): boolean {
for (const [, edge] of node.outgoingEdges) {
const baseId = extractBaseBlockId(edge.target)
if (blocksSet.has(baseId)) {
return true
}
private resolveOriginalChildBoundaryId(dag: DAG, blockId: string, side: 'start' | 'end'): string {
if (dag.parallelConfigs.has(blockId)) {
return side === 'start'
? buildParallelSentinelStartId(blockId)
: buildParallelSentinelEndId(blockId)
}
return false
if (dag.loopConfigs.has(blockId)) {
return side === 'start' ? buildSentinelStartId(blockId) : buildSentinelEndId(blockId)
}
return buildBranchNodeId(blockId, 0)
}
private resolveBranchChildBoundaryId(
dag: DAG,
blockId: string,
localBranchIndex: number,
globalBranchIndex: number,
side: 'start' | 'end'
): string {
if (!dag.parallelConfigs.has(blockId) && !dag.loopConfigs.has(blockId)) {
return buildBranchNodeId(blockId, localBranchIndex)
}
const effectiveSubflowId =
globalBranchIndex === 0 ? blockId : buildClonedSubflowId(blockId, globalBranchIndex)
if (dag.parallelConfigs.has(blockId)) {
return side === 'start'
? buildParallelSentinelStartId(effectiveSubflowId)
: buildParallelSentinelEndId(effectiveSubflowId)
}
return side === 'start'
? buildSentinelStartId(effectiveSubflowId)
: buildSentinelEndId(effectiveSubflowId)
}
/**
@@ -440,53 +522,34 @@ export class ParallelExpander {
if (!origNode) continue
const clonedNodeId = idMap.get(origId)!
this.cloneDAGNode(dag, origNode, clonedNodeId, clonedId, isParallel, idMap)
this.cloneDAGNode(dag, origNode, clonedNodeId, clonedId, isParallel)
}
this.remapClonedEdges(dag, idMap)
return { startId: clonedStartId, endId: clonedEndId, idMap }
}
/**
* Clones a single DAG node with remapped edges and updated metadata.
* Clones a single DAG node with updated metadata. Edge wiring is owned by remapClonedEdges
* after the full clone id map is available.
*/
private cloneDAGNode(
dag: DAG,
origNode: DAGNode,
clonedNodeId: string,
parentClonedId: string,
parentIsParallel: boolean,
idMap: Map<string, string>
parentIsParallel: boolean
): void {
const clonedOutgoing = new Map<
string,
{ target: string; sourceHandle?: string; targetHandle?: string }
>()
for (const [, edge] of origNode.outgoingEdges) {
const clonedTarget = idMap.get(edge.target) ?? edge.target
const edgeId = edge.sourceHandle
? `${clonedNodeId}→${clonedTarget}-${edge.sourceHandle}`
: `${clonedNodeId}→${clonedTarget}`
clonedOutgoing.set(edgeId, {
target: clonedTarget,
sourceHandle: edge.sourceHandle,
targetHandle: edge.targetHandle,
})
}
const clonedIncoming = new Set<string>()
for (const incomingId of origNode.incomingEdges) {
clonedIncoming.add(idMap.get(incomingId) ?? incomingId)
}
const metadataOverride = parentIsParallel
? { parallelId: parentClonedId }
: { loopId: parentClonedId }
? { subflowId: parentClonedId, subflowType: 'parallel' as const }
: { subflowId: parentClonedId, subflowType: 'loop' as const }
dag.nodes.set(clonedNodeId, {
id: clonedNodeId,
block: { ...origNode.block, id: clonedNodeId },
incomingEdges: clonedIncoming,
outgoingEdges: clonedOutgoing,
incomingEdges: new Set(),
outgoingEdges: new Map(),
metadata: {
...origNode.metadata,
...metadataOverride,
@@ -497,6 +560,43 @@ export class ParallelExpander {
})
}
private remapClonedEdges(dag: DAG, idMap: Map<string, string>): void {
for (const [origId, clonedNodeId] of idMap) {
const origNode = dag.nodes.get(origId)
const clonedNode = dag.nodes.get(clonedNodeId)
if (!origNode || !clonedNode) continue
const remappedOutgoing = new Map<
string,
{ target: string; sourceHandle?: string; targetHandle?: string }
>()
for (const [, edge] of origNode.outgoingEdges) {
const clonedTarget = idMap.get(edge.target)
if (!clonedTarget) continue
const edgeId = edge.sourceHandle
? `${clonedNodeId}→${clonedTarget}-${edge.sourceHandle}`
: `${clonedNodeId}→${clonedTarget}`
remappedOutgoing.set(edgeId, {
target: clonedTarget,
sourceHandle: edge.sourceHandle,
targetHandle: edge.targetHandle,
})
}
const remappedIncoming = new Set<string>()
for (const incomingId of origNode.incomingEdges) {
const clonedIncomingId = idMap.get(incomingId)
if (clonedIncomingId) {
remappedIncoming.add(clonedIncomingId)
}
}
clonedNode.outgoingEdges = remappedOutgoing
clonedNode.incomingEdges = remappedIncoming
}
}
private wireSentinelEdges(
dag: DAG,
parallelId: string,
@@ -515,6 +615,7 @@ export class ParallelExpander {
}
sentinelStart.outgoingEdges.clear()
sentinelEnd.incomingEdges.clear()
for (const entryNodeId of entryNodes) {
const entryNode = dag.nodes.get(entryNodeId)
if (!entryNode) continue
@@ -528,8 +629,14 @@ export class ParallelExpander {
const terminalNode = dag.nodes.get(terminalNodeId)
if (!terminalNode) continue
const handle = isLoopSentinelNodeId(terminalNodeId) ? EDGE.LOOP_EXIT : EDGE.PARALLEL_EXIT
const edgeId = `${terminalNodeId}→${sentinelEndId}-${handle}`
const handle = isLoopSentinelNodeId(terminalNodeId)
? EDGE.LOOP_EXIT
: isParallelSentinelNodeId(terminalNodeId)
? EDGE.PARALLEL_EXIT
: undefined
const edgeId = handle
? `${terminalNodeId}→${sentinelEndId}-${handle}`
: `${terminalNodeId}→${sentinelEndId}`
terminalNode.outgoingEdges.set(edgeId, {
target: sentinelEndId,
sourceHandle: handle,
+149 -20
View File
@@ -287,7 +287,13 @@ describe('validateRunFromBlock', () => {
})
it('rejects blocks inside loops', () => {
const dag = createDAG([createNode('A', [], { isLoopNode: true, loopId: 'loop-1' })])
const dag = createDAG([
createNode('A', [], {
isLoopNode: true,
subflowId: 'loop-1',
subflowType: 'loop',
}),
])
const executedBlocks = new Set(['A'])
const result = validateRunFromBlock('A', dag, executedBlocks)
@@ -299,7 +305,11 @@ describe('validateRunFromBlock', () => {
it('rejects blocks inside parallels', () => {
const dag = createDAG([
createNode('A', [], { isParallelBranch: true, parallelId: 'parallel-1' }),
createNode('A', [], {
isParallelBranch: true,
subflowId: 'parallel-1',
subflowType: 'parallel',
}),
])
const executedBlocks = new Set(['A'])
@@ -387,13 +397,19 @@ describe('validateRunFromBlock', () => {
createNode(sentinelStartId, [{ target: 'B' }], {
isSentinel: true,
sentinelType: 'start',
loopId,
subflowId: loopId,
subflowType: 'loop',
}),
createNode('B', [{ target: sentinelEndId }], {
isLoopNode: true,
subflowId: loopId,
subflowType: 'loop',
}),
createNode('B', [{ target: sentinelEndId }], { isLoopNode: true, loopId }),
createNode(sentinelEndId, [{ target: 'C' }], {
isSentinel: true,
sentinelType: 'end',
loopId,
subflowId: loopId,
subflowType: 'loop',
}),
createNode('C'),
])
@@ -405,6 +421,69 @@ describe('validateRunFromBlock', () => {
expect(result.valid).toBe(true)
})
it('rejects loop containers nested inside another loop', () => {
const outerLoopId = 'outer-loop'
const innerLoopId = 'inner-loop'
const innerStartId = `loop-${innerLoopId}-sentinel-start`
const dag = createDAG([
createNode(innerStartId, [], {
isSentinel: true,
sentinelType: 'start',
subflowId: innerLoopId,
subflowType: 'loop',
}),
])
dag.loopConfigs.set(outerLoopId, {
id: outerLoopId,
nodes: [innerLoopId],
iterations: 2,
loopType: 'for',
} as any)
dag.loopConfigs.set(innerLoopId, {
id: innerLoopId,
nodes: ['B'],
iterations: 2,
loopType: 'for',
} as any)
const result = validateRunFromBlock(innerLoopId, dag, new Set([innerStartId]))
expect(result.valid).toBe(false)
expect(result.error).toContain('inside loop')
expect(result.error).toContain(outerLoopId)
})
it('rejects containers nested inside a parallel', () => {
const outerParallelId = 'outer-parallel'
const innerLoopId = 'inner-loop'
const innerStartId = `loop-${innerLoopId}-sentinel-start`
const dag = createDAG([
createNode(innerStartId, [], {
isSentinel: true,
sentinelType: 'start',
subflowId: innerLoopId,
subflowType: 'loop',
}),
])
dag.parallelConfigs.set(outerParallelId, {
id: outerParallelId,
nodes: [innerLoopId],
count: 2,
} as any)
dag.loopConfigs.set(innerLoopId, {
id: innerLoopId,
nodes: ['B'],
iterations: 2,
loopType: 'for',
} as any)
const result = validateRunFromBlock(innerLoopId, dag, new Set([innerStartId]))
expect(result.valid).toBe(false)
expect(result.error).toContain('inside parallel')
expect(result.error).toContain(outerParallelId)
})
it('accepts parallel container when executed', () => {
// Parallel container with sentinel nodes
const parallelId = 'parallel-container-1'
@@ -415,13 +494,19 @@ describe('validateRunFromBlock', () => {
createNode(sentinelStartId, [{ target: 'B₍0₎' }], {
isSentinel: true,
sentinelType: 'start',
parallelId,
subflowId: parallelId,
subflowType: 'parallel',
}),
createNode('B₍0₎', [{ target: sentinelEndId }], {
isParallelBranch: true,
subflowId: parallelId,
subflowType: 'parallel',
}),
createNode('B₍0₎', [{ target: sentinelEndId }], { isParallelBranch: true, parallelId }),
createNode(sentinelEndId, [{ target: 'C' }], {
isSentinel: true,
sentinelType: 'end',
parallelId,
subflowId: parallelId,
subflowType: 'parallel',
}),
createNode('C'),
])
@@ -433,13 +518,40 @@ describe('validateRunFromBlock', () => {
expect(result.valid).toBe(true)
})
it('rejects container when sentinel-start upstream dependency was not executed', () => {
const loopId = 'loop-container-1'
const sentinelStartId = `loop-${loopId}-sentinel-start`
const dag = createDAG([
createNode('A', [{ target: sentinelStartId }]),
createNode('X', [{ target: 'B' }]),
createNode('B', [{ target: sentinelStartId }]),
createNode(sentinelStartId, [], {
isSentinel: true,
sentinelType: 'start',
subflowId: loopId,
subflowType: 'loop',
}),
])
dag.loopConfigs.set(loopId, { id: loopId, nodes: [], iterations: 3, loopType: 'for' } as any)
const result = validateRunFromBlock(loopId, dag, new Set(['A']))
expect(result.valid).toBe(false)
expect(result.error).toContain('Upstream dependency not executed: B')
})
it('allows loop container with no upstream dependencies', () => {
// Loop containers are validated via their sentinel nodes, not incoming edges on the container itself
// If the loop has no upstream dependencies, it should be valid
const loopId = 'loop-container-1'
const sentinelStartId = `loop-${loopId}-sentinel-start`
const dag = createDAG([
createNode(sentinelStartId, [], { isSentinel: true, sentinelType: 'start', loopId }),
createNode(sentinelStartId, [], {
isSentinel: true,
sentinelType: 'start',
subflowId: loopId,
subflowType: 'loop',
}),
])
dag.loopConfigs.set(loopId, { id: loopId, nodes: [], iterations: 3, loopType: 'for' } as any)
const executedBlocks = new Set<string>() // Nothing executed but loop has no deps
@@ -463,13 +575,19 @@ describe('computeDirtySet with containers', () => {
createNode(sentinelStartId, [{ target: 'B' }], {
isSentinel: true,
sentinelType: 'start',
loopId,
subflowId: loopId,
subflowType: 'loop',
}),
createNode('B', [{ target: sentinelEndId }], {
isLoopNode: true,
subflowId: loopId,
subflowType: 'loop',
}),
createNode('B', [{ target: sentinelEndId }], { isLoopNode: true, loopId }),
createNode(sentinelEndId, [{ target: 'C' }], {
isSentinel: true,
sentinelType: 'end',
loopId,
subflowId: loopId,
subflowType: 'loop',
}),
createNode('C'),
])
@@ -497,13 +615,19 @@ describe('computeDirtySet with containers', () => {
createNode(sentinelStartId, [{ target: 'B₍0₎' }], {
isSentinel: true,
sentinelType: 'start',
parallelId,
subflowId: parallelId,
subflowType: 'parallel',
}),
createNode('B₍0₎', [{ target: sentinelEndId }], {
isParallelBranch: true,
subflowId: parallelId,
subflowType: 'parallel',
}),
createNode('B₍0₎', [{ target: sentinelEndId }], { isParallelBranch: true, parallelId }),
createNode(sentinelEndId, [{ target: 'C' }], {
isSentinel: true,
sentinelType: 'end',
parallelId,
subflowId: parallelId,
subflowType: 'parallel',
}),
createNode('C'),
])
@@ -999,28 +1123,33 @@ describe('run from subflow (loop) scenarios', () => {
createNode(outerStartId, [{ target: innerStartId }], {
isSentinel: true,
sentinelType: 'start',
loopId: outerLoopId,
subflowId: outerLoopId,
subflowType: 'loop',
}),
createNode(innerStartId, [{ target: 'B' }], {
isSentinel: true,
sentinelType: 'start',
loopId: innerLoopId,
subflowId: innerLoopId,
subflowType: 'loop',
isLoopNode: true,
}),
createNode('B', [{ target: innerEndId }], {
isLoopNode: true,
loopId: innerLoopId,
subflowId: innerLoopId,
subflowType: 'loop',
}),
createNode(innerEndId, [{ target: outerEndId }], {
isSentinel: true,
sentinelType: 'end',
loopId: innerLoopId,
subflowId: innerLoopId,
subflowType: 'loop',
isLoopNode: true,
}),
createNode(outerEndId, [{ target: 'C' }], {
isSentinel: true,
sentinelType: 'end',
loopId: outerLoopId,
subflowId: outerLoopId,
subflowType: 'loop',
}),
createNode('C'),
])
+41 -3
View File
@@ -162,12 +162,29 @@ export function validateRunFromBlock(
const isLoopContainer = dag.loopConfigs.has(blockId)
const isParallelContainer = dag.parallelConfigs.has(blockId)
const isContainer = isLoopContainer || isParallelContainer
let validationNode = node
if (!node && !isContainer) {
return { valid: false, error: `Block not found in workflow: ${blockId}` }
}
if (isContainer) {
const parentLoopId = findParentLoop(blockId, dag)
if (parentLoopId) {
return {
valid: false,
error: `Cannot run from block inside loop: ${parentLoopId}`,
}
}
const parentParallelId = findParentParallel(blockId, dag)
if (parentParallelId) {
return {
valid: false,
error: `Cannot run from block inside parallel: ${parentParallelId}`,
}
}
const sentinelStartId = resolveContainerToSentinelStart(blockId, dag)
if (!sentinelStartId || !dag.nodes.has(sentinelStartId)) {
return {
@@ -175,29 +192,32 @@ export function validateRunFromBlock(
error: `Container sentinel not found for: ${blockId}`,
}
}
validationNode = dag.nodes.get(sentinelStartId)
}
if (node) {
if (node.metadata.isLoopNode) {
return {
valid: false,
error: `Cannot run from block inside loop: ${node.metadata.loopId}`,
error: `Cannot run from block inside loop: ${node.metadata.subflowId}`,
}
}
if (node.metadata.isParallelBranch) {
return {
valid: false,
error: `Cannot run from block inside parallel: ${node.metadata.parallelId}`,
error: `Cannot run from block inside parallel: ${node.metadata.subflowId}`,
}
}
if (node.metadata.isSentinel) {
return { valid: false, error: 'Cannot run from sentinel node' }
}
}
if (validationNode) {
// Check immediate upstream dependencies were executed
for (const sourceId of node.incomingEdges) {
for (const sourceId of validationNode.incomingEdges) {
const sourceNode = dag.nodes.get(sourceId)
// Skip sentinel nodes - they're internal and not in executedBlocks
if (sourceNode?.metadata.isSentinel) continue
@@ -217,3 +237,21 @@ export function validateRunFromBlock(
return { valid: true }
}
function findParentLoop(blockId: string, dag: DAG): string | undefined {
for (const [loopId, config] of dag.loopConfigs) {
if (loopId !== blockId && config.nodes?.includes(blockId)) {
return loopId
}
}
return undefined
}
function findParentParallel(blockId: string, dag: DAG): string | undefined {
for (const [parallelId, config] of dag.parallelConfigs) {
if (parallelId !== blockId && config.nodes?.includes(blockId)) {
return parallelId
}
}
return undefined
}
@@ -76,7 +76,8 @@ async function normalizeCollectionValue(ctx: ExecutionContext, value: unknown):
export async function resolveArrayInputAsync(
ctx: ExecutionContext,
items: any,
resolver: VariableResolver | null
resolver: VariableResolver | null,
currentNodeId = ''
): Promise<any[]> {
if (typeof items !== 'string') {
if (items === null) {
@@ -87,7 +88,7 @@ export async function resolveArrayInputAsync(
return []
}
try {
const resolved = (await resolver.resolveInputs(ctx, 'subflow_items', { items })).items
const resolved = (await resolver.resolveInputs(ctx, currentNodeId, { items })).items
return normalizeCollectionValue(ctx, resolved)
} catch (error) {
if (error instanceof Error && error.message.startsWith('Resolved items')) {
@@ -101,7 +102,7 @@ export async function resolveArrayInputAsync(
if (items.startsWith(REFERENCE.START) && items.endsWith(REFERENCE.END) && resolver) {
try {
const resolved = await resolver.resolveSingleReference(ctx, '', items, undefined, {
const resolved = await resolver.resolveSingleReference(ctx, currentNodeId, items, undefined, {
allowLargeValueRefs: true,
})
return normalizeCollectionValue(ctx, resolved)
+103 -71
View File
@@ -4,6 +4,7 @@ import { DEFAULTS, LOOP, PARALLEL } from '@/executor/constants'
import type { ContextExtensions } from '@/executor/execution/types'
import { type BlockLog, type ExecutionContext, getNextExecutionOrder } from '@/executor/types'
import { buildContainerIterationContext } from '@/executor/utils/iteration-context'
import type { SerializedWorkflow } from '@/serializer/types'
const logger = createLogger('SubflowUtils')
@@ -107,6 +108,12 @@ export function extractOuterBranchIndex(clonedId: string): number | undefined {
return match ? Number.parseInt(match[1], 10) : undefined
}
export function extractInnermostOuterBranchIndex(clonedId: string): number | undefined {
const matches = Array.from(clonedId.matchAll(/__obranch-(\d+)/g))
const lastMatch = matches.at(-1)
return lastMatch ? Number.parseInt(lastMatch[1], 10) : undefined
}
/**
* Strips all clone suffixes (`__obranch-N`) and branch subscripts (`₍N₎`)
* from a node ID, returning the original workflow-level block ID.
@@ -117,11 +124,18 @@ export function stripCloneSuffixes(nodeId: string): string {
)
}
/**
* Builds a stable ID for an output scoped to a global outer parallel branch.
*/
export function buildOuterBranchScopedId(originalId: string, branchIndex: number): string {
return `${originalId}__obranch-${branchIndex}`
}
/**
* Builds a cloned subflow ID from an original ID and outer branch index.
*/
export function buildClonedSubflowId(originalId: string, branchIndex: number): string {
return `${originalId}__obranch-${branchIndex}`
return buildOuterBranchScopedId(originalId, branchIndex)
}
/**
@@ -146,8 +160,23 @@ export function stripOuterBranchSuffix(id: string): string {
export function findEffectiveContainerId(
originalId: string,
currentNodeId: string,
executionMap: Map<string, unknown>
executionMap: Map<string, unknown>,
mappedBranchIndex?: number
): string {
if (mappedBranchIndex !== undefined && mappedBranchIndex > 0) {
const cloneSuffix = `__obranch-${mappedBranchIndex}`
const candidateId = buildClonedSubflowId(originalId, mappedBranchIndex)
if (executionMap.has(candidateId)) {
return candidateId
}
for (const scopeId of executionMap.keys()) {
if (scopeId.endsWith(cloneSuffix) && stripOuterBranchSuffix(scopeId) === originalId) {
return scopeId
}
}
}
// Prefer the cloned variant when currentNodeId carries an __obranch-N suffix.
// During concurrent parallel-in-loop execution both the original (branch 0)
// and cloned variants coexist in the map; the clone is the correct scope.
@@ -197,6 +226,76 @@ export function normalizeNodeId(nodeId: string): string {
return nodeId
}
type SubflowContainerType = 'loop' | 'parallel'
function getSubflowNodes(
workflow: Pick<SerializedWorkflow, 'loops' | 'parallels'>,
type: SubflowContainerType,
id: string
): string[] | undefined {
return type === 'loop' ? workflow.loops?.[id]?.nodes : workflow.parallels?.[id]?.nodes
}
export function subflowContainsBlock(
workflow: Pick<SerializedWorkflow, 'loops' | 'parallels'>,
containerType: SubflowContainerType,
containerId: string,
baseBlockId: string,
visited = new Set<string>()
): boolean {
const visitKey = `${containerType}:${containerId}`
if (visited.has(visitKey)) return false
visited.add(visitKey)
const nodes = getSubflowNodes(workflow, containerType, containerId)
if (!nodes) return false
for (const nodeId of nodes) {
if (nodeId === baseBlockId) return true
if (workflow.loops?.[nodeId]) {
if (subflowContainsBlock(workflow, 'loop', nodeId, baseBlockId, visited)) return true
} else if (workflow.parallels?.[nodeId]) {
if (subflowContainsBlock(workflow, 'parallel', nodeId, baseBlockId, visited)) return true
}
}
return false
}
export function isSubflowNestedInside(
workflow: Pick<SerializedWorkflow, 'loops' | 'parallels'>,
childType: SubflowContainerType,
childId: string,
ancestorType: SubflowContainerType,
ancestorId: string,
visited = new Set<string>()
): boolean {
const visitKey = `${ancestorType}:${ancestorId}`
if (visited.has(visitKey)) return false
visited.add(visitKey)
const nodes = getSubflowNodes(workflow, ancestorType, ancestorId)
if (!nodes) return false
for (const nodeId of nodes) {
if (
nodeId === childId &&
(childType === 'loop' ? workflow.loops?.[childId] : workflow.parallels?.[childId])
) {
return true
}
if (workflow.loops?.[nodeId]) {
if (isSubflowNestedInside(workflow, childType, childId, 'loop', nodeId, visited)) {
return true
}
} else if (workflow.parallels?.[nodeId]) {
if (isSubflowNestedInside(workflow, childType, childId, 'parallel', nodeId, visited)) {
return true
}
}
}
return false
}
/**
* Creates and logs an error for a subflow (loop or parallel).
*/
@@ -262,77 +361,10 @@ export async function addSubflowErrorLog(
}
}
/**
* Emits block log + SSE events for a loop/parallel that was skipped due to an
* empty collection or false initial condition. This ensures the container block
* appears in terminal logs, execution snapshots, and edge highlighting.
*/
export async function emitEmptySubflowEvents(
ctx: ExecutionContext,
blockId: string,
blockType: 'loop' | 'parallel',
contextExtensions: ContextExtensions | null
): Promise<void> {
const now = new Date().toISOString()
const executionOrder = getNextExecutionOrder(ctx)
const output = { results: [] }
const block = ctx.workflow?.blocks.find((b) => b.id === blockId)
const blockName = block?.metadata?.name ?? blockType
const iterationContext = buildContainerIterationContext(ctx, blockId)
ctx.blockLogs.push({
blockId,
blockName,
blockType,
startedAt: now,
endedAt: now,
durationMs: DEFAULTS.EXECUTION_TIME,
success: true,
output,
executionOrder,
})
if (contextExtensions?.onBlockStart) {
try {
await contextExtensions.onBlockStart(blockId, blockName, blockType, executionOrder)
} catch (error) {
logger.warn('Empty subflow start callback failed', {
blockId,
blockType,
error: toError(error).message,
})
}
}
if (contextExtensions?.onBlockComplete) {
try {
await contextExtensions.onBlockComplete(
blockId,
blockName,
blockType,
{
output,
executionTime: DEFAULTS.EXECUTION_TIME,
startedAt: now,
executionOrder,
endedAt: now,
},
iterationContext
)
} catch (error) {
logger.warn('Empty subflow completion callback failed', {
blockId,
blockType,
error: toError(error).message,
})
}
}
}
/**
* Emits the BlockLog + onBlockComplete callback for a loop/parallel container that
* finished successfully with at least one iteration. Without this, successful container
* runs produce no top-level BlockLog, which forces the trace-span builder to fall back
* finished successfully. Without this, successful container runs produce no top-level BlockLog,
* which forces the trace-span builder to fall back
* to generic counter-based names ("Loop 1", "Parallel 1") instead of the user-configured
* block name.
*/
@@ -141,7 +141,8 @@ function createTestWorkflow(
name?: string
type?: string
outputs?: Record<string, any>
}> = []
}> = [],
subflows: { loops?: Record<string, any>; parallels?: Record<string, any> } = {}
) {
return {
version: '1.0',
@@ -155,8 +156,8 @@ function createTestWorkflow(
enabled: true,
})),
connections: [],
loops: {},
parallels: {},
loops: subflows.loops ?? {},
parallels: subflows.parallels ?? {},
}
}
@@ -166,7 +167,8 @@ function createTestWorkflow(
function createTestContext(
currentNodeId: string,
blockOutputs: Record<string, any> = {},
contextBlockStates?: Map<string, { output: any }>
contextBlockStates?: Map<string, { output: any }>,
parallelBlockMapping?: Map<string, any>
): ResolutionContext {
const state = new ExecutionState()
for (const [blockId, output] of Object.entries(blockOutputs)) {
@@ -179,6 +181,7 @@ function createTestContext(
workflowId: 'workflow-1',
executionId: 'execution-1',
blockStates: contextBlockStates ?? new Map(),
parallelBlockMapping,
},
executionState: state,
currentNodeId,
@@ -251,6 +254,114 @@ describe('BlockResolver', () => {
expect(resolver.resolve('<source.user.profile.email>', ctx)).toBe('alice@test.com')
})
it('does not fall back to unscoped block state inside cloned subflow branches', () => {
const workflow = createTestWorkflow([{ id: 'source' }], {
parallels: { 'parallel-1': { id: 'parallel-1', nodes: ['source'] } },
})
const resolver = new BlockResolver(workflow)
const ctx = createTestContext(
'consumer__clone-inner__obranch-1₍0₎',
{},
new Map([['source', { output: { result: 'branch-0' } }]])
)
expect(resolver.resolve('<source.result>', ctx)).toBe(RESOLVED_EMPTY)
})
it('allows cloned subflows to resolve top-level upstream block state', () => {
const workflow = createTestWorkflow([{ id: 'source' }])
const resolver = new BlockResolver(workflow)
const ctx = createTestContext(
'consumer__clone-inner__obranch-1₍0₎',
{},
new Map([['source', { output: { result: 'global' } }]])
)
expect(resolver.resolve('<source.result>', ctx)).toBe('global')
})
it('uses parallel block mappings to resolve cloned subflow outputs in later batches', () => {
const workflow = createTestWorkflow(
[
{ id: 'nested-loop', name: 'Nested Loop' },
{ id: 'consumer', name: 'Consumer' },
],
{
loops: { 'nested-loop': { id: 'nested-loop', nodes: ['loop-task'] } },
parallels: { 'parallel-1': { id: 'parallel-1', nodes: ['nested-loop', 'consumer'] } },
}
)
const resolver = new BlockResolver(workflow)
const ctx = createTestContext(
'consumer₍0₎',
{
'nested-loop': { results: ['branch-0'] },
'nested-loop__obranch-2': { results: ['branch-2'] },
},
undefined,
new Map([
[
'consumer₍0₎',
{ originalBlockId: 'consumer', parallelId: 'parallel-1', iterationIndex: 2 },
],
])
)
expect(resolver.resolve('<nestedloop.results>', ctx)).toEqual(['branch-2'])
})
it('resolves regular block outputs from the same cloned branch scope', () => {
const workflow = createTestWorkflow(
[
{ id: 'source', name: 'Source' },
{ id: 'consumer', name: 'Consumer' },
],
{
parallels: { 'parallel-1': { id: 'parallel-1', nodes: ['source', 'consumer'] } },
}
)
const resolver = new BlockResolver(workflow)
const ctx = createTestContext('consumer__cloneabc__obranch-2₍0₎', {
'source__obranch-2': { result: 'wrong-container-alias' },
'source__cloneabc__obranch-2₍0₎': { result: 'same-branch' },
})
expect(resolver.resolve('<source.result>', ctx)).toBe('same-branch')
})
it('uses outer branch suffix over inner parallel mappings for cloned subflow outputs', () => {
const workflow = createTestWorkflow(
[
{ id: 'sibling-loop', name: 'Sibling Loop' },
{ id: 'inner-task', name: 'Inner Task' },
],
{
loops: { 'sibling-loop': { id: 'sibling-loop', nodes: ['loop-task'] } },
parallels: {
'outer-parallel': { id: 'outer-parallel', nodes: ['sibling-loop', 'inner-parallel'] },
'inner-parallel': { id: 'inner-parallel', nodes: ['inner-task'] },
},
}
)
const resolver = new BlockResolver(workflow)
const ctx = createTestContext(
'inner-task__clone-inner__obranch-2₍0₎',
{
'sibling-loop__obranch-1': { results: ['inner-branch-1'] },
'sibling-loop__obranch-2': { results: ['outer-branch-2'] },
},
undefined,
new Map([
[
'inner-task__clone-inner__obranch-2₍0₎',
{ originalBlockId: 'inner-task', parallelId: 'inner-parallel', iterationIndex: 1 },
],
])
)
expect(resolver.resolve('<siblingloop.results>', ctx)).toEqual(['outer-branch-2'])
})
it('should resolve nested scalar paths inside compacted block references', async () => {
const workflow = createTestWorkflow([{ id: 'source' }])
const resolver = new BlockResolver(workflow)
@@ -13,6 +13,7 @@ import {
resolveBlockReferenceAsync,
} from '@/executor/utils/block-reference'
import { formatLiteralForCode } from '@/executor/utils/code-formatting'
import { buildClonedSubflowId, extractOuterBranchIndex } from '@/executor/utils/subflow-utils'
import {
type AsyncPathNavigator,
navigatePath,
@@ -25,6 +26,8 @@ import type { SerializedBlock, SerializedWorkflow } from '@/serializer/types'
export class BlockResolver implements Resolver {
private nameToBlockId: Map<string, string>
private blockById: Map<string, SerializedBlock>
private blockIdsInSubflows: Set<string>
private subflowContainerIds: Set<string>
constructor(
private workflow: SerializedWorkflow,
@@ -32,12 +35,27 @@ export class BlockResolver implements Resolver {
) {
this.nameToBlockId = new Map()
this.blockById = new Map()
this.blockIdsInSubflows = new Set()
this.subflowContainerIds = new Set([
...Object.keys(workflow.loops ?? {}),
...Object.keys(workflow.parallels ?? {}),
])
for (const block of workflow.blocks) {
this.blockById.set(block.id, block)
if (block.metadata?.name) {
this.nameToBlockId.set(normalizeName(block.metadata.name), block.id)
}
}
for (const loop of Object.values(workflow.loops ?? {})) {
for (const blockId of loop.nodes ?? []) {
this.blockIdsInSubflows.add(blockId)
}
}
for (const parallel of Object.values(workflow.parallels ?? {})) {
for (const blockId of parallel.nodes ?? []) {
this.blockIdsInSubflows.add(blockId)
}
}
}
canResolve(reference: string): boolean {
@@ -282,10 +300,36 @@ export class BlockResolver implements Resolver {
}
private getBlockOutput(blockId: string, context: ResolutionContext): any {
const outerBranchIndex = extractOuterBranchIndex(context.currentNodeId)
const mappedBranchIndex =
outerBranchIndex ??
context.executionContext.parallelBlockMapping?.get(context.currentNodeId)?.iterationIndex
const shouldResolveClonedSubflowOutput =
mappedBranchIndex !== undefined &&
mappedBranchIndex > 0 &&
this.subflowContainerIds.has(blockId)
if (shouldResolveClonedSubflowOutput) {
const clonedStateOutput = context.executionState.getBlockOutput(
buildClonedSubflowId(blockId, mappedBranchIndex)
)
if (clonedStateOutput !== undefined) {
return clonedStateOutput
}
}
const stateOutput = context.executionState.getBlockOutput(blockId, context.currentNodeId)
if (stateOutput !== undefined) {
return stateOutput
}
if (
shouldResolveClonedSubflowOutput ||
(outerBranchIndex !== undefined && this.blockIdsInSubflows.has(blockId))
) {
return undefined
}
const contextState = context.executionContext.blockStates?.get(blockId)
if (contextState?.output) {
return contextState.output
@@ -17,7 +17,11 @@ interface BlockDef {
name: string
}
function createTestWorkflow(loops: Record<string, LoopDef> = {}, blockDefs: BlockDef[] = []) {
function createTestWorkflow(
loops: Record<string, LoopDef> = {},
blockDefs: BlockDef[] = [],
parallels: Record<string, { id?: string; nodes: string[] }> = {}
) {
const normalizedLoops: Record<string, { id: string; nodes: string[]; iterations: number }> = {}
for (const [key, loop] of Object.entries(loops)) {
normalizedLoops[key] = {
@@ -41,7 +45,7 @@ function createTestWorkflow(loops: Record<string, LoopDef> = {}, blockDefs: Bloc
blocks,
connections: [],
loops: normalizedLoops,
parallels: {},
parallels,
}
}
@@ -58,7 +62,8 @@ function createTestContext(
currentNodeId: string,
loopScope?: LoopScope,
loopExecutions?: Map<string, LoopScope>,
blockOutputs?: Record<string, any>
blockOutputs?: Record<string, any>,
parallelBlockMapping?: Map<string, any>
): ResolutionContext {
return {
executionContext: {
@@ -66,6 +71,7 @@ function createTestContext(
workflowId: 'workflow-1',
executionId: 'execution-1',
loopExecutions: loopExecutions ?? new Map(),
parallelBlockMapping,
},
executionState: {
getBlockOutput: (id: string) => blockOutputs?.[id],
@@ -249,6 +255,34 @@ describe('LoopResolver', () => {
expect(resolver.resolve('<loop.index>', ctx)).toBe(0)
})
it('resolves generic loop context from inside a parallel nested in a loop', () => {
const workflow = createTestWorkflow({ 'loop-1': { nodes: ['parallel-1'] } }, [], {
'parallel-1': { id: 'parallel-1', nodes: ['block-1'] },
})
const resolver = new LoopResolver(workflow)
const loopScope = createLoopScope({ iteration: 3 })
const ctx = createTestContext('block-1₍0₎', undefined, new Map([['loop-1', loopScope]]))
expect(resolver.resolve('<loop.index>', ctx)).toBe(3)
})
it('resolves inner cloned loop context independently from outer clone indexes', () => {
const workflow = createTestWorkflow({ 'loop-1': { nodes: ['task'] } })
const resolver = new LoopResolver(workflow)
const loopExecutions = new Map<string, LoopScope>([
['loop-1__obranch-1', createLoopScope({ iteration: 4, item: 'inner-branch-1' })],
['loop-1__obranch-2', createLoopScope({ iteration: 9, item: 'outer-branch-2' })],
])
const ctx = createTestContext(
'task__cloneabc__obranch-2__clonedef__obranch-1',
undefined,
loopExecutions
)
expect(resolver.resolve('<loop.index>', ctx)).toBe(4)
expect(resolver.resolve('<loop.currentItem>', ctx)).toBe('inner-branch-1')
})
it.concurrent('should handle null item value', () => {
const resolver = new LoopResolver(createTestWorkflow())
const loopScope = createLoopScope({ item: null })
@@ -395,6 +429,62 @@ describe('LoopResolver', () => {
expect(resolver.resolve('<loop1.results>', ctx)).toEqual(results)
})
it('uses parallel block mappings to resolve cloned loop outputs in later batches', () => {
const workflow = createTestWorkflow({ 'loop-1': { nodes: ['block-1'] } }, [
{ id: 'loop-1', name: 'Loop 1' },
])
const resolver = new LoopResolver(workflow)
const loopExecutions = new Map<string, LoopScope>([
['loop-1', createLoopScope()],
['loop-1__obranch-2', createLoopScope()],
])
const ctx = createTestContext(
'consumer₍0₎',
undefined,
loopExecutions,
{
'loop-1': { results: ['branch-0'] },
'loop-1__obranch-2': { results: ['branch-2'] },
},
new Map([
[
'consumer₍0₎',
{ originalBlockId: 'consumer', parallelId: 'parallel-1', iterationIndex: 2 },
],
])
)
expect(resolver.resolve('<loop1.results>', ctx)).toEqual(['branch-2'])
})
it('uses outer branch suffix over inner parallel mappings for cloned loop outputs', () => {
const workflow = createTestWorkflow({ 'loop-1': { nodes: ['block-1'] } }, [
{ id: 'loop-1', name: 'Loop 1' },
])
const resolver = new LoopResolver(workflow)
const loopExecutions = new Map<string, LoopScope>([
['loop-1__obranch-1', createLoopScope()],
['loop-1__obranch-2', createLoopScope()],
])
const ctx = createTestContext(
'consumer__cloneabc__obranch-2₍0₎',
undefined,
loopExecutions,
{
'loop-1__obranch-1': { results: ['outer-branch-1'] },
'loop-1__obranch-2': { results: ['outer-branch-2'] },
},
new Map([
[
'consumer__cloneabc__obranch-2₍0₎',
{ originalBlockId: 'consumer', parallelId: 'inner-parallel', iterationIndex: 1 },
],
])
)
expect(resolver.resolve('<loop1.results>', ctx)).toEqual(['outer-branch-2'])
})
it.concurrent('should resolve result with nested path', () => {
const workflow = createTestWorkflow({ 'loop-1': { nodes: ['block-1'] } }, [
{ id: 'loop-1', name: 'Loop 1' },
+16 -43
View File
@@ -3,9 +3,13 @@ import { assertNoLargeValueRefs } from '@/lib/execution/payloads/large-value-ref
import { isReference, normalizeName, parseReferencePath, REFERENCE } from '@/executor/constants'
import { InvalidFieldError } from '@/executor/utils/block-reference'
import {
extractInnermostOuterBranchIndex,
extractOuterBranchIndex,
findEffectiveContainerId,
isSubflowNestedInside,
stripCloneSuffixes,
stripOuterBranchSuffix,
subflowContainsBlock,
} from '@/executor/utils/subflow-utils'
import {
type AsyncPathNavigator,
@@ -98,10 +102,16 @@ export class LoopResolver implements Resolver {
// Resolve the effective (possibly cloned) loop ID for scope/output lookups
if (targetLoopId && context.executionContext.loopExecutions) {
const mappedBranchIndex =
(isGenericRef
? extractInnermostOuterBranchIndex(context.currentNodeId)
: extractOuterBranchIndex(context.currentNodeId)) ??
context.executionContext.parallelBlockMapping?.get(context.currentNodeId)?.iterationIndex
targetLoopId = findEffectiveContainerId(
targetLoopId,
context.currentNodeId,
context.executionContext.loopExecutions
context.executionContext.loopExecutions,
mappedBranchIndex
)
}
@@ -243,7 +253,7 @@ export class LoopResolver implements Resolver {
const baseId = stripCloneSuffixes(blockId)
const loops = this.workflow.loops || {}
const candidateLoopIds = Object.keys(loops).filter((loopId) =>
loops[loopId].nodes.includes(baseId)
subflowContainsBlock(this.workflow, 'loop', loopId, baseId)
)
if (candidateLoopIds.length === 0) return undefined
if (candidateLoopIds.length === 1) return candidateLoopIds[0]
@@ -252,7 +262,9 @@ export class LoopResolver implements Resolver {
// In a valid DAG, exactly one candidate will satisfy this (circular containment is impossible).
return candidateLoopIds.find((candidateId) =>
candidateLoopIds.every(
(otherId) => otherId === candidateId || !loops[candidateId].nodes.includes(otherId)
(otherId) =>
otherId === candidateId ||
!isSubflowNestedInside(this.workflow, 'loop', otherId, 'loop', candidateId)
)
)
}
@@ -260,46 +272,7 @@ export class LoopResolver implements Resolver {
private isBlockInLoopOrDescendant(blockId: string, targetLoopId: string): boolean {
const baseId = stripCloneSuffixes(blockId)
const originalLoopId = stripOuterBranchSuffix(targetLoopId)
const targetLoop = this.workflow.loops?.[originalLoopId]
if (!targetLoop) {
return false
}
if (targetLoop.nodes.includes(baseId)) {
return true
}
const directLoopId = this.findInnermostLoopForBlock(blockId)
if (!directLoopId) {
return false
}
if (directLoopId === originalLoopId) {
return true
}
return this.isLoopNestedInside(directLoopId, originalLoopId)
}
private isLoopNestedInside(
childLoopId: string,
ancestorLoopId: string,
visited = new Set<string>()
): boolean {
if (visited.has(ancestorLoopId)) return false
visited.add(ancestorLoopId)
const ancestorLoop = this.workflow.loops?.[ancestorLoopId]
if (!ancestorLoop) {
return false
}
if (ancestorLoop.nodes.includes(childLoopId)) {
return true
}
for (const nodeId of ancestorLoop.nodes) {
if (this.workflow.loops[nodeId]) {
if (this.isLoopNestedInside(childLoopId, nodeId, visited)) {
return true
}
}
}
return false
return subflowContainsBlock(this.workflow, 'loop', originalLoopId, baseId)
}
private isForEachLoop(loopId: string): boolean {
@@ -22,7 +22,8 @@ function createTestWorkflow(
parallelType?: 'count' | 'collection'
}
> = {},
blockDefs: BlockDef[] = []
blockDefs: BlockDef[] = [],
loops: Record<string, { id?: string; nodes: string[] }> = {}
) {
const normalizedParallels: Record<
string,
@@ -54,7 +55,7 @@ function createTestWorkflow(
version: '1.0',
blocks,
connections: [],
loops: {},
loops,
parallels: normalizedParallels,
}
}
@@ -78,7 +79,8 @@ function createTestContext(
currentNodeId: string,
parallelExecutions?: Map<string, any>,
blockOutputs?: Record<string, any>,
parallelBlockMapping?: Map<string, any>
parallelBlockMapping?: Map<string, any>,
subflowParentMap?: Map<string, any>
): ResolutionContext {
return {
executionContext: {
@@ -87,6 +89,7 @@ function createTestContext(
executionId: 'execution-1',
parallelExecutions: parallelExecutions ?? new Map(),
parallelBlockMapping,
subflowParentMap,
},
executionState: {
getBlockOutput: (id: string) => blockOutputs?.[id],
@@ -452,6 +455,71 @@ describe('ParallelResolver', () => {
expect(resolver.resolve('<parallel1.results>', ctx)).toEqual(results)
})
it('uses parallel block mappings to resolve cloned parallel outputs in later batches', () => {
const workflow = createTestWorkflow(
{ 'nested-parallel': { nodes: ['block-1'], distribution: ['a', 'b'] } },
[{ id: 'nested-parallel', name: 'Nested Parallel' }]
)
const resolver = new ParallelResolver(workflow)
const parallelExecutions = new Map<string, any>([
['nested-parallel', { parallelId: 'nested-parallel', branchOutputs: new Map() }],
[
'nested-parallel__obranch-2',
{ parallelId: 'nested-parallel__obranch-2', branchOutputs: new Map() },
],
])
const ctx = createTestContext(
'consumer₍0₎',
parallelExecutions,
{
'nested-parallel': { results: ['branch-0'] },
'nested-parallel__obranch-2': { results: ['branch-2'] },
},
new Map([
[
'consumer₍0₎',
{ originalBlockId: 'consumer', parallelId: 'parallel-1', iterationIndex: 2 },
],
])
)
expect(resolver.resolve('<nestedparallel.results>', ctx)).toEqual(['branch-2'])
})
it('uses outer branch suffix over inner parallel mappings for cloned parallel outputs', () => {
const workflow = createTestWorkflow(
{ 'nested-parallel': { nodes: ['block-1'], distribution: ['a', 'b'] } },
[{ id: 'nested-parallel', name: 'Nested Parallel' }]
)
const resolver = new ParallelResolver(workflow)
const parallelExecutions = new Map<string, any>([
[
'nested-parallel__obranch-1',
{ parallelId: 'nested-parallel__obranch-1', branchOutputs: new Map() },
],
[
'nested-parallel__obranch-2',
{ parallelId: 'nested-parallel__obranch-2', branchOutputs: new Map() },
],
])
const ctx = createTestContext(
'consumer__cloneabc__obranch-2₍0₎',
parallelExecutions,
{
'nested-parallel__obranch-1': { results: ['outer-branch-1'] },
'nested-parallel__obranch-2': { results: ['outer-branch-2'] },
},
new Map([
[
'consumer__cloneabc__obranch-2₍0₎',
{ originalBlockId: 'consumer', parallelId: 'inner-parallel', iterationIndex: 1 },
],
])
)
expect(resolver.resolve('<nestedparallel.results>', ctx)).toEqual(['outer-branch-2'])
})
it.concurrent('should resolve result with nested path', () => {
const workflow = createTestWorkflow(
{ 'parallel-1': { nodes: ['block-1'], distribution: ['a', 'b'] } },
@@ -597,5 +665,77 @@ describe('ParallelResolver', () => {
expect(resolver.resolve('<parallel.results>', ctx)).toEqual(results)
expect(resolver.resolve('<parallel.result>', ctx)).toEqual(results)
})
it('resolves generic parallel context from inside a loop nested in a parallel', () => {
const workflow = createTestWorkflow(
{
'parallel-1': {
nodes: ['loop-1'],
distribution: ['a', 'b', 'c'],
parallelType: 'collection',
},
},
[],
{ 'loop-1': { id: 'loop-1', nodes: ['block-1'] } }
)
const resolver = new ParallelResolver(workflow)
const ctx = createTestContext(
'block-1__cloneaaa__obranch-2',
new Map([['parallel-1', createParallelScope(['a', 'b', 'c'])]])
)
expect(resolver.resolve('<parallel.index>', ctx)).toBe(2)
expect(resolver.resolve('<parallel.currentItem>', ctx)).toBe('c')
})
it('resolves inner parallel branch context independently from the outer clone index', () => {
const workflow = createTestWorkflow({
'outer-parallel': {
nodes: ['inner-parallel'],
distribution: ['outer0', 'outer1', 'outer2'],
},
'inner-parallel': {
nodes: ['block-1'],
distribution: ['inner0', 'inner1'],
},
})
const resolver = new ParallelResolver(workflow)
const parallelExecutions = new Map([
['inner-parallel__obranch-2', createParallelScope(['inner0', 'inner1'])],
])
const ctx = createTestContext('block-1__cloneabc__obranch-2₍1₎', parallelExecutions)
expect(resolver.resolve('<parallel.index>', ctx)).toBe(1)
expect(resolver.resolve('<parallel.currentItem>', ctx)).toBe('inner1')
})
it('resolves parent parallel context for branch-zero nested subflow descendants', () => {
const workflow = createTestWorkflow(
{
'outer-parallel': {
nodes: ['inner-loop'],
distribution: ['outer0', 'outer1'],
},
},
[],
{ 'inner-loop': { id: 'inner-loop', nodes: ['loop-task'] } }
)
const resolver = new ParallelResolver(workflow)
const parallelExecutions = new Map([
['outer-parallel', createParallelScope(['outer0', 'outer1'])],
])
const ctx = createTestContext(
'loop-task',
parallelExecutions,
undefined,
undefined,
new Map([
['inner-loop', { parentId: 'outer-parallel', parentType: 'parallel', branchIndex: 0 }],
])
)
expect(resolver.resolve('<parallel.index>', ctx)).toBe(0)
expect(resolver.resolve('<parallel.currentItem>', ctx)).toBe('outer0')
})
})
})
@@ -4,10 +4,13 @@ import { isReference, normalizeName, parseReferencePath, REFERENCE } from '@/exe
import { InvalidFieldError } from '@/executor/utils/block-reference'
import {
extractBranchIndex,
extractInnermostOuterBranchIndex,
extractOuterBranchIndex,
findEffectiveContainerId,
isSubflowNestedInside,
stripCloneSuffixes,
stripOuterBranchSuffix,
subflowContainsBlock,
} from '@/executor/utils/subflow-utils'
import {
type AsyncPathNavigator,
@@ -98,10 +101,16 @@ export class ParallelResolver implements Resolver {
// Resolve the effective (possibly cloned) parallel ID for scope lookups
if (context.executionContext.parallelExecutions) {
const mappedBranchIndex =
(isGenericRef
? extractInnermostOuterBranchIndex(context.currentNodeId)
: extractOuterBranchIndex(context.currentNodeId)) ??
context.executionContext.parallelBlockMapping?.get(context.currentNodeId)?.iterationIndex
targetParallelId = findEffectiveContainerId(
targetParallelId,
context.currentNodeId,
context.executionContext.parallelExecutions
context.executionContext.parallelExecutions,
mappedBranchIndex
)
}
@@ -188,16 +197,59 @@ export class ParallelResolver implements Resolver {
private resolveBranchIndex(targetParallelId: string, context: ResolutionContext): number | null {
const mapping = context.executionContext.parallelBlockMapping?.get(context.currentNodeId)
if (mapping?.parallelId === targetParallelId) {
const originalTargetParallelId = stripOuterBranchSuffix(targetParallelId)
if (
mapping?.parallelId === targetParallelId ||
mapping?.parallelId === originalTargetParallelId
) {
return mapping.iterationIndex
}
const branchIndex = extractBranchIndex(context.currentNodeId)
if (targetParallelId !== originalTargetParallelId && branchIndex !== null) {
return branchIndex
}
const outerBranchIndex = extractOuterBranchIndex(context.currentNodeId)
if (outerBranchIndex !== undefined) {
return outerBranchIndex
}
return extractBranchIndex(context.currentNodeId)
const parentBranchIndex = this.resolveParentParallelBranchIndex(
originalTargetParallelId,
context
)
if (parentBranchIndex !== undefined) {
return parentBranchIndex
}
return branchIndex
}
private resolveParentParallelBranchIndex(
targetParallelId: string,
context: ResolutionContext
): number | undefined {
const parentMap = context.executionContext.subflowParentMap
if (!parentMap) return undefined
const baseId = stripCloneSuffixes(context.currentNodeId)
for (const [subflowId, entry] of parentMap) {
if (entry.parentType !== 'parallel' || entry.parentId !== targetParallelId) continue
if (entry.branchIndex === undefined) continue
const originalSubflowId = stripOuterBranchSuffix(subflowId)
if (this.workflow.loops?.[originalSubflowId]) {
if (subflowContainsBlock(this.workflow, 'loop', originalSubflowId, baseId)) {
return entry.branchIndex
}
} else if (this.workflow.parallels?.[originalSubflowId]) {
if (subflowContainsBlock(this.workflow, 'parallel', originalSubflowId, baseId)) {
return entry.branchIndex
}
}
}
return undefined
}
private findInnermostParallelForBlock(blockId: string): string | undefined {
@@ -206,7 +258,7 @@ export class ParallelResolver implements Resolver {
if (!parallels) return undefined
const candidateIds = Object.keys(parallels).filter((parallelId) =>
parallels[parallelId]?.nodes.includes(baseId)
subflowContainsBlock(this.workflow, 'parallel', parallelId, baseId)
)
if (candidateIds.length === 0) return undefined
if (candidateIds.length === 1) return candidateIds[0]
@@ -215,49 +267,16 @@ export class ParallelResolver implements Resolver {
// In a valid DAG, exactly one candidate will satisfy this (circular containment is impossible).
return candidateIds.find((candidateId) =>
candidateIds.every(
(otherId) => otherId === candidateId || !parallels[candidateId]?.nodes.includes(otherId)
(otherId) =>
otherId === candidateId ||
!isSubflowNestedInside(this.workflow, 'parallel', otherId, 'parallel', candidateId)
)
)
}
private isBlockInParallelOrDescendant(blockId: string, targetParallelId: string): boolean {
const baseId = stripCloneSuffixes(blockId)
const parallels = this.workflow.parallels
if (!parallels) return false
const targetConfig = parallels[targetParallelId]
if (!targetConfig) return false
if (targetConfig.nodes.includes(baseId)) return true
const directParallelId = this.findInnermostParallelForBlock(blockId)
if (!directParallelId) return false
if (directParallelId === targetParallelId) return true
return this.isParallelNestedInside(directParallelId, targetParallelId)
}
private isParallelNestedInside(
childParallelId: string,
ancestorParallelId: string,
visited = new Set<string>()
): boolean {
if (visited.has(ancestorParallelId)) return false
visited.add(ancestorParallelId)
const ancestorConfig = this.workflow.parallels?.[ancestorParallelId]
if (!ancestorConfig) return false
if (ancestorConfig.nodes.includes(childParallelId)) return true
for (const nodeId of ancestorConfig.nodes) {
if (this.workflow.parallels?.[nodeId]) {
if (this.isParallelNestedInside(childParallelId, nodeId, visited)) {
return true
}
}
}
return false
return subflowContainsBlock(this.workflow, 'parallel', targetParallelId, baseId)
}
private resolveCurrentItem(
@@ -0,0 +1,46 @@
import { describe, expect, it } from 'vitest'
import { executeWorkflowBodySchema } from '@/lib/api/contracts/workflows'
describe('workflow contracts', () => {
it('normalizes null React Flow edge handles in execution overrides', () => {
const parsed = executeWorkflowBodySchema.parse({
workflowStateOverride: {
blocks: {
source: {
id: 'source',
type: 'start_trigger',
name: 'Start',
position: { x: 0, y: 0 },
subBlocks: {},
outputs: {},
enabled: true,
},
target: {
id: 'target',
type: 'function',
name: 'Function',
position: { x: 100, y: 0 },
subBlocks: {},
outputs: {},
enabled: true,
},
},
edges: [
{
id: 'edge-1',
source: 'source',
target: 'target',
sourceHandle: null,
targetHandle: null,
type: 'workflowEdge',
},
],
loops: {},
parallels: {},
},
})
expect(parsed.workflowStateOverride?.edges[0].sourceHandle).toBeUndefined()
expect(parsed.workflowStateOverride?.edges[0].targetHandle).toBeUndefined()
})
})
+6 -2
View File
@@ -40,6 +40,10 @@ const workflowSubBlockStateSchema = z.object({
})
const workflowBlockOutputSchema = z.unknown()
const workflowEdgeHandleSchema = z
.string()
.nullish()
.transform((value) => value ?? undefined)
const workflowBlockStateSchema = z.object({
id: z.string(),
@@ -61,8 +65,8 @@ const workflowEdgeSchema = z.object({
id: z.string(),
source: z.string(),
target: z.string(),
sourceHandle: z.string().optional(),
targetHandle: z.string().optional(),
sourceHandle: workflowEdgeHandleSchema,
targetHandle: workflowEdgeHandleSchema,
type: z.string().optional(),
animated: z.boolean().optional(),
style: z.record(z.string(), z.unknown()).optional(),
@@ -35,8 +35,14 @@ const dbMocks = vi.hoisted(() => {
}
})
const { completeWorkflowExecutionMock } = vi.hoisted(() => ({
const {
completeWorkflowExecutionMock,
startWorkflowExecutionMock,
loadWorkflowStateForExecutionMock,
} = vi.hoisted(() => ({
completeWorkflowExecutionMock: vi.fn(),
startWorkflowExecutionMock: vi.fn(),
loadWorkflowStateForExecutionMock: vi.fn(),
}))
vi.mock('@sim/db', () => ({
@@ -55,7 +61,7 @@ vi.mock('drizzle-orm', () => ({
vi.mock('@/lib/logs/execution/logger', () => ({
executionLogger: {
startWorkflowExecution: vi.fn(),
startWorkflowExecution: startWorkflowExecutionMock,
completeWorkflowExecution: completeWorkflowExecutionMock,
},
}))
@@ -75,11 +81,76 @@ vi.mock('@/lib/logs/execution/logging-factory', () => ({
createEnvironmentObject: vi.fn(),
createTriggerObject: vi.fn(),
loadDeployedWorkflowStateForLogging: vi.fn(),
loadWorkflowStateForExecution: vi.fn(),
loadWorkflowStateForExecution: loadWorkflowStateForExecutionMock,
}))
import { LoggingSession } from './logging-session'
describe('LoggingSession start snapshots', () => {
beforeEach(() => {
vi.clearAllMocks()
startWorkflowExecutionMock.mockResolvedValue({})
loadWorkflowStateForExecutionMock.mockResolvedValue({
blocks: {
stale: {
id: 'stale',
type: 'function',
name: 'Stale',
position: { x: 0, y: 0 },
subBlocks: {},
outputs: {},
enabled: true,
},
},
edges: [],
loops: {},
parallels: {},
})
})
it('uses the executed workflow state override for execution snapshots', async () => {
const session = new LoggingSession('workflow-1', 'execution-1', 'manual', 'req-1')
const executedWorkflowState = {
blocks: {
loop: {
id: 'loop',
type: 'loop',
name: 'Loop',
position: { x: 0, y: 0 },
subBlocks: {},
outputs: {},
enabled: true,
},
parallel: {
id: 'parallel',
type: 'parallel',
name: 'Parallel',
position: { x: 100, y: 80 },
subBlocks: {},
outputs: {},
enabled: true,
data: { parentId: 'loop', extent: 'parent' as const },
},
},
edges: [],
loops: { loop: { id: 'loop', nodes: ['parallel'], iterations: 1, loopType: 'for' as const } },
parallels: { parallel: { id: 'parallel', nodes: [], count: 1 } },
}
await session.start({
workspaceId: 'workspace-1',
workflowState: executedWorkflowState,
})
expect(loadWorkflowStateForExecutionMock).not.toHaveBeenCalled()
expect(startWorkflowExecutionMock).toHaveBeenCalledWith(
expect.objectContaining({
workflowState: executedWorkflowState,
})
)
})
})
describe('LoggingSession completion retries', () => {
beforeEach(() => {
vi.clearAllMocks()
+19 -11
View File
@@ -82,6 +82,7 @@ export interface SessionStartParams {
triggerData?: TriggerData
skipLogCreation?: boolean // For resume executions - reuse existing log entry
deploymentVersionId?: string // ID of the deployment version used (null for manual/editor executions)
workflowState?: WorkflowState
}
export interface SessionCompleteParams {
@@ -413,8 +414,15 @@ export class LoggingSession {
}
async start(params: SessionStartParams): Promise<void> {
const { userId, workspaceId, variables, triggerData, skipLogCreation, deploymentVersionId } =
params
const {
userId,
workspaceId,
variables,
triggerData,
skipLogCreation,
deploymentVersionId,
workflowState,
} = params
try {
this.trigger = createTriggerObject(this.triggerType, triggerData)
@@ -426,11 +434,11 @@ export class LoggingSession {
workspaceId,
variables
)
// Use deployed state if deploymentVersionId is provided (non-manual execution)
// Otherwise fall back to loading from normalized tables (manual/draft execution)
this.workflowState = deploymentVersionId
? await loadDeployedWorkflowStateForLogging(this.workflowId)
: await loadWorkflowStateForExecution(this.workflowId)
this.workflowState =
workflowState ??
(deploymentVersionId
? await loadDeployedWorkflowStateForLogging(this.workflowId)
: await loadWorkflowStateForExecution(this.workflowId))
if (!skipLogCreation) {
await executionLogger.startWorkflowExecution({
@@ -895,7 +903,8 @@ export class LoggingSession {
// Fallback: create a minimal logging session without full workflow state
try {
const { userId, workspaceId, variables, triggerData, deploymentVersionId } = params
const { userId, workspaceId, variables, triggerData, deploymentVersionId, workflowState } =
params
this.trigger = createTriggerObject(this.triggerType, triggerData)
this.correlation = triggerData?.correlation
this.environment = createEnvironmentObject(
@@ -905,14 +914,13 @@ export class LoggingSession {
workspaceId,
variables
)
// Minimal workflow state when normalized/deployed data is unavailable
const minimalWorkflowState: WorkflowState = {
const fallbackWorkflowState: WorkflowState = workflowState ?? {
blocks: {},
edges: [],
loops: {},
parallels: {},
}
this.workflowState = minimalWorkflowState
this.workflowState = fallbackWorkflowState
await executionLogger.startWorkflowExecution({
workflowId: this.workflowId,
@@ -215,6 +215,49 @@ describe('executeWorkflowCore terminal finalization sequencing', () => {
)
})
it('starts logging with the workflow state that will be executed', async () => {
const executedWorkflowState = {
blocks: {
loop: { id: 'loop', type: 'loop', name: 'Loop', subBlocks: {} },
parallel: {
id: 'parallel',
type: 'parallel',
name: 'Parallel',
subBlocks: {},
data: { parentId: 'loop', extent: 'parent' },
},
},
edges: [],
loops: { loop: { id: 'loop', nodes: ['parallel'], iterations: 1, loopType: 'for' } },
parallels: { parallel: { id: 'parallel', nodes: [], count: 1 } },
}
executorExecuteMock.mockResolvedValue({
success: true,
status: 'completed',
output: { done: true },
logs: [],
metadata: { duration: 123, startTime: 'start', endTime: 'end' },
})
await executeWorkflowCore({
snapshot: {
...createSnapshot(),
metadata: {
...createSnapshot().metadata,
workflowStateOverride: executedWorkflowState,
},
} as any,
callbacks: {},
loggingSession: loggingSession as any,
})
expect(safeStartMock).toHaveBeenCalledWith(
expect.objectContaining({
workflowState: executedWorkflowState,
})
)
})
it('uses external trigger selection for webhook executions without an explicit triggerBlockId', async () => {
executorExecuteMock.mockResolvedValue({
success: true,
@@ -376,6 +376,7 @@ export async function executeWorkflowCore(
triggerData: metadata.correlation ? { correlation: metadata.correlation } : undefined,
skipLogCreation,
deploymentVersionId,
workflowState: { blocks, edges, loops, parallels },
})
// Use edges directly - trigger-to-trigger edges are prevented at creation time
@@ -0,0 +1,145 @@
/**
* @vitest-environment node
*/
import { describe, expect, it } from 'vitest'
import { updateResumeOutputInAggregationBuffers } from '@/lib/workflows/executor/human-in-the-loop-manager'
import type { SerializableExecutionState } from '@/executor/execution/types'
function createExecutionState(): SerializableExecutionState {
return {
blockStates: {},
executedBlocks: [],
blockLogs: [],
decisions: { router: {}, condition: {} },
completedLoops: [],
activeExecutionPath: [],
}
}
describe('updateResumeOutputInAggregationBuffers', () => {
it('replaces a paused parallel branch placeholder with the resumed HITL output', () => {
const pausedOutput = {
response: { status: 'paused' },
_pauseMetadata: {
contextId: 'pause-context-1',
blockId: 'hitl₍1₎',
},
}
const siblingOutput = { value: 'already-complete' }
const mergedOutput = {
response: { data: { submission: { approved: true } } },
submission: { approved: true },
_resumed: true,
}
const state = createExecutionState()
state.parallelExecutions = {
'parallel-1': {
branchOutputs: {
0: [siblingOutput],
1: [pausedOutput],
},
},
}
updateResumeOutputInAggregationBuffers(
state,
'hitl₍1₎',
'hitl',
'pause-context-1',
mergedOutput
)
expect(state.parallelExecutions['parallel-1'].branchOutputs).toEqual({
0: [siblingOutput],
1: [mergedOutput],
})
})
it('does not replace unrelated paused parallel branch outputs', () => {
const unrelatedPausedOutput = {
response: { status: 'paused' },
_pauseMetadata: {
contextId: 'different-context',
blockId: 'hitl₍1₎',
},
}
const mergedOutput = {
response: { data: { submission: { approved: true } } },
submission: { approved: true },
_resumed: true,
}
const state = createExecutionState()
state.parallelExecutions = {
'parallel-1': {
branchOutputs: {
1: [unrelatedPausedOutput],
},
},
}
updateResumeOutputInAggregationBuffers(
state,
'hitl₍1₎',
'hitl',
'pause-context-1',
mergedOutput
)
expect(state.parallelExecutions['parallel-1'].branchOutputs).toEqual({
1: [unrelatedPausedOutput],
})
})
it('replaces paused loop iteration outputs using the resumed state block key', () => {
const pausedOutput = {
response: { status: 'paused' },
_pauseMetadata: {
contextId: 'pause-context-1',
blockId: 'hitl',
},
}
const unrelatedPausedOutput = {
response: { status: 'paused' },
_pauseMetadata: {
contextId: 'different-context',
blockId: 'hitl',
},
}
const siblingOutput = { value: 'already-complete' }
const mergedOutput = {
response: { data: { submission: { approved: true } } },
submission: { approved: true },
_resumed: true,
}
const state = createExecutionState()
state.loopExecutions = {
'loop-1': {
currentIterationOutputs: {
hitl: pausedOutput,
sibling: siblingOutput,
},
},
'loop-2': {
currentIterationOutputs: {
hitl: unrelatedPausedOutput,
},
},
}
updateResumeOutputInAggregationBuffers(
state,
'hitl₍1₎',
'hitl',
'pause-context-1',
mergedOutput
)
expect(state.loopExecutions['loop-1'].currentIterationOutputs).toEqual({
'hitl₍1₎': mergedOutput,
sibling: siblingOutput,
})
expect(state.loopExecutions['loop-2'].currentIterationOutputs).toEqual({
hitl: unrelatedPausedOutput,
})
})
})
@@ -52,6 +52,58 @@ function isRecord(value: unknown): value is Record<string, unknown> {
return value !== null && typeof value === 'object' && !Array.isArray(value)
}
function isPausedOutputForContext(output: unknown, contextId: string): boolean {
if (!isRecord(output)) return false
const metadata = output._pauseMetadata
return isRecord(metadata) && metadata.contextId === contextId
}
export function updateResumeOutputInAggregationBuffers(
state: SerializableExecutionState,
stateBlockKey: string,
pauseBlockId: string,
contextId: string,
mergedOutput: Record<string, unknown>
): void {
for (const scope of Object.values(state.loopExecutions ?? {})) {
if (!isRecord(scope) || !isRecord(scope.currentIterationOutputs)) continue
const outputs = scope.currentIterationOutputs
const pausedEntry =
outputs[stateBlockKey] !== undefined
? stateBlockKey
: outputs[pauseBlockId] !== undefined
? pauseBlockId
: undefined
if (pausedEntry !== undefined && isPausedOutputForContext(outputs[pausedEntry], contextId)) {
if (pausedEntry !== stateBlockKey) {
delete outputs[pausedEntry]
}
outputs[stateBlockKey] = mergedOutput
}
}
for (const scope of Object.values(state.parallelExecutions ?? {})) {
if (!isRecord(scope) || !isRecord(scope.branchOutputs)) continue
for (const [branchIndex, branchOutputs] of Object.entries(scope.branchOutputs)) {
if (!Array.isArray(branchOutputs)) continue
const outputIndex = branchOutputs.findIndex((output) =>
isPausedOutputForContext(output, contextId)
)
if (outputIndex !== -1) {
scope.branchOutputs[branchIndex] = [
...branchOutputs.slice(0, outputIndex),
mergedOutput,
...branchOutputs.slice(outputIndex + 1),
]
}
}
}
}
function parseSnapshotForReferenceTracking(snapshotSeed: SerializedSnapshot): unknown {
try {
return { ...snapshotSeed, snapshot: JSON.parse(snapshotSeed.snapshot) }
@@ -782,7 +834,7 @@ export class PauseResumeManager {
resume: existingResponse.resume ?? existingOutput.resume,
}
const mergedOutput: Record<string, any> = {
const mergedOutput: Record<string, unknown> = {
...existingOutput,
response: mergedResponse,
submission: submissionPayload,
@@ -827,6 +879,13 @@ export class PauseResumeManager {
}
stateCopy.blockStates[stateBlockKey] = pauseBlockState
updateResumeOutputInAggregationBuffers(
stateCopy,
stateBlockKey,
pauseBlockId,
contextId,
mergedOutput
)
// Update the block log entry with the merged output so logs show the submission data
if (Array.isArray(stateCopy.blockLogs)) {