From 5100e2e2fc7fb4be072591e70d8c2e119a3ab43a Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 11 Feb 2025 13:47:22 -0800 Subject: [PATCH] Modified evaluator, runs similar to router and selects correct route but doesn't actually continue down that route. WIP --- blocks/blocks/evaluator.ts | 81 +++++++++----- executor/index.ts | 221 +++++++++++++++++++++++++++---------- 2 files changed, 214 insertions(+), 88 deletions(-) diff --git a/blocks/blocks/evaluator.ts b/blocks/blocks/evaluator.ts index 460d43df12..bafda497d4 100644 --- a/blocks/blocks/evaluator.ts +++ b/blocks/blocks/evaluator.ts @@ -3,6 +3,16 @@ import { ToolResponse } from '@/tools/types' import { MODEL_TOOLS, ModelType } from '../consts' import { BlockConfig, ParamType } from '../types' +interface TargetBlock { + id: string + type?: string + title?: string + description?: string + category?: string + subBlocks?: Record + currentState?: any +} + interface EvaluatorResponse extends ToolResponse { output: { content: string @@ -25,44 +35,55 @@ interface EvaluatorResponse extends ToolResponse { } } -export const generateEvaluatorPrompt = (prompt: string, content: string): string => { - const basePrompt = `You are an objective and meticulous evaluation agent—your role is to act as an impartial judge. Your task is to evaluate content provided in a separate sub‑block strictly based on the specific evaluation criteria supplied by the user. +export const generateEvaluatorPrompt = ( + evaluationCriteria: string, + content: string, + targetBlocks?: TargetBlock[] +): string => { + const basePrompt = `You are an objective evaluation agent. Analyze the content against the provided criteria and determine the next step based on the evaluation score. -Guidelines: -1. First, carefully read the evaluation criteria provided by the user. These criteria define the standards against which the content must be judged. -2. Review the content that has been separately provided. -3. Assess the content on the following key metrics: - - Accuracy: How precisely does the content align with the defined criteria? - - Completeness: Does the content thoroughly address every aspect outlined in the criteria? - - Quality: Is the content clear, coherent, and professionally presented? - - Relevance: How well does the content match the expectations and requirements stated in the criteria? +Evaluation Instructions: +1. Score the content (0 to 1) using these metrics: + - Accuracy: How well does it meet requirements? + - Completeness: Are all aspects addressed? + - Quality: Is it clear and professional? + - Relevance: Does it match the criteria? -Instructions: -- Analyze the content in the context of the provided evaluation criteria. -- Assign a numerical score between 0 (poor) and 1 (excellent) that reflects the overall performance. -- For each metric, compute a score and provide a detailed, step‑by‑step explanation of your evaluation. -- Your final output must be a valid JSON object that strictly matches the following format. Do not include any extra text, commentary, or formatting outside of this JSON structure. +2. Calculate final score: + - Average all metrics + - Round to 2 decimal places -Content to Evaluate: +Content: ${content} -Evaluation Request: ${prompt} +Criteria: +${evaluationCriteria}` + + const targetBlocksInfo = targetBlocks + ? ` +Available Destinations: +${targetBlocks + .map( + (block) => ` +ID: ${block.id} +Type: ${block.type} +Title: ${block.title} +Description: ${block.description}` + ) + .join('\n---\n')} + +Routing Rules: +- Score greater than or equal to 0.85: Choose success path block +- Score less than 0.85: Choose failure path block` + : '' + + return `${basePrompt}${targetBlocksInfo} Response Format: -{ - "score": , - "reasoning": "", - "metrics": { - "accuracy": , - "completeness": , - "quality": , - "relevance": - } -} +Return ONLY the destination block ID as a single word, no punctuation or explanation. +Example: "2acd9007-27e8-4510-a487-73d3b825e7c1" -Remember: Your evaluation must be entirely unbiased and based solely on the provided criteria and content. Any output outside of the valid JSON format is unacceptable.` - - return basePrompt +Remember: Your response must be ONLY the block ID.` } export const EvaluatorBlock: BlockConfig = { diff --git a/executor/index.ts b/executor/index.ts index 8173c34a62..cadf5ce79e 100644 --- a/executor/index.ts +++ b/executor/index.ts @@ -15,6 +15,7 @@ * - Meaningful error messages are provided. */ import { getAllBlocks } from '@/blocks' +import { generateEvaluatorPrompt } from '@/blocks/blocks/evaluator' import { generateRouterPrompt } from '@/blocks/blocks/router' import { BlockOutput } from '@/blocks/types' import { BlockConfig } from '@/blocks/types' @@ -101,22 +102,47 @@ export class Executor { adjacency.set(block.id, []) } - // Populate inDegree and adjacency. For conditional connections, inDegree is handled dynamically. + // Set to track which connections are counted in inDegree. + const countedEdges = new Set<(typeof connections)[number]>() + + // Populate inDegree and adjacency. for (const conn of connections) { - // Increase inDegree only for regular (non-conditional) connections. - if (!conn.condition) { + const sourceBlock = blocks.find((b) => b.id === conn.source) + let countEdge = true + if (conn.condition) { + countEdge = false + } else if (sourceBlock && sourceBlock.metadata?.type === 'evaluator') { + // For evaluator edges, count the dependency only if the target block's config references the evaluator output. + const targetBlock = blocks.find((b) => b.id === conn.target) + if (targetBlock) { + const paramsStr = JSON.stringify(targetBlock.config.params || {}) + // Look for the evaluator block's id or normalized title (lowercase, no spaces) in the template. + const evaluatorIdRef = `<${sourceBlock.id}` + const evaluatorTitleRef = sourceBlock.metadata?.title + ? `<${sourceBlock.metadata.title.toLowerCase().replace(/\s+/g, '')}` + : '' + if ( + !( + paramsStr.includes(evaluatorIdRef) || + (evaluatorTitleRef && paramsStr.includes(evaluatorTitleRef)) + ) + ) { + countEdge = false + } + } + } + if (countEdge) { inDegree.set(conn.target, (inDegree.get(conn.target) || 0) + 1) + countedEdges.add(conn) } adjacency.get(conn.source)?.push(conn.target) } // Maps for router and conditional decisions. - // routerDecisions: router block id -> chosen target block id. - // activeConditionalPaths: conditional block id -> selected condition id. const routerDecisions = new Map() const activeConditionalPaths = new Map() - // Queue initially contains all blocks without dependencies. + // Initial queue: all blocks with zero inDegree. const queue: string[] = [] for (const [blockId, degree] of inDegree) { if (degree === 0) { @@ -124,30 +150,21 @@ export class Executor { } } - // This variable will store the output of the latest executed block. let lastOutput: BlockOutput = { response: {} } - // Process blocks layer by layer. while (queue.length > 0) { const currentLayer = [...queue] queue.length = 0 - // Filtering: only execute blocks that match router and conditional decisions. + // Filtering: only execute blocks that satisfy router and condition decisions. const executableBlocks = currentLayer.filter((blockId) => { - // First check if block is enabled const block = blocks.find((b) => b.id === blockId) - if (!block || block.enabled === false) { - return false - } + if (!block || block.enabled === false) return false - // Verify if block lies on the router's chosen path for (const [routerId, chosenPath] of routerDecisions) { - if (!this.isInChosenPath(blockId, chosenPath, routerId)) { - return false - } + if (!this.isInChosenPath(blockId, chosenPath, routerId)) return false } - // Verify if block lies on the selected conditional path for (const [conditionBlockId, selectedConditionId] of activeConditionalPaths) { const connection = connections.find( (conn) => @@ -156,43 +173,30 @@ export class Executor { conn.sourceHandle?.startsWith('condition-') ) if (connection) { - // Extract condition id from sourceHandle (format: "condition-") const connConditionId = connection.sourceHandle?.replace('condition-', '') - if (connConditionId !== selectedConditionId) { - return false - } + if (connConditionId !== selectedConditionId) return false } } return true }) - // Execute blocks in the current layer in parallel. + // Execute all blocks in the current layer in parallel. const layerResults = await Promise.all( executableBlocks.map(async (blockId) => { const block = blocks.find((b) => b.id === blockId) - if (!block) { - throw new Error(`Block ${blockId} not found`) - } + if (!block) throw new Error(`Block ${blockId} not found`) - // Resolve inputs (including template variables and env vars) for the block. const inputs = this.resolveInputs(block, context) const result = await this.executeBlock(block, inputs, context) - // Store the block output in context for later reference. context.blockStates.set(block.id, result) - // Update lastOutput to reflect the latest executed block. lastOutput = result - // For router or condition blocks, update decision maps accordingly. if (block.metadata?.type === 'router') { const routerResult = result as { response: { content: string model: string - tokens: { - prompt: number - completion: number - total: number - } + tokens: { prompt: number; completion: number; total: number } selectedPath: { blockId: string } } } @@ -215,32 +219,22 @@ export class Executor { }) ) - // After executing a layer, update in-degrees for all adjacent blocks. + // After executing a layer, update inDegree for all adjacent blocks using only counted edges. for (const finishedBlockId of layerResults) { - const neighbors = adjacency.get(finishedBlockId) || [] - for (const neighbor of neighbors) { - // Find the relevant connection from finishedBlockId to neighbor. - const connection = connections.find( - (conn) => conn.source === finishedBlockId && conn.target === neighbor - ) - if (!connection) continue - - // Regular (non-conditional) connection: always decrement. - if (!connection.sourceHandle || !connection.sourceHandle.startsWith('condition-')) { - const newDegree = (inDegree.get(neighbor) || 0) - 1 - inDegree.set(neighbor, newDegree) - if (newDegree === 0) { - queue.push(neighbor) - } + const outgoingConns = connections.filter( + (conn) => conn.source === finishedBlockId && countedEdges.has(conn) + ) + for (const conn of outgoingConns) { + if (!conn.sourceHandle || !conn.sourceHandle.startsWith('condition-')) { + const newDegree = (inDegree.get(conn.target) || 0) - 1 + inDegree.set(conn.target, newDegree) + if (newDegree === 0) queue.push(conn.target) } else { - // For a conditional connection, only decrement if the active condition matches. - const conditionId = connection.sourceHandle.replace('condition-', '') + const conditionId = conn.sourceHandle.replace('condition-', '') if (activeConditionalPaths.get(finishedBlockId) === conditionId) { - const newDegree = (inDegree.get(neighbor) || 0) - 1 - inDegree.set(neighbor, newDegree) - if (newDegree === 0) { - queue.push(neighbor) - } + const newDegree = (inDegree.get(conn.target) || 0) - 1 + inDegree.set(conn.target, newDegree) + if (newDegree === 0) queue.push(conn.target) } } } @@ -291,6 +285,16 @@ export class Executor { selectedPath: routerOutput.selectedPath, }, } + } else if (block.metadata?.type === 'evaluator') { + const evaluatorOutput = await this.executeEvaluatorBlock(block, context) + output = { + response: { + content: evaluatorOutput.content, + model: evaluatorOutput.model, + tokens: evaluatorOutput.tokens, + selectedPath: evaluatorOutput.selectedPath, + }, + } } else if (block.metadata?.type === 'condition') { const conditionResult = await this.executeConditionalBlock(block, context) output = { @@ -662,6 +666,107 @@ export class Executor { } } + /** + * Executes a router block which calculates branching decisions based on a prompt. + */ + private async executeEvaluatorBlock( + block: SerializedBlock, + context: ExecutionContext + ): Promise<{ + content: string + model: string + tokens: { + prompt: number + completion: number + total: number + } + selectedPath: { + blockId: string + blockType: string + blockTitle: string + } + }> { + // Resolve inputs for the evaluator block. + console.log('Evaluator: Resolving inputs for the evaluator block.') + const resolvedInputs = this.resolveInputs(block, context) + console.log('Evaluator: Resolved inputs:', resolvedInputs) + + console.log('Evaluator: Filtering outgoing connections for the block.') + const outgoingConnections = this.workflow.connections.filter((conn) => conn.source === block.id) + console.log('Evaluator: Outgoing connections:', outgoingConnections) + + console.log('Evaluator: Mapping target blocks from outgoing connections.') + const targetBlocks = outgoingConnections.map((conn) => { + const targetBlock = this.workflow.blocks.find((b) => b.id === conn.target) + if (!targetBlock) { + throw new Error(`Target block ${conn.target} not found`) + } + console.log('Evaluator: Found target block:', targetBlock) + return { + id: targetBlock.id, + type: targetBlock.metadata?.type, + title: targetBlock.metadata?.title, + description: targetBlock.metadata?.description, + subBlocks: targetBlock.config.params, + currentState: context.blockStates.get(targetBlock.id), + } + }) + console.log('Evaluator: Mapped target blocks:', targetBlocks) + + const evaluatorConfig = { + prompt: resolvedInputs.prompt, + content: resolvedInputs.content, + model: resolvedInputs.model, + apiKey: resolvedInputs.apiKey, + temperature: resolvedInputs.temperature || 0, + } + + const model = evaluatorConfig.model || 'gpt-4o' + const providerId = getProviderFromModel(model) + + // Add logging before making the request + console.log('Evaluator: Sending request with prompt:', evaluatorConfig.prompt) + console.log('Evaluator: Available target blocks:', targetBlocks) + + const response = await executeProviderRequest(providerId, { + model: evaluatorConfig.model, + systemPrompt: generateEvaluatorPrompt( + evaluatorConfig.prompt, + evaluatorConfig.content, + targetBlocks + ), + messages: [{ role: 'user', content: evaluatorConfig.prompt }], + temperature: evaluatorConfig.temperature, + apiKey: evaluatorConfig.apiKey, + }) + + // Add logging after getting the response + console.log('Evaluator: Raw response:', response) + console.log('Evaluator: Chosen block ID:', response.content.trim().toLowerCase()) + + const chosenBlockId = response.content.trim().toLowerCase() + const chosenBlock = targetBlocks.find((b) => b.id === chosenBlockId) + if (!chosenBlock) { + throw new Error(`Invalid routing decision: ${chosenBlockId}`) + } + + const tokens = response.tokens || { prompt: 0, completion: 0, total: 0 } + return { + content: resolvedInputs.prompt, + model: response.model, + tokens: { + prompt: tokens.prompt || 0, + completion: tokens.completion || 0, + total: tokens.total || 0, + }, + selectedPath: { + blockId: chosenBlock.id, + blockType: chosenBlock.type || 'unknown', + blockTitle: chosenBlock.title || 'Untitled Block', + }, + } + } + /** * Determines whether a block is reachable along the chosen router path. *