diff --git a/blocks/blocks/evaluator.ts b/blocks/blocks/evaluator.ts index c4eb746a7d..5fe3f1c348 100644 --- a/blocks/blocks/evaluator.ts +++ b/blocks/blocks/evaluator.ts @@ -32,6 +32,7 @@ interface EvaluatorResponse extends ToolResponse { blockType: string blockTitle: string } + justification: string } } @@ -64,9 +65,13 @@ ${evaluationCriteria}` return `${basePrompt} Response Format: -Return ONLY "end" to indicate no further routing is needed. +Return a JSON object with the following structure: +{ + "decision": "end", + "justification": "Brief explanation of the pure evaluation of the content. DO NOT include any information about the target blocks." +} -Remember: Your response must be ONLY the word "end" - no additional text, formatting, or explanation.` +Remember: Your response must be ONLY the JSON object - no additional text, formatting, or explanation.` } const targetBlocksInfo = ` @@ -92,11 +97,14 @@ ${ return `${basePrompt}${targetBlocksInfo} Response Format: -Return ONLY the destination block ID as a single word, no punctuation or explanation. -Example: "2acd9007-27e8-4510-a487-73d3b825e7c1" +Return a JSON object with the following structure: +{ + "decision": "block-id-here", + "justification": "Brief explanation of the pure evaluation of the content. DO NOT include any information about the target blocks." +} -Remember: Your response must be ONLY the block ID - no additional text, formatting, or explanation. -If there is only one available destination, return that block's ID regardless of the score.` +Remember: Your response must be ONLY the JSON object - no additional text, formatting, or explanation. +If there is only one available destination, return that block's ID in the decision field regardless of the score.` } export const EvaluatorBlock: BlockConfig = { @@ -146,6 +154,7 @@ export const EvaluatorBlock: BlockConfig = { tokens: 'any', evaluation: 'json', selectedPath: 'json', + justification: 'string', }, }, }, diff --git a/executor/index.ts b/executor/index.ts index 9c24506c3f..16aa6302ce 100644 --- a/executor/index.ts +++ b/executor/index.ts @@ -234,6 +234,7 @@ export class Executor { model: string tokens: { prompt: number; completion: number; total: number } selectedPath: { blockId: string } + justification: string } } evaluatorDecisions.set(block.id, evaluatorResult.response.selectedPath.blockId) @@ -400,6 +401,7 @@ export class Executor { model: evaluatorOutput.model, tokens: evaluatorOutput.tokens, selectedPath: evaluatorOutput.selectedPath, + justification: evaluatorOutput.justification, }, } } else if (block.metadata?.type === 'condition') { @@ -672,6 +674,7 @@ export class Executor { blockType: string blockTitle: string } + justification: string }> { // Resolve inputs for the evaluator block. const resolvedInputs = this.resolveInputs(block, context) @@ -686,44 +689,45 @@ export class Executor { } return { id: targetBlock.id, - type: targetBlock.metadata?.type, - title: targetBlock.metadata?.title, + type: targetBlock.metadata?.type || 'unknown', + title: targetBlock.metadata?.title || 'Untitled Block', description: targetBlock.metadata?.description, subBlocks: targetBlock.config.params, currentState: context.blockStates.get(targetBlock.id), } }) - const evaluatorConfig = { - prompt: resolvedInputs.prompt, - content: resolvedInputs.content, - model: resolvedInputs.model, - apiKey: resolvedInputs.apiKey, - temperature: resolvedInputs.temperature || 0, - } - - const model = evaluatorConfig.model || 'gpt-4o' + const model = resolvedInputs.model || 'gpt-4o' const providerId = getProviderFromModel(model) // Generate and execute the evaluator prompt const response = await executeProviderRequest(providerId, { - model: evaluatorConfig.model, + model: resolvedInputs.model, systemPrompt: generateEvaluatorPrompt( - evaluatorConfig.prompt, - evaluatorConfig.content, + resolvedInputs.prompt, + resolvedInputs.content, targetBlocks ), - messages: [{ role: 'user', content: evaluatorConfig.prompt }], - temperature: evaluatorConfig.temperature, - apiKey: evaluatorConfig.apiKey, + messages: [{ role: 'user', content: resolvedInputs.prompt }], + temperature: resolvedInputs.temperature || 0, + apiKey: resolvedInputs.apiKey, }) - const chosenBlockId = response.content.trim().toLowerCase() + // Parse the evaluator response as JSON + let evaluatorResponse + try { + evaluatorResponse = JSON.parse(response.content.trim()) + } catch (e) { + throw new Error(`Invalid evaluator response format: ${response.content}`) + } + + const chosenBlockId = evaluatorResponse.decision + const justification = evaluatorResponse.justification // Handle case where evaluator has no targets if (chosenBlockId === 'end') { const result = { - content: evaluatorConfig.content, + content: resolvedInputs.content, model: response.model, tokens: { prompt: response.tokens?.prompt || 0, @@ -735,6 +739,7 @@ export class Executor { blockType: '', blockTitle: '', }, + justification, } context.blockStates.set(block.id, { @@ -749,24 +754,22 @@ export class Executor { throw new Error(`Invalid evaluation decision: ${chosenBlockId}`) } - // Store the evaluation result in the context - const tokens = response.tokens || { prompt: 0, completion: 0, total: 0 } const result = { - content: evaluatorConfig.content, + content: resolvedInputs.content, model: response.model, tokens: { - prompt: tokens.prompt || 0, - completion: tokens.completion || 0, - total: tokens.total || 0, + prompt: response.tokens?.prompt || 0, + completion: response.tokens?.completion || 0, + total: response.tokens?.total || 0, }, selectedPath: { blockId: chosenBlock.id, - blockType: chosenBlock.type || 'unknown', - blockTitle: chosenBlock.title || 'Untitled Block', + blockType: chosenBlock.type, + blockTitle: chosenBlock.title, }, + justification, } - // ADDED: Explicitly store the evaluation decision in the context context.blockStates.set(block.id, { response: result, })