diff --git a/blocks/blocks/evaluator.ts b/blocks/blocks/evaluator.ts index 5fe3f1c348..8fa940fe8b 100644 --- a/blocks/blocks/evaluator.ts +++ b/blocks/blocks/evaluator.ts @@ -33,13 +33,15 @@ interface EvaluatorResponse extends ToolResponse { blockTitle: string } justification: string + history: Array<{ response: string; justification: string }> } } export const generateEvaluatorPrompt = ( evaluationCriteria: string, content: string, - targetBlocks?: TargetBlock[] + targetBlocks?: TargetBlock[], + history?: Array<{ response: string; justification: string }> ): string => { const basePrompt = `You are an objective evaluation agent. Analyze the content against the provided criteria and determine the next step based on the evaluation score. @@ -52,7 +54,22 @@ Evaluation Instructions: 2. Calculate final score: - Average all metrics - - Round to 2 decimal places + - Round to 2 decimal places${ + history && history.length > 0 + ? ` + +Previous Attempts: +${history + .map( + (entry, i) => ` +Attempt ${i + 1}: +Response: ${entry.response} +Evaluation: ${entry.justification} +---` + ) + .join('\n')}` + : '' + } Content: ${content} @@ -145,6 +162,7 @@ export const EvaluatorBlock: BlockConfig = { model: { type: 'string' as ParamType, required: true }, apiKey: { type: 'string' as ParamType, required: true }, content: { type: 'string' as ParamType, required: true }, + history: { type: 'json' as ParamType, required: false }, }, outputs: { response: { @@ -155,6 +173,7 @@ export const EvaluatorBlock: BlockConfig = { evaluation: 'json', selectedPath: 'json', justification: 'string', + history: 'json', }, }, }, @@ -196,7 +215,12 @@ export const EvaluatorBlock: BlockConfig = { layout: 'full', hidden: true, value: (params: Record) => { - return generateEvaluatorPrompt(params.prompt || '', params.content || '') + return generateEvaluatorPrompt( + params.prompt || '', + params.content || '', + undefined, + params.history || [] + ) }, }, ], diff --git a/executor/index.ts b/executor/index.ts index 16aa6302ce..6ed3cfdc05 100644 --- a/executor/index.ts +++ b/executor/index.ts @@ -235,6 +235,7 @@ export class Executor { tokens: { prompt: number; completion: number; total: number } selectedPath: { blockId: string } justification: string + history: Array<{ response: string; justification: string }> } } evaluatorDecisions.set(block.id, evaluatorResult.response.selectedPath.blockId) @@ -402,6 +403,7 @@ export class Executor { tokens: evaluatorOutput.tokens, selectedPath: evaluatorOutput.selectedPath, justification: evaluatorOutput.justification, + history: evaluatorOutput.history, }, } } else if (block.metadata?.type === 'condition') { @@ -675,6 +677,7 @@ export class Executor { blockTitle: string } justification: string + history: Array<{ response: string; justification: string }> }> { // Resolve inputs for the evaluator block. const resolvedInputs = this.resolveInputs(block, context) @@ -697,6 +700,16 @@ export class Executor { } }) + // Get history from previous state if it exists, otherwise initialize empty + let history: Array<{ response: string; justification: string }> = [] + const previousState = context.blockStates.get(block.id) + if (previousState && typeof previousState === 'object' && 'response' in previousState) { + const response = previousState.response + if (response && typeof response === 'object' && 'history' in response) { + history = response.history as Array<{ response: string; justification: string }> + } + } + const model = resolvedInputs.model || 'gpt-4o' const providerId = getProviderFromModel(model) @@ -706,7 +719,8 @@ export class Executor { systemPrompt: generateEvaluatorPrompt( resolvedInputs.prompt, resolvedInputs.content, - targetBlocks + targetBlocks, + history ), messages: [{ role: 'user', content: resolvedInputs.prompt }], temperature: resolvedInputs.temperature || 0, @@ -724,6 +738,15 @@ export class Executor { const chosenBlockId = evaluatorResponse.decision const justification = evaluatorResponse.justification + // Update history with current response and evaluation + const updatedHistory = [ + ...history, + { + response: resolvedInputs.content, + justification, + }, + ] + // Handle case where evaluator has no targets if (chosenBlockId === 'end') { const result = { @@ -740,6 +763,7 @@ export class Executor { blockTitle: '', }, justification, + history: updatedHistory, } context.blockStates.set(block.id, { @@ -768,6 +792,7 @@ export class Executor { blockTitle: chosenBlock.title, }, justification, + history: updatedHistory, } context.blockStates.set(block.id, {