From bb5ca1ad745215247f69bc3880cfa0ea06f7edfc Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 10 Feb 2025 13:51:29 -0800 Subject: [PATCH] Added evaluator block --- blocks/blocks/evaluator.ts | 204 +++++++++++++++++++++++++++++++++++++ blocks/index.ts | 3 + 2 files changed, 207 insertions(+) create mode 100644 blocks/blocks/evaluator.ts diff --git a/blocks/blocks/evaluator.ts b/blocks/blocks/evaluator.ts new file mode 100644 index 0000000000..5dd1e3b94c --- /dev/null +++ b/blocks/blocks/evaluator.ts @@ -0,0 +1,204 @@ +import { ChartBarIcon } from '@/components/icons' +import { ToolResponse } from '@/tools/types' +import { MODEL_TOOLS, ModelType } from '../consts' +import { BlockConfig, ParamType } from '../types' + +interface EvaluatorResponse extends ToolResponse { + output: { + content: string + model: string + tokens?: { + prompt?: number + completion?: number + total?: number + } + evaluation: { + score: number + reasoning: string + metrics: Record + } + } +} + +interface EvaluationTarget { + id: string + type?: string + title?: string + description?: string + category?: string + subBlocks?: Record + currentState?: any + expectedOutput?: any + actualOutput?: any +} + +export const generateEvaluatorPrompt = (prompt: string, target?: EvaluationTarget): string => { + const basePrompt = `You are an intelligent evaluation agent responsible for assessing the quality and accuracy of workflow outputs. Your task is to analyze the input and provide a detailed evaluation with scoring. + +Key Instructions: +1. You MUST provide a numerical score between 0 and 1 (0 being worst, 1 being perfect) +2. You MUST provide detailed reasoning for your evaluation +3. You MUST evaluate based on multiple metrics where applicable + +Evaluation Framework: +- Accuracy: How well does the output match the expected result? +- Completeness: Does the output address all aspects of the task? +- Quality: Is the output well-formed and professional? +- Relevance: Is the output relevant to the original request?` + + const targetInfo = target + ? ` + +Evaluation Target: +${JSON.stringify( + { + id: target.id, + type: target.type, + title: target.title, + description: target.description, + category: target.category, + expectedOutput: target.expectedOutput, + actualOutput: target.actualOutput, + }, + null, + 2 +)} + +Evaluation Instructions: +1. Compare the actual output against: + - Expected output (if provided) + - Task requirements + - Quality standards + - Technical constraints + +2. Scoring Criteria: + - Use objective metrics where possible + - Consider both technical and functional aspects + - Factor in any specific requirements or constraints + - Evaluate against industry best practices` + : '' + + return `${basePrompt}${targetInfo} + +Evaluation Request: ${prompt} + +Response Format: +{ + "score": , + "reasoning": "", + "metrics": { + "accuracy": , + "completeness": , + "quality": , + "relevance": + } +} + +Remember: Your response must be valid JSON following the exact format above.` +} + +export const EvaluatorBlock: BlockConfig = { + type: 'evaluator', + toolbar: { + title: 'Evaluator', + description: 'Add an evaluator', + bgColor: '#FF69B4', + icon: ChartBarIcon, + category: 'blocks', + }, + tools: { + access: [ + 'openai_chat', + 'anthropic_chat', + 'google_chat', + 'xai_chat', + 'deepseek_chat', + 'deepseek_reasoner', + ], + config: { + tool: (params: Record) => { + const model = params.model || 'gpt-4o' + if (!model) { + throw new Error('No model selected') + } + const tool = MODEL_TOOLS[model as ModelType] + if (!tool) { + throw new Error(`Invalid model selected: ${model}`) + } + return tool + }, + }, + }, + workflow: { + inputs: { + prompt: { type: 'string' as ParamType, required: true }, + model: { type: 'string' as ParamType, required: true }, + apiKey: { type: 'string' as ParamType, required: true }, + expectedOutput: { type: 'object' as ParamType, required: false }, + actualOutput: { type: 'object' as ParamType, required: true }, + }, + outputs: { + response: { + type: { + content: 'string', + model: 'string', + tokens: 'any', + evaluation: 'json', + }, + }, + }, + subBlocks: [ + { + id: 'prompt', + title: 'Evaluation Prompt', + type: 'long-input', + layout: 'full', + placeholder: 'Evaluate the output based on the following criteria...', + }, + { + id: 'model', + title: 'Model', + type: 'dropdown', + layout: 'half', + options: Object.keys(MODEL_TOOLS), + }, + { + id: 'apiKey', + title: 'API Key', + type: 'short-input', + layout: 'full', + placeholder: 'Enter your API key', + password: true, + connectionDroppable: false, + }, + { + id: 'expectedOutput', + title: 'Expected Output', + type: 'code', + layout: 'full', + placeholder: 'Enter the expected output (optional)', + }, + { + id: 'actualOutput', + title: 'Actual Output', + type: 'code', + layout: 'full', + placeholder: 'Enter the actual output to evaluate', + }, + { + id: 'systemPrompt', + title: 'System Prompt', + type: 'code', + layout: 'full', + hidden: true, + value: (params: Record) => { + return generateEvaluatorPrompt(params.prompt || '', { + id: 'runtime-evaluation', + expectedOutput: params.expectedOutput, + actualOutput: params.actualOutput, + }) + }, + }, + ], + }, +} diff --git a/blocks/index.ts b/blocks/index.ts index ebf4393321..0310d4fac0 100644 --- a/blocks/index.ts +++ b/blocks/index.ts @@ -3,6 +3,7 @@ import { AgentBlock } from './blocks/agent' import { ApiBlock } from './blocks/api' import { ConditionBlock } from './blocks/condition' import { CrewAIVisionBlock } from './blocks/crewai' +import { EvaluatorBlock } from './blocks/evaluator' import { FirecrawlScrapeBlock } from './blocks/firecrawl' import { FunctionBlock } from './blocks/function' import { GitHubBlock } from './blocks/github' @@ -33,6 +34,7 @@ export { SerperBlock, TavilyBlock, RouterBlock, + EvaluatorBlock, YouTubeSearchBlock, NotionBlock, GmailBlock, @@ -46,6 +48,7 @@ const blocks: Record = { condition: ConditionBlock, function: FunctionBlock, router: RouterBlock, + evaluator: EvaluatorBlock, crewai_vision: CrewAIVisionBlock, firecrawl_scrape: FirecrawlScrapeBlock, jina_reader: JinaBlock,