From dfa9275aa86dcd1146429127596e52305eaa4dd3 Mon Sep 17 00:00:00 2001
From: Waleed Latif <walif6@gmail.com>
Date: Sat, 15 Mar 2025 04:19:23 -0700
Subject: [PATCH] fix(evaluator): fix evaluator block to return lowercase
 metric names

---
 sim/app/blocks/blocks/evaluator.ts | 10 +++++-----
 sim/app/executor/handlers.ts       | 13 +++++++------
 2 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/sim/app/blocks/blocks/evaluator.ts b/sim/app/blocks/blocks/evaluator.ts
index 40621fc6c0..9ec6eb222c 100644
--- a/sim/app/blocks/blocks/evaluator.ts
+++ b/sim/app/blocks/blocks/evaluator.ts
@@ -62,7 +62,7 @@ export const generateEvaluatorPrompt = (metrics: Metric[], content: string): str
   // Generate an example of the expected output format
   const exampleOutput = metrics.reduce(
     (acc, metric) => {
-      acc[metric.name] = Math.floor((metric.range.min + metric.range.max) / 2) // Use middle of range as example
+      acc[metric.name.toLowerCase()] = Math.floor((metric.range.min + metric.range.max) / 2) // Use middle of range as example
       return acc
     },
     {} as Record<string, number>
@@ -74,7 +74,7 @@ Evaluation Instructions:
 - You MUST evaluate the content against each metric
 - For each metric, provide a numeric score within the specified range
 - Your response MUST be a valid JSON object with each metric name as a key and a numeric score as the value
-- Use EXACTLY the metric names provided (case-sensitive, no modifications)
+- IMPORTANT: Use lowercase versions of the metric names as keys in your JSON response
 - Follow the exact schema of the response format provided to you
 - Do not include explanations in the JSON - only numeric scores
 - Do not add any additional fields not specified in the schema
@@ -89,7 +89,7 @@ ${formattedContent}
 Example of expected response format (with different scores):
 ${JSON.stringify(exampleOutput, null, 2)}
 
-Remember: Your response MUST be a valid JSON object containing only the metrics as keys with their numeric scores as values. No text explanations.`
+Remember: Your response MUST be a valid JSON object containing only the lowercase metric names as keys with their numeric scores as values. No text explanations.`
 }
 
 // Simplified response format generator that matches the agent block schema structure
@@ -99,7 +99,7 @@ const generateResponseFormat = (metrics: Metric[]) => {
 
   // Add each metric as a property
   metrics.forEach((metric) => {
-    properties[metric.name] = {
+    properties[metric.name.toLowerCase()] = {
       type: 'number',
       description: `${metric.description} (Score between ${metric.range.min}-${metric.range.max})`,
     }
@@ -111,7 +111,7 @@ const generateResponseFormat = (metrics: Metric[]) => {
     schema: {
       type: 'object',
       properties,
-      required: metrics.map((metric) => metric.name),
+      required: metrics.map((metric) => metric.name.toLowerCase()),
       additionalProperties: false,
     },
     strict: true,
diff --git a/sim/app/executor/handlers.ts b/sim/app/executor/handlers.ts
index 0197e58ebd..dd822a5103 100644
--- a/sim/app/executor/handlers.ts
+++ b/sim/app/executor/handlers.ts
@@ -693,14 +693,15 @@ export class EvaluatorBlockHandler implements BlockHandler {
       if (Object.keys(parsedContent).length > 0) {
         metrics.forEach((metric: { name: string }) => {
           const metricName = metric.name
+          const lowerCaseMetricName = metricName.toLowerCase()
 
           // Try multiple possible ways the metric might be represented
           if (parsedContent[metricName] !== undefined) {
-            metricScores[metricName] = Number(parsedContent[metricName])
+            metricScores[lowerCaseMetricName] = Number(parsedContent[metricName])
           } else if (parsedContent[metricName.toLowerCase()] !== undefined) {
-            metricScores[metricName] = Number(parsedContent[metricName.toLowerCase()])
+            metricScores[lowerCaseMetricName] = Number(parsedContent[metricName.toLowerCase()])
           } else if (parsedContent[metricName.toUpperCase()] !== undefined) {
-            metricScores[metricName] = Number(parsedContent[metricName.toUpperCase()])
+            metricScores[lowerCaseMetricName] = Number(parsedContent[metricName.toUpperCase()])
           } else {
             // Last resort - try to find any key that might contain this metric name
             const matchingKey = Object.keys(parsedContent).find((key) =>
@@ -708,17 +709,17 @@ export class EvaluatorBlockHandler implements BlockHandler {
             )
 
             if (matchingKey) {
-              metricScores[metricName] = Number(parsedContent[matchingKey])
+              metricScores[lowerCaseMetricName] = Number(parsedContent[matchingKey])
             } else {
               logger.warn(`Metric "${metricName}" not found in LLM response`)
-              metricScores[metricName] = 0
+              metricScores[lowerCaseMetricName] = 0
             }
           }
         })
       } else {
         // If we couldn't parse any content, set all metrics to 0
         metrics.forEach((metric: { name: string }) => {
-          metricScores[metric.name] = 0
+          metricScores[metric.name.toLowerCase()] = 0
         })
       }
     } catch (e) {