fix(evaluator): fix evaluator block to return lowercase metric names

This commit is contained in:
Waleed Latif
2025-03-15 04:19:24 -07:00
parent 1f71f5cc47
commit dfa9275aa8
2 changed files with 12 additions and 11 deletions
+5 -5
View File
@@ -62,7 +62,7 @@ export const generateEvaluatorPrompt = (metrics: Metric[], content: string): str
// Generate an example of the expected output format
const exampleOutput = metrics.reduce(
(acc, metric) => {
acc[metric.name] = Math.floor((metric.range.min + metric.range.max) / 2) // Use middle of range as example
acc[metric.name.toLowerCase()] = Math.floor((metric.range.min + metric.range.max) / 2) // Use middle of range as example
return acc
},
{} as Record<string, number>
@@ -74,7 +74,7 @@ Evaluation Instructions:
- You MUST evaluate the content against each metric
- For each metric, provide a numeric score within the specified range
- Your response MUST be a valid JSON object with each metric name as a key and a numeric score as the value
- Use EXACTLY the metric names provided (case-sensitive, no modifications)
- IMPORTANT: Use lowercase versions of the metric names as keys in your JSON response
- Follow the exact schema of the response format provided to you
- Do not include explanations in the JSON - only numeric scores
- Do not add any additional fields not specified in the schema
@@ -89,7 +89,7 @@ ${formattedContent}
Example of expected response format (with different scores):
${JSON.stringify(exampleOutput, null, 2)}
Remember: Your response MUST be a valid JSON object containing only the metrics as keys with their numeric scores as values. No text explanations.`
Remember: Your response MUST be a valid JSON object containing only the lowercase metric names as keys with their numeric scores as values. No text explanations.`
}
// Simplified response format generator that matches the agent block schema structure
@@ -99,7 +99,7 @@ const generateResponseFormat = (metrics: Metric[]) => {
// Add each metric as a property
metrics.forEach((metric) => {
properties[metric.name] = {
properties[metric.name.toLowerCase()] = {
type: 'number',
description: `${metric.description} (Score between ${metric.range.min}-${metric.range.max})`,
}
@@ -111,7 +111,7 @@ const generateResponseFormat = (metrics: Metric[]) => {
schema: {
type: 'object',
properties,
required: metrics.map((metric) => metric.name),
required: metrics.map((metric) => metric.name.toLowerCase()),
additionalProperties: false,
},
strict: true,
+7 -6
View File
@@ -693,14 +693,15 @@ export class EvaluatorBlockHandler implements BlockHandler {
if (Object.keys(parsedContent).length > 0) {
metrics.forEach((metric: { name: string }) => {
const metricName = metric.name
const lowerCaseMetricName = metricName.toLowerCase()
// Try multiple possible ways the metric might be represented
if (parsedContent[metricName] !== undefined) {
metricScores[metricName] = Number(parsedContent[metricName])
metricScores[lowerCaseMetricName] = Number(parsedContent[metricName])
} else if (parsedContent[metricName.toLowerCase()] !== undefined) {
metricScores[metricName] = Number(parsedContent[metricName.toLowerCase()])
metricScores[lowerCaseMetricName] = Number(parsedContent[metricName.toLowerCase()])
} else if (parsedContent[metricName.toUpperCase()] !== undefined) {
metricScores[metricName] = Number(parsedContent[metricName.toUpperCase()])
metricScores[lowerCaseMetricName] = Number(parsedContent[metricName.toUpperCase()])
} else {
// Last resort - try to find any key that might contain this metric name
const matchingKey = Object.keys(parsedContent).find((key) =>
@@ -708,17 +709,17 @@ export class EvaluatorBlockHandler implements BlockHandler {
)
if (matchingKey) {
metricScores[metricName] = Number(parsedContent[matchingKey])
metricScores[lowerCaseMetricName] = Number(parsedContent[matchingKey])
} else {
logger.warn(`Metric "${metricName}" not found in LLM response`)
metricScores[metricName] = 0
metricScores[lowerCaseMetricName] = 0
}
}
})
} else {
// If we couldn't parse any content, set all metrics to 0
metrics.forEach((metric: { name: string }) => {
metricScores[metric.name] = 0
metricScores[metric.name.toLowerCase()] = 0
})
}
} catch (e) {