changeset

.changeset/breezy-zebras-enjoy.md

@@ -0,0 +1,5 @@
+---
+"claude-dev": patch
+---
+
+Improved eval framework with more logging and run options

evals/cli/src/commands/runDiffEval.ts

@@ -6,6 +6,7 @@ interface RunDiffEvalOptions {
 	modelIds: string
 	systemPromptName: string
 	validAttemptsPerCase: number
+	maxAttemptsPerCase?: number
 	parsingFunction: string
 	diffEditFunction: string
 	thinkingBudget: number
@@ -71,6 +72,10 @@ export async function runDiffEvalHandler(options: RunDiffEvalOptions) {
 		args.push("--verbose")
 	}
 
+	if (options.maxAttemptsPerCase) {
+		args.push("--max-attempts-per-case", String(options.maxAttemptsPerCase))
+	}
+
 	if (options.maxCases) {
 		args.push("--max-cases", String(options.maxCases))
 	}

evals/cli/src/index.ts

@@ -87,6 +87,7 @@ program
 	.option("--model-ids <model_ids>", "Comma-separated list of model IDs to test")
 	.option("--system-prompt-name <name>", "The name of the system prompt to use", "basicSystemPrompt")
 	.option("-n, --valid-attempts-per-case <number>", "Number of valid attempts per test case per model (will retry until this many valid attempts are collected)", "1")
+	.option("--max-attempts-per-case <number>", "Maximum total attempts per test case (default: 10x valid attempts)")
 	.option("--max-cases <number>", "Maximum number of test cases to run (limits total cases loaded)")
 	.option("--parsing-function <name>", "The parsing function to use", "parseAssistantMessageV2")
 	.option("--diff-edit-function <name>", "The diff editing function to use", "constructNewFileContentV2")
@@ -102,6 +103,7 @@ program
 			const fullOptions = {
 				...options,
 				validAttemptsPerCase: parseInt(options.validAttemptsPerCase, 10),
+				maxAttemptsPerCase: options.maxAttemptsPerCase ? parseInt(options.maxAttemptsPerCase, 10) : undefined,
 				thinkingBudget: parseInt(options.thinkingBudget, 10),
 				maxCases: options.maxCases ? parseInt(options.maxCases, 10) : undefined,
 			}

evals/diff-edits/ClineWrapper.ts

@@ -30,6 +30,7 @@ const diffEditingFunctions: Record<string, ConstructNewFileContentFn> = {
 }
 
 import { TestInput, TestResult, ExtractedToolCall } from "./types"
+import { log } from "./helpers"
 export { TestInput, TestResult, ExtractedToolCall }
 
 interface StreamResult {
@@ -284,21 +285,21 @@ export async function runSingleEvaluation(input: TestInput): Promise<TestResult>
 		}
 
 		// check that we are editing the correct file path
-		console.log(`Expected file path: "${originalFilePath}"`);
-		console.log(`Actual file path used: "${diffToolPath}"`);
+		log(input.isVerbose, `Expected file path: "${originalFilePath}"`)
+		log(input.isVerbose, `Actual file path used: "${diffToolPath}"`)
 		if (diffToolPath !== originalFilePath) {
-			console.log(`❌ File path mismatch detected!`);
+			log(input.isVerbose, `❌ File path mismatch detected!`)
 			// Enhanced logging:
 			if (streamResult?.assistantMessage) {
-				console.log(`   Full model output (assistantMessage):`);
-				console.log(`   -----------------------------------------`);
-				console.log(`   ${streamResult.assistantMessage}`);
-				console.log(`   -----------------------------------------`);
+				log(input.isVerbose, `   Full model output (assistantMessage):`)
+				log(input.isVerbose, `   -----------------------------------------`)
+				log(input.isVerbose, `   ${streamResult.assistantMessage}`)
+				log(input.isVerbose, `   -----------------------------------------`)
 			}
 			if (toolCall) {
-				console.log(`   Parsed tool call that caused mismatch:`);
-				console.log(`   ${JSON.stringify(toolCall, null, 2)}`);
-				console.log(`   -----------------------------------------`);
+				log(input.isVerbose, `   Parsed tool call that caused mismatch:`)
+				log(input.isVerbose, `   ${JSON.stringify(toolCall, null, 2)}`)
+				log(input.isVerbose, `   -----------------------------------------`)
 			}
 			return {
 				success: false,
@@ -321,7 +322,7 @@ export async function runSingleEvaluation(input: TestInput): Promise<TestResult>
 			// If it's just a string, diffSuccess stays true and replacementData stays undefined
 		} catch (error: any) {
 			diffSuccess = false
-			console.log("ERROR:",error)
+			log(input.isVerbose, `ERROR: ${error}`)
 		}
 
 		return {

evals/diff-edits/TestRunner.ts

@@ -7,7 +7,7 @@ import { constructNewFileContent as constructNewFileContent_06_26_25 } from "./d
 import { constructNewFileContent as constructNewFileContentV3 } from "../../src/core/assistant-message/diff"
 import { basicSystemPrompt } from "./prompts/basicSystemPrompt-06-06-25"
 import { claude4SystemPrompt } from "./prompts/claude4SystemPrompt-06-06-25"
-import { formatResponse } from "./helpers"
+import { formatResponse, log } from "./helpers"
 import { Anthropic } from "@anthropic-ai/sdk"
 import * as fs from "fs"
 import * as path from "path"
@@ -40,12 +40,6 @@ const encoding = get_encoding("cl100k_base");
 
 let openRouterModelDataGlobal: Record<string, EvalOpenRouterModelInfo> = {}; // Global to store fetched data
 
-function log(isVerbose: boolean, message: string) {
-	if (isVerbose) {
-		console.log(message)
-	}
-}
-
 const systemPromptGeneratorLookup: Record<string, ConstructSystemPromptFn> = {
 	basicSystemPrompt: basicSystemPrompt,
 	claude4SystemPrompt: claude4SystemPrompt,
@@ -641,6 +635,7 @@ class NodeTestRunner {
 			thinkingBudgetTokens: testConfig.thinking_tokens_budget,
 			originalDiffEditToolCallMessage: testConfig.replay ? testCase.original_diff_edit_tool_call_message : undefined,
 			diffApplyFile: testConfig.diff_apply_file,
+			isVerbose: isVerbose,
 		}
 
 		if (isVerbose) {
@@ -807,8 +802,8 @@ class NodeTestRunner {
 					log(isVerbose, `Warning: Failed to store result in database: ${error}`);
 				}
 				
-				// Safety check to prevent infinite loops - limit to 10 attempts per valid attempt requested
-				if (totalAttempts >= testConfig.number_of_runs * 10) {
+				// Safety check to prevent infinite loops - use configurable max attempts limit
+				if (totalAttempts >= testConfig.max_attempts_per_case) {
 					log(isVerbose, `  ⚠️ Reached maximum attempts (${totalAttempts}) for test case ${testCase.test_id}. Only got ${validAttempts}/${testConfig.number_of_runs} valid attempts.`);
 					break;
 				}
@@ -927,6 +922,7 @@ async function main() {
 		.option("--model-ids <model_ids>", "Comma-separated list of model IDs to test")
 		.option("--system-prompt-name <name>", "The name of the system prompt to use", "basicSystemPrompt")
 		.option("-n, --valid-attempts-per-case <number>", "Number of valid attempts per test case per model (will retry until this many valid attempts are collected)", "1")
+		.option("--max-attempts-per-case <number>", "Maximum total attempts per test case (default: 10x valid attempts)")
 		.option("--max-cases <number>", "Maximum number of test cases to run (limits total cases loaded)")
 		.option("--parsing-function <name>", "The parsing function to use", "parseAssistantMessageV2")
 		.option("--diff-edit-function <name>", "The diff editing function to use", "diff-06-25-25")
@@ -957,6 +953,11 @@ async function main() {
 	}
 
 	const validAttemptsPerCase = parseInt(options.validAttemptsPerCase, 10);
+	
+	// Compute dynamic default for max attempts: 10x valid attempts if not specified
+	const maxAttemptsPerCase = options.maxAttemptsPerCase 
+		? parseInt(options.maxAttemptsPerCase, 10)
+		: validAttemptsPerCase * 10;
 
 	const runner = new NodeTestRunner(options.replay || !!options.replayRunId)
 
@@ -1062,6 +1063,7 @@ async function main() {
 					model_id: modelId,
 					system_prompt_name: options.systemPromptName,
 					number_of_runs: validAttemptsPerCase,
+					max_attempts_per_case: maxAttemptsPerCase,
 					parsing_function: options.parsingFunction,
 					diff_edit_function: options.diffEditFunction,
 					thinking_tokens_budget: parseInt(options.thinkingBudget, 10),
@@ -1129,7 +1131,7 @@ async function main() {
 
 			remainingTasks = remainingTasks.filter(task => {
 				const taskId = `${task.modelId}-${task.testCase.test_id}`;
-				if (taskStates[taskId].total >= validAttemptsPerCase * 10) {
+				if (taskStates[taskId].total >= task.testConfig.max_attempts_per_case) {
 					log(isVerbose, `  ⚠️ Reached maximum attempts for ${task.testCase.test_id} with ${task.modelId}.`);
 					return false;
 				}

evals/diff-edits/helpers.ts

@@ -23,3 +23,9 @@ export const formatResponse = {
 		return formatImagesIntoBlocks(images)
 	},
 }
+
+export function log(isVerbose: boolean, message: string) {
+	if (isVerbose) {
+		console.log(message)
+	}
+}

evals/diff-edits/types.ts

@@ -29,6 +29,7 @@ export interface TestConfig {
 	model_id: string
 	system_prompt_name: string
 	number_of_runs: number
+	max_attempts_per_case: number
 	parsing_function: string
 	diff_edit_function: string
 	thinking_tokens_budget: number
@@ -103,4 +104,5 @@ export interface TestInput {
 	thinkingBudgetTokens: number
 	originalDiffEditToolCallMessage?: string
 	diffApplyFile?: string
+	isVerbose: boolean
 }