test server

evals/.gitignore

@@ -0,0 +1,3 @@
+repositories
+
+results/evals.db

evals/README.md

@@ -0,0 +1,162 @@
+# Cline Evaluation System
+
+This directory contains the evaluation system for benchmarking Cline against various coding evaluation frameworks.
+
+## Overview
+
+The Cline Evaluation System allows you to:
+
+1. Run Cline against standardized coding benchmarks
+2. Collect comprehensive metrics on performance
+3. Generate detailed reports on evaluation results
+4. Compare performance across different models and benchmarks
+
+## Architecture
+
+The evaluation system consists of two main components:
+
+1. **Test Server**: Enhanced HTTP server in `src/services/test/TestServer.ts` that provides detailed task results
+2. **CLI Tool**: Command-line interface in `evals/cli/` for orchestrating evaluations
+
+## Directory Structure
+
+```
+cline-repo/
+├── src/
+│   ├── services/
+│   │   ├── test/
+│   │   │   ├── TestServer.ts         # Enhanced HTTP server for task execution
+│   │   │   ├── GitHelper.ts          # Git utilities for file tracking
+│   │   │   └── ...
+│   │   └── ...
+│   └── ...
+├── evals/                            # Main directory for evaluation system
+│   ├── cli/                          # CLI tool for orchestrating evaluations
+│   │   ├── src/
+│   │   │   ├── index.ts              # CLI entry point
+│   │   │   ├── commands/             # CLI commands (setup, run, report)
+│   │   │   ├── adapters/             # Benchmark adapters
+│   │   │   ├── db/                   # Database management
+│   │   │   └── utils/                # Utility functions
+│   │   ├── package.json
+│   │   └── tsconfig.json
+│   ├── repositories/                 # Cloned benchmark repositories
+│   │   ├── exercism/                 # Modified Exercism (from pashpashpash/evals)
+│   │   ├── swe-bench/                # SWE-Bench repository
+│   │   ├── swelancer/                # SWELancer repository
+│   │   └── multi-swe/                # Multi-SWE-Bench repository
+│   ├── results/                      # Evaluation results storage
+│   │   ├── runs/                     # Individual run results
+│   │   └── reports/                  # Generated reports
+│   └── README.md                     # This file
+└── ...
+```
+
+## Getting Started
+
+### Prerequisites
+
+- Node.js 16+
+- VSCode with Cline extension installed
+- Git
+
+### Installation
+
+1. Build the CLI tool:
+
+```bash
+cd evals/cli
+npm install
+npm run build
+```
+
+### Usage
+
+#### Setting Up Benchmarks
+
+```bash
+cd evals/cli
+node dist/index.js setup
+```
+
+This will clone and set up all benchmark repositories. You can specify specific benchmarks:
+
+```bash
+node dist/index.js setup --benchmarks exercism
+```
+
+#### Running Evaluations
+
+```bash
+node dist/index.js run --model claude-3-opus-20240229 --benchmark exercism
+```
+
+Options:
+- `--model`: The model to evaluate (default: claude-3-opus-20240229)
+- `--benchmark`: Specific benchmark to run (default: all)
+- `--count`: Number of tasks to run (default: all)
+
+#### Generating Reports
+
+```bash
+node dist/index.js report
+```
+
+Options:
+- `--format`: Report format (json, markdown) (default: markdown)
+- `--output`: Output path for the report
+
+## Benchmarks
+
+### Exercism
+
+Modified Exercism exercises from the [pashpashpash/evals](https://github.com/pashpashpash/evals) repository. These are small, focused programming exercises in various languages.
+
+### SWE-Bench (Coming Soon)
+
+Real-world software engineering tasks from the [SWE-bench](https://github.com/SWE-bench/SWE-bench) repository.
+
+### SWELancer (Coming Soon)
+
+Freelance-style programming tasks from the SWELancer benchmark.
+
+### Multi-SWE-Bench (Coming Soon)
+
+Multi-file software engineering tasks from the Multi-SWE-Bench repository.
+
+## Metrics
+
+The evaluation system collects the following metrics:
+
+- **Token Usage**: Input and output tokens
+- **Cost**: Estimated cost of API calls
+- **Duration**: Time taken to complete tasks
+- **Tool Usage**: Number of tool calls and failures
+- **Success Rate**: Percentage of tasks completed successfully
+- **Functional Correctness**: Percentage of tests passed
+
+## Reports
+
+Reports are generated in Markdown or JSON format and include:
+
+- Overall summary
+- Benchmark-specific results
+- Model-specific results
+- Tool usage statistics
+- Charts and visualizations
+
+## Development
+
+### Adding a New Benchmark
+
+1. Create a new adapter in `evals/cli/src/adapters/`
+2. Implement the `BenchmarkAdapter` interface
+3. Register the adapter in `evals/cli/src/adapters/index.ts`
+
+### Extending Metrics
+
+To add new metrics:
+
+1. Update the database schema in `evals/cli/src/db/schema.ts`
+2. Add collection logic in `evals/cli/src/utils/results.ts`
+3. Update report generation in `evals/cli/src/commands/report.ts`

evals/cli/package.json

@@ -0,0 +1,39 @@
+{
+	"name": "cline-evaluation-cli",
+	"version": "0.1.0",
+	"description": "CLI tool for orchestrating Cline evaluations across multiple benchmarks",
+	"main": "dist/index.js",
+	"scripts": {
+		"build": "tsc",
+		"start": "node dist/index.js",
+		"dev": "ts-node src/index.ts",
+		"test": "echo \"Error: no test specified\" && exit 1"
+	},
+	"keywords": [
+		"cline",
+		"evaluation",
+		"benchmark"
+	],
+	"author": "",
+	"license": "MIT",
+	"dependencies": {
+		"better-sqlite3": "^8.0.0",
+		"chalk": "^4.1.2",
+		"commander": "^9.4.1",
+		"execa": "^5.1.1",
+		"node-fetch": "^2.7.0",
+		"ora": "^5.4.1",
+		"sqlite": "^4.1.2",
+		"uuid": "^9.0.0",
+		"yargs": "^17.6.2"
+	},
+	"devDependencies": {
+		"@types/better-sqlite3": "^7.6.3",
+		"@types/node": "^18.11.18",
+		"@types/node-fetch": "^2.6.12",
+		"@types/uuid": "^9.0.0",
+		"@types/yargs": "^17.0.19",
+		"ts-node": "^10.9.1",
+		"typescript": "^4.9.4"
+	}
+}

evals/cli/src/adapters/exercism.ts

@@ -0,0 +1,190 @@
+import * as path from "path"
+import * as fs from "fs"
+import execa from "execa"
+import { BenchmarkAdapter, Task, VerificationResult } from "./types"
+
+const EVALS_DIR = path.resolve(__dirname, "../../../")
+
+/**
+ * Adapter for the modified Exercism benchmark
+ */
+export class ExercismAdapter implements BenchmarkAdapter {
+	name = "exercism"
+
+	/**
+	 * Set up the Exercism benchmark repository
+	 */
+	async setup(): Promise<void> {
+		// Clone repository if needed
+		const exercismDir = path.join(EVALS_DIR, "repositories", "exercism")
+
+		if (!fs.existsSync(exercismDir)) {
+			console.log(`Cloning Exercism repository to ${exercismDir}...`)
+			await execa("git", ["clone", "https://github.com/pashpashpash/evals.git", exercismDir])
+			console.log("Exercism repository cloned successfully")
+		} else {
+			console.log(`Exercism repository already exists at ${exercismDir}`)
+
+			// Pull latest changes
+			console.log("Pulling latest changes...")
+			await execa("git", ["pull"], { cwd: exercismDir })
+			console.log("Repository updated successfully")
+		}
+	}
+
+	/**
+	 * List all available tasks in the Exercism benchmark
+	 */
+	async listTasks(): Promise<Task[]> {
+		const tasks: Task[] = []
+		const exercisesDir = path.join(EVALS_DIR, "repositories", "exercism")
+
+		// Ensure the repository exists
+		if (!fs.existsSync(exercisesDir)) {
+			throw new Error(`Exercism repository not found at ${exercisesDir}. Run setup first.`)
+		}
+
+		// Read language directories
+		const languages = fs
+			.readdirSync(exercisesDir)
+			.filter((dir) => fs.statSync(path.join(exercisesDir, dir)).isDirectory())
+			.filter((dir) => !dir.startsWith(".") && !["node_modules", ".git"].includes(dir))
+
+		for (const language of languages) {
+			const languageDir = path.join(exercisesDir, language)
+
+			// Read exercise directories
+			const exercises = fs.readdirSync(languageDir).filter((dir) => fs.statSync(path.join(languageDir, dir)).isDirectory())
+
+			for (const exercise of exercises) {
+				const exerciseDir = path.join(languageDir, exercise)
+
+				// Read instructions
+				let description = ""
+				const instructionsPath = path.join(exerciseDir, "docs", "instructions.md")
+				if (fs.existsSync(instructionsPath)) {
+					description = fs.readFileSync(instructionsPath, "utf-8")
+				}
+
+				// Determine test commands based on language
+				let testCommands: string[] = []
+				switch (language) {
+					case "javascript":
+						testCommands = ["npm install", "npm test"]
+						break
+					case "python":
+						testCommands = ["python -m pytest -o markers=task *_test.py"]
+						break
+					case "go":
+						testCommands = ["go test"]
+						break
+					case "java":
+						testCommands = ["./gradlew test"]
+						break
+					case "rust":
+						testCommands = ["cargo test"]
+						break
+					default:
+						testCommands = []
+				}
+
+				tasks.push({
+					id: `exercism-${language}-${exercise}`,
+					name: exercise,
+					description,
+					workspacePath: exerciseDir,
+					setupCommands: [],
+					verificationCommands: testCommands,
+					metadata: {
+						language,
+						type: "exercism",
+					},
+				})
+			}
+		}
+
+		return tasks
+	}
+
+	/**
+	 * Prepare a specific task for execution
+	 * @param taskId The ID of the task to prepare
+	 */
+	async prepareTask(taskId: string): Promise<Task> {
+		const tasks = await this.listTasks()
+		const task = tasks.find((t) => t.id === taskId)
+
+		if (!task) {
+			throw new Error(`Task ${taskId} not found`)
+		}
+
+		// Check if Git repository is already initialized
+		const gitDirExists = fs.existsSync(path.join(task.workspacePath, ".git"))
+
+		try {
+			// Initialize Git repository if needed
+			if (!gitDirExists) {
+				await execa("git", ["init"], { cwd: task.workspacePath })
+			}
+
+			// Create a dummy file to ensure there's something to commit
+			const dummyFilePath = path.join(task.workspacePath, ".eval-timestamp")
+			fs.writeFileSync(dummyFilePath, new Date().toISOString())
+
+			// Add all files and commit
+			await execa("git", ["add", "."], { cwd: task.workspacePath })
+
+			try {
+				await execa("git", ["commit", "-m", "Initial commit"], { cwd: task.workspacePath })
+			} catch (error: any) {
+				// If commit fails because there are no changes, that's okay
+				if (!error.stderr?.includes("nothing to commit")) {
+					throw error
+				}
+			}
+		} catch (error: any) {
+			console.warn(`Warning: Git operations failed: ${error.message}`)
+			console.warn("Continuing without Git initialization")
+		}
+
+		return task
+	}
+
+	/**
+	 * Verify the result of a task execution
+	 * @param task The task that was executed
+	 * @param result The result of the task execution
+	 */
+	async verifyResult(task: Task, result: any): Promise<VerificationResult> {
+		// Run verification commands
+		let success = true
+		let output = ""
+
+		for (const command of task.verificationCommands) {
+			try {
+				const [cmd, ...args] = command.split(" ")
+				const { stdout } = await execa(cmd, args, { cwd: task.workspacePath })
+				output += stdout + "\n"
+			} catch (error: any) {
+				success = false
+				if (error.stdout) output += error.stdout + "\n"
+				if (error.stderr) output += error.stderr + "\n"
+			}
+		}
+
+		// Parse test results
+		const testsPassed = (output.match(/PASS/g) || []).length
+		const testsFailed = (output.match(/FAIL/g) || []).length
+		const testsTotal = testsPassed + testsFailed
+
+		return {
+			success,
+			metrics: {
+				testsPassed,
+				testsFailed,
+				testsTotal,
+				functionalCorrectness: testsTotal > 0 ? testsPassed / testsTotal : 0,
+			},
+		}
+	}
+}

evals/cli/src/adapters/index.ts

@@ -0,0 +1,47 @@
+import { BenchmarkAdapter } from "./types"
+import { ExercismAdapter } from "./exercism"
+import { SWEBenchAdapter } from "./swe-bench"
+import { SWELancerAdapter } from "./swelancer"
+import { MultiSWEAdapter } from "./multi-swe"
+
+// Registry of all available adapters
+const adapters: Record<string, BenchmarkAdapter> = {
+	// Exercism is the primary adapter with real implementation
+	exercism: new ExercismAdapter(),
+
+	// Dummy adapters for testing
+	"swe-bench": new SWEBenchAdapter(),
+	swelancer: new SWELancerAdapter(),
+	"multi-swe": new MultiSWEAdapter(),
+}
+
+/**
+ * Get a specific adapter by name
+ * @param name The name of the adapter to get
+ * @returns The requested adapter
+ * @throws Error if the adapter is not found
+ */
+export function getAdapter(name: string): BenchmarkAdapter {
+	const adapter = adapters[name]
+	if (!adapter) {
+		throw new Error(`Adapter for benchmark '${name}' not found`)
+	}
+	return adapter
+}
+
+/**
+ * Get all available adapters
+ * @returns Array of all registered adapters
+ */
+export function getAllAdapters(): BenchmarkAdapter[] {
+	return Object.values(adapters)
+}
+
+/**
+ * Register a new adapter
+ * @param name The name to register the adapter under
+ * @param adapter The adapter to register
+ */
+export function registerAdapter(name: string, adapter: BenchmarkAdapter): void {
+	adapters[name] = adapter
+}

evals/cli/src/adapters/multi-swe.ts

@@ -0,0 +1,192 @@
+import * as path from "path"
+import * as fs from "fs"
+import execa from "execa"
+import { BenchmarkAdapter, Task, VerificationResult } from "./types"
+
+const EVALS_DIR = path.resolve(__dirname, "../../../")
+
+/**
+ * Dummy adapter for the Multi-SWE-Bench benchmark
+ */
+export class MultiSWEAdapter implements BenchmarkAdapter {
+	name = "multi-swe"
+
+	/**
+	 * Set up the Multi-SWE-Bench benchmark repository (dummy implementation)
+	 */
+	async setup(): Promise<void> {
+		console.log("Multi-SWE-Bench dummy setup completed")
+
+		// Create repositories directory if it doesn't exist
+		const repoDir = path.join(EVALS_DIR, "repositories", "multi-swe")
+		if (!fs.existsSync(repoDir)) {
+			fs.mkdirSync(repoDir, { recursive: true })
+			console.log(`Created dummy Multi-SWE-Bench directory at ${repoDir}`)
+		}
+	}
+
+	/**
+	 * List all available tasks in the Multi-SWE-Bench benchmark (dummy implementation)
+	 */
+	async listTasks(): Promise<Task[]> {
+		return [
+			{
+				id: "multi-swe-task-1",
+				name: "Multi-Language API Integration",
+				description:
+					"Implement a system that integrates a Python backend with a TypeScript frontend and a Rust processing service.",
+				workspacePath: path.join(EVALS_DIR, "repositories", "multi-swe"),
+				setupCommands: [],
+				verificationCommands: [],
+				metadata: {
+					languages: ["python", "typescript", "rust"],
+					complexity: "high",
+					type: "multi-swe",
+				},
+			},
+			{
+				id: "multi-swe-task-2",
+				name: "Cross-Platform Mobile App",
+				description: "Create a cross-platform mobile app using React Native with native modules in Swift and Kotlin.",
+				workspacePath: path.join(EVALS_DIR, "repositories", "multi-swe"),
+				setupCommands: [],
+				verificationCommands: [],
+				metadata: {
+					languages: ["javascript", "swift", "kotlin"],
+					complexity: "medium",
+					type: "multi-swe",
+				},
+			},
+			{
+				id: "multi-swe-task-3",
+				name: "Microservice Architecture",
+				description: "Design and implement a microservice architecture with services written in Go, Node.js, and Java.",
+				workspacePath: path.join(EVALS_DIR, "repositories", "multi-swe"),
+				setupCommands: [],
+				verificationCommands: [],
+				metadata: {
+					languages: ["go", "javascript", "java"],
+					complexity: "high",
+					type: "multi-swe",
+				},
+			},
+		]
+	}
+
+	/**
+	 * Prepare a specific task for execution (dummy implementation)
+	 * @param taskId The ID of the task to prepare
+	 */
+	async prepareTask(taskId: string): Promise<Task> {
+		const tasks = await this.listTasks()
+		const task = tasks.find((t) => t.id === taskId)
+
+		if (!task) {
+			throw new Error(`Task ${taskId} not found`)
+		}
+
+		// Create a dummy workspace for the task
+		const taskDir = path.join(task.workspacePath, taskId)
+		if (!fs.existsSync(taskDir)) {
+			fs.mkdirSync(taskDir, { recursive: true })
+
+			// Create a dummy file for the task
+			fs.writeFileSync(
+				path.join(taskDir, "README.md"),
+				`# ${task.name}\n\n${task.description}\n\nThis is a dummy task for testing purposes.`,
+			)
+
+			// Create additional dummy files based on task type
+			if (task.id === "multi-swe-task-1") {
+				// Python backend
+				fs.mkdirSync(path.join(taskDir, "backend"), { recursive: true })
+				fs.writeFileSync(
+					path.join(taskDir, "backend", "app.py"),
+					`# TODO: Implement Python backend\nfrom flask import Flask\n\napp = Flask(__name__)\n\n@app.route('/')\ndef hello():\n    return "Hello, World!"\n`,
+				)
+
+				// TypeScript frontend
+				fs.mkdirSync(path.join(taskDir, "frontend"), { recursive: true })
+				fs.writeFileSync(
+					path.join(taskDir, "frontend", "app.ts"),
+					`// TODO: Implement TypeScript frontend\nconsole.log('Frontend starting...');\n`,
+				)
+
+				// Rust processing service
+				fs.mkdirSync(path.join(taskDir, "processor"), { recursive: true })
+				fs.writeFileSync(
+					path.join(taskDir, "processor", "main.rs"),
+					`// TODO: Implement Rust processing service\nfn main() {\n    println!("Processor starting...");\n}\n`,
+				)
+			} else if (task.id === "multi-swe-task-2") {
+				// React Native app
+				fs.mkdirSync(path.join(taskDir, "app"), { recursive: true })
+				fs.writeFileSync(
+					path.join(taskDir, "app", "App.js"),
+					`// TODO: Implement React Native app\nimport React from 'react';\nimport { View, Text } from 'react-native';\n\nexport default function App() {\n  return (\n    <View>\n      <Text>Hello, World!</Text>\n    </View>\n  );\n}\n`,
+				)
+
+				// Swift native module
+				fs.mkdirSync(path.join(taskDir, "ios"), { recursive: true })
+				fs.writeFileSync(
+					path.join(taskDir, "ios", "NativeModule.swift"),
+					`// TODO: Implement Swift native module\nimport Foundation\n\n@objc(NativeModule)\nclass NativeModule: NSObject {\n  @objc\n  func hello() -> String {\n    return "Hello from Swift"\n  }\n}\n`,
+				)
+
+				// Kotlin native module
+				fs.mkdirSync(path.join(taskDir, "android"), { recursive: true })
+				fs.writeFileSync(
+					path.join(taskDir, "android", "NativeModule.kt"),
+					`// TODO: Implement Kotlin native module\npackage com.example.app\n\nclass NativeModule {\n  fun hello(): String {\n    return "Hello from Kotlin"\n  }\n}\n`,
+				)
+			} else if (task.id === "multi-swe-task-3") {
+				// Go service
+				fs.mkdirSync(path.join(taskDir, "service-go"), { recursive: true })
+				fs.writeFileSync(
+					path.join(taskDir, "service-go", "main.go"),
+					`// TODO: Implement Go service\npackage main\n\nimport "fmt"\n\nfunc main() {\n\tfmt.Println("Go service starting...")\n}\n`,
+				)
+
+				// Node.js service
+				fs.mkdirSync(path.join(taskDir, "service-node"), { recursive: true })
+				fs.writeFileSync(
+					path.join(taskDir, "service-node", "server.js"),
+					`// TODO: Implement Node.js service\nconsole.log('Node.js service starting...');\n`,
+				)
+
+				// Java service
+				fs.mkdirSync(path.join(taskDir, "service-java"), { recursive: true })
+				fs.writeFileSync(
+					path.join(taskDir, "service-java", "Main.java"),
+					`// TODO: Implement Java service\npublic class Main {\n    public static void main(String[] args) {\n        System.out.println("Java service starting...");\n    }\n}\n`,
+				)
+			}
+		}
+
+		// Update the task's workspace path to the task-specific directory
+		return {
+			...task,
+			workspacePath: taskDir,
+		}
+	}
+
+	/**
+	 * Verify the result of a task execution (dummy implementation)
+	 * @param task The task that was executed
+	 * @param result The result of the task execution
+	 */
+	async verifyResult(task: Task, result: any): Promise<VerificationResult> {
+		// Always return success for dummy implementation
+		return {
+			success: true,
+			metrics: {
+				testsPassed: 1,
+				testsFailed: 0,
+				testsTotal: 1,
+				functionalCorrectness: 1.0,
+				crossLanguageIntegration: 0.9, // Dummy metric specific to Multi-SWE
+				architectureQuality: 0.85, // Dummy metric specific to Multi-SWE
+			},
+		}
+	}
+}

evals/cli/src/adapters/swe-bench.ts

@@ -0,0 +1,125 @@
+import * as path from "path"
+import * as fs from "fs"
+import execa from "execa"
+import { BenchmarkAdapter, Task, VerificationResult } from "./types"
+
+const EVALS_DIR = path.resolve(__dirname, "../../../")
+
+/**
+ * Dummy adapter for the SWE-Bench benchmark
+ */
+export class SWEBenchAdapter implements BenchmarkAdapter {
+	name = "swe-bench"
+
+	/**
+	 * Set up the SWE-Bench benchmark repository (dummy implementation)
+	 */
+	async setup(): Promise<void> {
+		console.log("SWE-Bench dummy setup completed")
+
+		// Create repositories directory if it doesn't exist
+		const repoDir = path.join(EVALS_DIR, "repositories", "swe-bench")
+		if (!fs.existsSync(repoDir)) {
+			fs.mkdirSync(repoDir, { recursive: true })
+			console.log(`Created dummy SWE-Bench directory at ${repoDir}`)
+		}
+	}
+
+	/**
+	 * List all available tasks in the SWE-Bench benchmark (dummy implementation)
+	 */
+	async listTasks(): Promise<Task[]> {
+		return [
+			{
+				id: "swe-bench-task-1",
+				name: "Fix React Component Bug",
+				description: "Fix a bug in a React component where the state is not properly updated.",
+				workspacePath: path.join(EVALS_DIR, "repositories", "swe-bench"),
+				setupCommands: [],
+				verificationCommands: [],
+				metadata: {
+					repository: "facebook/react",
+					issue: "#12345",
+					type: "swe-bench",
+				},
+			},
+			{
+				id: "swe-bench-task-2",
+				name: "Optimize Database Query",
+				description: "Optimize a slow database query in a Django application.",
+				workspacePath: path.join(EVALS_DIR, "repositories", "swe-bench"),
+				setupCommands: [],
+				verificationCommands: [],
+				metadata: {
+					repository: "django/django",
+					issue: "#6789",
+					type: "swe-bench",
+				},
+			},
+			{
+				id: "swe-bench-task-3",
+				name: "Fix Memory Leak",
+				description: "Fix a memory leak in a Node.js application.",
+				workspacePath: path.join(EVALS_DIR, "repositories", "swe-bench"),
+				setupCommands: [],
+				verificationCommands: [],
+				metadata: {
+					repository: "nodejs/node",
+					issue: "#9876",
+					type: "swe-bench",
+				},
+			},
+		]
+	}
+
+	/**
+	 * Prepare a specific task for execution (dummy implementation)
+	 * @param taskId The ID of the task to prepare
+	 */
+	async prepareTask(taskId: string): Promise<Task> {
+		const tasks = await this.listTasks()
+		const task = tasks.find((t) => t.id === taskId)
+
+		if (!task) {
+			throw new Error(`Task ${taskId} not found`)
+		}
+
+		// Create a dummy workspace for the task
+		const taskDir = path.join(task.workspacePath, taskId)
+		if (!fs.existsSync(taskDir)) {
+			fs.mkdirSync(taskDir, { recursive: true })
+
+			// Create a dummy file for the task
+			fs.writeFileSync(
+				path.join(taskDir, "README.md"),
+				`# ${task.name}\n\n${task.description}\n\nThis is a dummy task for testing purposes.`,
+			)
+		}
+
+		// Update the task's workspace path to the task-specific directory
+		return {
+			...task,
+			workspacePath: taskDir,
+		}
+	}
+
+	/**
+	 * Verify the result of a task execution (dummy implementation)
+	 * @param task The task that was executed
+	 * @param result The result of the task execution
+	 */
+	async verifyResult(task: Task, result: any): Promise<VerificationResult> {
+		// Always return success for dummy implementation
+		return {
+			success: true,
+			metrics: {
+				testsPassed: 1,
+				testsFailed: 0,
+				testsTotal: 1,
+				functionalCorrectness: 1.0,
+				performanceImprovement: 0.25, // Dummy metric specific to SWE-Bench
+				codeQuality: 0.9, // Dummy metric specific to SWE-Bench
+			},
+		}
+	}
+}

evals/cli/src/adapters/swelancer.ts

@@ -0,0 +1,143 @@
+import * as path from "path"
+import * as fs from "fs"
+import execa from "execa"
+import { BenchmarkAdapter, Task, VerificationResult } from "./types"
+
+const EVALS_DIR = path.resolve(__dirname, "../../../")
+
+/**
+ * Dummy adapter for the SWELancer benchmark
+ */
+export class SWELancerAdapter implements BenchmarkAdapter {
+	name = "swelancer"
+
+	/**
+	 * Set up the SWELancer benchmark repository (dummy implementation)
+	 */
+	async setup(): Promise<void> {
+		console.log("SWELancer dummy setup completed")
+
+		// Create repositories directory if it doesn't exist
+		const repoDir = path.join(EVALS_DIR, "repositories", "swelancer")
+		if (!fs.existsSync(repoDir)) {
+			fs.mkdirSync(repoDir, { recursive: true })
+			console.log(`Created dummy SWELancer directory at ${repoDir}`)
+		}
+	}
+
+	/**
+	 * List all available tasks in the SWELancer benchmark (dummy implementation)
+	 */
+	async listTasks(): Promise<Task[]> {
+		return [
+			{
+				id: "swelancer-task-1",
+				name: "Create Landing Page",
+				description: "Create a responsive landing page for a new product using HTML, CSS, and JavaScript.",
+				workspacePath: path.join(EVALS_DIR, "repositories", "swelancer"),
+				setupCommands: [],
+				verificationCommands: [],
+				metadata: {
+					client: "TechStartup Inc.",
+					difficulty: "medium",
+					type: "swelancer",
+				},
+			},
+			{
+				id: "swelancer-task-2",
+				name: "Build REST API",
+				description: "Create a RESTful API for a blog application using Node.js and Express.",
+				workspacePath: path.join(EVALS_DIR, "repositories", "swelancer"),
+				setupCommands: [],
+				verificationCommands: [],
+				metadata: {
+					client: "BlogCo",
+					difficulty: "hard",
+					type: "swelancer",
+				},
+			},
+			{
+				id: "swelancer-task-3",
+				name: "Fix CSS Layout Issues",
+				description: "Fix layout issues in a responsive website across different screen sizes.",
+				workspacePath: path.join(EVALS_DIR, "repositories", "swelancer"),
+				setupCommands: [],
+				verificationCommands: [],
+				metadata: {
+					client: "DesignAgency",
+					difficulty: "easy",
+					type: "swelancer",
+				},
+			},
+		]
+	}
+
+	/**
+	 * Prepare a specific task for execution (dummy implementation)
+	 * @param taskId The ID of the task to prepare
+	 */
+	async prepareTask(taskId: string): Promise<Task> {
+		const tasks = await this.listTasks()
+		const task = tasks.find((t) => t.id === taskId)
+
+		if (!task) {
+			throw new Error(`Task ${taskId} not found`)
+		}
+
+		// Create a dummy workspace for the task
+		const taskDir = path.join(task.workspacePath, taskId)
+		if (!fs.existsSync(taskDir)) {
+			fs.mkdirSync(taskDir, { recursive: true })
+
+			// Create a dummy file for the task
+			fs.writeFileSync(
+				path.join(taskDir, "README.md"),
+				`# ${task.name}\n\n${task.description}\n\nThis is a dummy task for testing purposes.`,
+			)
+
+			// Create additional dummy files based on task type
+			if (task.id === "swelancer-task-1") {
+				fs.writeFileSync(
+					path.join(taskDir, "index.html"),
+					`<!DOCTYPE html>\n<html>\n<head>\n  <title>Landing Page</title>\n</head>\n<body>\n  <!-- TODO: Implement landing page -->\n</body>\n</html>`,
+				)
+			} else if (task.id === "swelancer-task-2") {
+				fs.writeFileSync(
+					path.join(taskDir, "server.js"),
+					`// TODO: Implement REST API\nconsole.log('Server starting...');`,
+				)
+			} else if (task.id === "swelancer-task-3") {
+				fs.writeFileSync(
+					path.join(taskDir, "styles.css"),
+					`/* TODO: Fix layout issues */\nbody {\n  margin: 0;\n  padding: 0;\n}`,
+				)
+			}
+		}
+
+		// Update the task's workspace path to the task-specific directory
+		return {
+			...task,
+			workspacePath: taskDir,
+		}
+	}
+
+	/**
+	 * Verify the result of a task execution (dummy implementation)
+	 * @param task The task that was executed
+	 * @param result The result of the task execution
+	 */
+	async verifyResult(task: Task, result: any): Promise<VerificationResult> {
+		// Always return success for dummy implementation
+		return {
+			success: true,
+			metrics: {
+				testsPassed: 1,
+				testsFailed: 0,
+				testsTotal: 1,
+				functionalCorrectness: 1.0,
+				clientSatisfaction: 0.95, // Dummy metric specific to SWELancer
+				timeEfficiency: 0.85, // Dummy metric specific to SWELancer
+			},
+		}
+	}
+}

evals/cli/src/adapters/types.ts

@@ -0,0 +1,31 @@
+/**
+ * Represents a task to be executed
+ */
+export interface Task {
+	id: string
+	name: string
+	description: string
+	workspacePath: string
+	setupCommands: string[]
+	verificationCommands: string[]
+	metadata: Record<string, any>
+}
+
+/**
+ * Result of verifying a task execution
+ */
+export interface VerificationResult {
+	success: boolean
+	metrics: Record<string, any>
+}
+
+/**
+ * Interface for benchmark adapters
+ */
+export interface BenchmarkAdapter {
+	name: string
+	setup(): Promise<void>
+	listTasks(): Promise<Task[]>
+	prepareTask(taskId: string): Promise<Task>
+	verifyResult(task: Task, result: any): Promise<VerificationResult>
+}

evals/cli/src/commands/report.ts

@@ -0,0 +1,237 @@
+import * as fs from "fs"
+import * as path from "path"
+import chalk from "chalk"
+import ora from "ora"
+import { ResultsDatabase } from "../db"
+import { generateMarkdownReport } from "../utils/markdown"
+
+interface ReportOptions {
+	format?: "json" | "markdown"
+	output?: string
+}
+
+/**
+ * Handler for the report command
+ * @param options Command options
+ */
+export async function reportHandler(options: ReportOptions): Promise<void> {
+	const format = options.format || "markdown"
+	const db = new ResultsDatabase()
+
+	try {
+		const spinner = ora("Generating report...").start()
+
+		// Get all runs
+		const runs = db.getRuns()
+
+		console.log(chalk.blue(`Found ${runs.length} evaluation runs`))
+
+		if (runs.length === 0) {
+			spinner.fail("No evaluation runs found")
+			return
+		}
+
+		// Generate summary report
+		const summary = {
+			runs: runs.length,
+			models: [...new Set(runs.map((run) => run.model))],
+			benchmarks: [...new Set(runs.map((run) => run.benchmark))],
+			tasks: 0,
+			successRate: 0,
+			averageTokens: 0,
+			averageCost: 0,
+			averageDuration: 0,
+			totalToolCalls: 0,
+			totalToolFailures: 0,
+			toolSuccessRate: 0,
+			toolUsage: {} as Record<string, { calls: number; failures: number }>,
+		}
+
+		let totalTasks = 0
+		let successfulTasks = 0
+		let totalTokens = 0
+		let totalCost = 0
+		let totalDuration = 0
+		let totalToolCalls = 0
+		let totalToolFailures = 0
+
+		for (const run of runs) {
+			const tasks = db.getRunTasks(run.id)
+			totalTasks += tasks.length
+
+			for (const task of tasks) {
+				if (task.success) {
+					successfulTasks++
+				}
+
+				const metrics = db.getTaskMetrics(task.id)
+
+				const tokensIn = metrics.find((m) => m.name === "tokensIn")?.value || 0
+				const tokensOut = metrics.find((m) => m.name === "tokensOut")?.value || 0
+				totalTokens += tokensIn + tokensOut
+
+				totalCost += metrics.find((m) => m.name === "cost")?.value || 0
+				totalDuration += metrics.find((m) => m.name === "duration")?.value || 0
+
+				// Collect tool call metrics
+				totalToolCalls += task.total_tool_calls || 0
+				totalToolFailures += task.total_tool_failures || 0
+
+				// Get detailed tool usage
+				const toolCalls = db.getTaskToolCalls(task.id)
+
+				for (const toolCall of toolCalls) {
+					if (!summary.toolUsage[toolCall.tool_name]) {
+						summary.toolUsage[toolCall.tool_name] = {
+							calls: 0,
+							failures: 0,
+						}
+					}
+
+					summary.toolUsage[toolCall.tool_name].calls += toolCall.call_count
+					summary.toolUsage[toolCall.tool_name].failures += toolCall.failure_count
+				}
+			}
+		}
+
+		// Calculate tool success rate
+		summary.totalToolCalls = totalToolCalls
+		summary.totalToolFailures = totalToolFailures
+		summary.toolSuccessRate = totalToolCalls > 0 ? 1 - totalToolFailures / totalToolCalls : 1.0
+
+		summary.tasks = totalTasks
+		summary.successRate = totalTasks > 0 ? successfulTasks / totalTasks : 0
+		summary.averageTokens = totalTasks > 0 ? totalTokens / totalTasks : 0
+		summary.averageCost = totalTasks > 0 ? totalCost / totalTasks : 0
+		summary.averageDuration = totalTasks > 0 ? totalDuration / totalTasks : 0
+
+		// Generate benchmark-specific reports
+		const benchmarkReports: Record<string, any> = {}
+
+		for (const benchmark of summary.benchmarks) {
+			const benchmarkRuns = runs.filter((run) => run.benchmark === benchmark)
+			const benchmarkSummary = {
+				runs: benchmarkRuns.length,
+				models: [...new Set(benchmarkRuns.map((run) => run.model))],
+				tasks: 0,
+				successRate: 0,
+				averageTokens: 0,
+				averageCost: 0,
+				averageDuration: 0,
+			}
+
+			let benchmarkTasks = 0
+			let benchmarkSuccessfulTasks = 0
+			let benchmarkTotalTokens = 0
+			let benchmarkTotalCost = 0
+			let benchmarkTotalDuration = 0
+
+			for (const run of benchmarkRuns) {
+				const tasks = db.getRunTasks(run.id)
+				benchmarkTasks += tasks.length
+
+				for (const task of tasks) {
+					if (task.success) {
+						benchmarkSuccessfulTasks++
+					}
+
+					const metrics = db.getTaskMetrics(task.id)
+
+					const tokensIn = metrics.find((m) => m.name === "tokensIn")?.value || 0
+					const tokensOut = metrics.find((m) => m.name === "tokensOut")?.value || 0
+					benchmarkTotalTokens += tokensIn + tokensOut
+
+					benchmarkTotalCost += metrics.find((m) => m.name === "cost")?.value || 0
+					benchmarkTotalDuration += metrics.find((m) => m.name === "duration")?.value || 0
+				}
+			}
+
+			benchmarkSummary.tasks = benchmarkTasks
+			benchmarkSummary.successRate = benchmarkTasks > 0 ? benchmarkSuccessfulTasks / benchmarkTasks : 0
+			benchmarkSummary.averageTokens = benchmarkTasks > 0 ? benchmarkTotalTokens / benchmarkTasks : 0
+			benchmarkSummary.averageCost = benchmarkTasks > 0 ? benchmarkTotalCost / benchmarkTasks : 0
+			benchmarkSummary.averageDuration = benchmarkTasks > 0 ? benchmarkTotalDuration / benchmarkTasks : 0
+
+			benchmarkReports[benchmark] = benchmarkSummary
+		}
+
+		// Generate model-specific reports
+		const modelReports: Record<string, any> = {}
+
+		for (const model of summary.models) {
+			const modelRuns = runs.filter((run) => run.model === model)
+			const modelSummary = {
+				runs: modelRuns.length,
+				benchmarks: [...new Set(modelRuns.map((run) => run.benchmark))],
+				tasks: 0,
+				successRate: 0,
+				averageTokens: 0,
+				averageCost: 0,
+				averageDuration: 0,
+			}
+
+			let modelTasks = 0
+			let modelSuccessfulTasks = 0
+			let modelTotalTokens = 0
+			let modelTotalCost = 0
+			let modelTotalDuration = 0
+
+			for (const run of modelRuns) {
+				const tasks = db.getRunTasks(run.id)
+				modelTasks += tasks.length
+
+				for (const task of tasks) {
+					if (task.success) {
+						modelSuccessfulTasks++
+					}
+
+					const metrics = db.getTaskMetrics(task.id)
+
+					const tokensIn = metrics.find((m) => m.name === "tokensIn")?.value || 0
+					const tokensOut = metrics.find((m) => m.name === "tokensOut")?.value || 0
+					modelTotalTokens += tokensIn + tokensOut
+
+					modelTotalCost += metrics.find((m) => m.name === "cost")?.value || 0
+					modelTotalDuration += metrics.find((m) => m.name === "duration")?.value || 0
+				}
+			}
+
+			modelSummary.tasks = modelTasks
+			modelSummary.successRate = modelTasks > 0 ? modelSuccessfulTasks / modelTasks : 0
+			modelSummary.averageTokens = modelTasks > 0 ? modelTotalTokens / modelTasks : 0
+			modelSummary.averageCost = modelTasks > 0 ? modelTotalCost / modelTasks : 0
+			modelSummary.averageDuration = modelTasks > 0 ? modelTotalDuration / modelTasks : 0
+
+			modelReports[model] = modelSummary
+		}
+
+		// Save reports
+		const reportDir = path.join(path.resolve(__dirname, "../../../"), "results", "reports")
+		fs.mkdirSync(reportDir, { recursive: true })
+
+		const timestamp = new Date().toISOString().replace(/:/g, "-")
+
+		if (format === "json") {
+			// Save JSON reports
+			fs.writeFileSync(path.join(reportDir, `summary-${timestamp}.json`), JSON.stringify(summary, null, 2))
+
+			fs.writeFileSync(path.join(reportDir, `benchmarks-${timestamp}.json`), JSON.stringify(benchmarkReports, null, 2))
+
+			fs.writeFileSync(path.join(reportDir, `models-${timestamp}.json`), JSON.stringify(modelReports, null, 2))
+
+			spinner.succeed(`JSON reports generated in ${reportDir}`)
+		} else {
+			// Generate markdown report
+			const outputPath = options.output || path.join(reportDir, `report-${timestamp}.md`)
+
+			generateMarkdownReport(summary, benchmarkReports, modelReports, outputPath)
+
+			spinner.succeed(`Markdown report generated at ${outputPath}`)
+		}
+	} catch (error: any) {
+		console.error(chalk.red(`Error generating report: ${error.message}`))
+		console.error(error.stack)
+	} finally {
+		db.close()
+	}
+}

evals/cli/src/commands/run.ts

@@ -0,0 +1,133 @@
+import * as path from "path"
+import { v4 as uuidv4 } from "uuid"
+import chalk from "chalk"
+import ora from "ora"
+import { getAdapter } from "../adapters"
+import { ResultsDatabase } from "../db"
+import { spawnVSCode, cleanupVSCode } from "../utils/vscode"
+import { sendTaskToServer } from "../utils/task"
+import { storeTaskResult } from "../utils/results"
+
+interface RunOptions {
+	benchmark?: string
+	model: string
+	count?: number
+	apiKey?: string
+}
+
+/**
+ * Handler for the run command
+ * @param options Command options
+ */
+export async function runHandler(options: RunOptions): Promise<void> {
+	// Determine which benchmarks to run
+	const benchmarks = options.benchmark ? [options.benchmark] : ["exercism"] // Default to exercism for now
+	const model = options.model
+	const count = options.count || Infinity
+
+	console.log(chalk.blue(`Running evaluations for model: ${model}`))
+	console.log(chalk.blue(`Benchmarks: ${benchmarks.join(", ")}`))
+
+	// Create a run for each benchmark
+	for (const benchmark of benchmarks) {
+		const runId = uuidv4()
+		const db = new ResultsDatabase()
+
+		console.log(chalk.green(`\nStarting run for benchmark: ${benchmark}`))
+
+		// Create run in database
+		db.createRun(runId, model, benchmark)
+
+		// Get adapter for this benchmark
+		try {
+			const adapter = getAdapter(benchmark)
+
+			// List tasks
+			const spinner = ora("Listing tasks...").start()
+			const tasks = await adapter.listTasks()
+			spinner.succeed(`Found ${tasks.length} tasks for ${benchmark}`)
+
+			// Limit number of tasks if specified
+			const tasksToRun = tasks.slice(0, count)
+
+			console.log(chalk.blue(`Running ${tasksToRun.length} tasks...`))
+
+			// Run each task
+			for (let i = 0; i < tasksToRun.length; i++) {
+				const task = tasksToRun[i]
+
+				console.log(chalk.cyan(`\nTask ${i + 1}/${tasksToRun.length}: ${task.name}`))
+
+				// Prepare task
+				const prepareSpinner = ora("Preparing task...").start()
+				const preparedTask = await adapter.prepareTask(task.id)
+				prepareSpinner.succeed("Task prepared")
+
+				// Spawn VSCode
+				console.log("Spawning VSCode...")
+				await spawnVSCode(preparedTask.workspacePath)
+
+				// Send task to server
+				const sendSpinner = ora("Sending task to server...").start()
+				try {
+					const result = await sendTaskToServer(preparedTask.description, options.apiKey)
+					sendSpinner.succeed("Task completed")
+
+					// Verify result
+					const verifySpinner = ora("Verifying result...").start()
+					const verification = await adapter.verifyResult(preparedTask, result)
+
+					if (verification.success) {
+						verifySpinner.succeed(
+							`Verification successful: ${verification.metrics.testsPassed}/${verification.metrics.testsTotal} tests passed`,
+						)
+					} else {
+						verifySpinner.fail(
+							`Verification failed: ${verification.metrics.testsPassed}/${verification.metrics.testsTotal} tests passed`,
+						)
+					}
+
+					// Store result
+					const storeSpinner = ora("Storing result...").start()
+					await storeTaskResult(runId, preparedTask, result, verification)
+					storeSpinner.succeed("Result stored")
+
+					console.log(chalk.green(`Task completed. Success: ${verification.success}`))
+
+					// Clean up VS Code and temporary files
+					const cleanupSpinner = ora("Cleaning up...").start()
+					try {
+						await cleanupVSCode(preparedTask.workspacePath)
+						cleanupSpinner.succeed("Cleanup completed")
+					} catch (cleanupError: any) {
+						cleanupSpinner.fail(`Cleanup failed: ${cleanupError.message}`)
+						console.error(chalk.yellow(cleanupError.stack))
+					}
+				} catch (error: any) {
+					sendSpinner.fail(`Task failed: ${error.message}`)
+					console.error(chalk.red(error.stack))
+
+					// Clean up VS Code and temporary files even if the task failed
+					const cleanupSpinner = ora("Cleaning up...").start()
+					try {
+						await cleanupVSCode(preparedTask.workspacePath)
+						cleanupSpinner.succeed("Cleanup completed")
+					} catch (cleanupError: any) {
+						cleanupSpinner.fail(`Cleanup failed: ${cleanupError.message}`)
+						console.error(chalk.yellow(cleanupError.stack))
+					}
+				}
+			}
+
+			// Mark run as complete
+			db.completeRun(runId)
+
+			console.log(chalk.green(`\nRun complete for benchmark: ${benchmark}`))
+		} catch (error: any) {
+			console.error(chalk.red(`Error running benchmark ${benchmark}: ${error.message}`))
+			console.error(error.stack)
+		}
+	}
+
+	console.log(chalk.green("\nAll evaluations complete"))
+}

evals/cli/src/commands/setup.ts

@@ -0,0 +1,72 @@
+import * as path from "path"
+import * as fs from "fs"
+import execa from "execa"
+import chalk from "chalk"
+import ora from "ora"
+import { getAllAdapters } from "../adapters/index"
+import { BenchmarkAdapter } from "../adapters/types"
+
+interface SetupOptions {
+	benchmarks: string
+}
+
+/**
+ * Handler for the setup command
+ * @param options Command options
+ */
+export async function setupHandler(options: SetupOptions): Promise<void> {
+	const benchmarks = options.benchmarks.split(",")
+
+	console.log(chalk.blue(`Setting up benchmarks: ${benchmarks.join(", ")}`))
+
+	// Create directories
+	const evalsDir = path.resolve(__dirname, "../../../")
+	const reposDir = path.join(evalsDir, "repositories")
+	const resultsDir = path.join(evalsDir, "results")
+
+	const spinner = ora("Creating directory structure").start()
+
+	try {
+		fs.mkdirSync(reposDir, { recursive: true })
+		fs.mkdirSync(resultsDir, { recursive: true })
+		fs.mkdirSync(path.join(resultsDir, "runs"), { recursive: true })
+		fs.mkdirSync(path.join(resultsDir, "reports"), { recursive: true })
+		spinner.succeed("Directory structure created")
+	} catch (error) {
+		spinner.fail(`Failed to create directory structure: ${(error as Error).message}`)
+		throw error
+	}
+
+	// Set up each benchmark
+	try {
+		const adapters = getAllAdapters().filter((adapter: BenchmarkAdapter) => benchmarks.includes(adapter.name))
+
+		if (adapters.length === 0) {
+			console.warn(chalk.yellow("No valid benchmarks specified. Available benchmarks:"))
+			console.warn(
+				chalk.yellow(
+					getAllAdapters()
+						.map((a: BenchmarkAdapter) => a.name)
+						.join(", "),
+				),
+			)
+			return
+		}
+
+		for (const adapter of adapters) {
+			const setupSpinner = ora(`Setting up ${adapter.name}...`).start()
+			try {
+				await adapter.setup()
+				setupSpinner.succeed(`${adapter.name} setup complete`)
+			} catch (error) {
+				setupSpinner.fail(`Failed to set up ${adapter.name}: ${(error as Error).message}`)
+				throw error
+			}
+		}
+
+		console.log(chalk.green("Setup complete"))
+	} catch (error) {
+		console.error(chalk.red(`Setup failed: ${(error as Error).message}`))
+		throw error
+	}
+}

evals/cli/src/db/index.ts

@@ -0,0 +1,211 @@
+import * as path from "path"
+import * as fs from "fs"
+import Database from "better-sqlite3"
+import { SCHEMA } from "./schema"
+
+const EVALS_DIR = path.resolve(__dirname, "../../../")
+
+/**
+ * Database class for storing evaluation results
+ */
+export class ResultsDatabase {
+	db: Database.Database
+
+	constructor() {
+		// Ensure results directory exists
+		const resultsDir = path.join(EVALS_DIR, "results")
+		fs.mkdirSync(resultsDir, { recursive: true })
+
+		// Create database file
+		const dbPath = path.join(resultsDir, "evals.db")
+		this.db = new Database(dbPath)
+
+		// Initialize schema
+		this.initSchema()
+	}
+
+	/**
+	 * Initialize the database schema
+	 */
+	private initSchema(): void {
+		this.db.exec(SCHEMA)
+	}
+
+	/**
+	 * Create a new evaluation run
+	 * @param id Run ID
+	 * @param model Model name
+	 * @param benchmark Benchmark name
+	 */
+	createRun(id: string, model: string, benchmark: string): void {
+		const stmt = this.db.prepare(`
+      INSERT INTO runs (id, timestamp, model, benchmark)
+      VALUES (?, ?, ?, ?)
+    `)
+
+		stmt.run(id, Date.now(), model, benchmark)
+	}
+
+	/**
+	 * Mark a run as completed
+	 * @param id Run ID
+	 */
+	completeRun(id: string): void {
+		const stmt = this.db.prepare(`
+      UPDATE runs SET completed = 1 WHERE id = ?
+    `)
+
+		stmt.run(id)
+	}
+
+	/**
+	 * Create a new task
+	 * @param id Task ID
+	 * @param runId Run ID
+	 * @param taskId Original task ID
+	 */
+	createTask(id: string, runId: string, taskId: string): void {
+		const stmt = this.db.prepare(`
+      INSERT INTO tasks (id, run_id, task_id, timestamp)
+      VALUES (?, ?, ?, ?)
+    `)
+
+		stmt.run(id, runId, taskId, Date.now())
+	}
+
+	/**
+	 * Mark a task as completed
+	 * @param id Task ID
+	 * @param success Whether the task was successful
+	 * @param toolCalls Total tool calls
+	 * @param toolFailures Total tool failures
+	 */
+	completeTask(id: string, success: boolean, toolCalls: number = 0, toolFailures: number = 0): void {
+		const stmt = this.db.prepare(`
+      UPDATE tasks 
+      SET success = ?, total_tool_calls = ?, total_tool_failures = ? 
+      WHERE id = ?
+    `)
+
+		stmt.run(success ? 1 : 0, toolCalls, toolFailures, id)
+	}
+
+	/**
+	 * Add a metric to a task
+	 * @param taskId Task ID
+	 * @param name Metric name
+	 * @param value Metric value
+	 */
+	addMetric(taskId: string, name: string, value: number): void {
+		const stmt = this.db.prepare(`
+      INSERT INTO metrics (task_id, name, value)
+      VALUES (?, ?, ?)
+    `)
+
+		stmt.run(taskId, name, value)
+	}
+
+	/**
+	 * Add a tool call record
+	 * @param taskId Task ID
+	 * @param toolName Tool name
+	 * @param callCount Number of calls
+	 * @param failureCount Number of failures
+	 */
+	addToolCall(taskId: string, toolName: string, callCount: number, failureCount: number): void {
+		const stmt = this.db.prepare(`
+      INSERT INTO tool_calls (task_id, tool_name, call_count, failure_count)
+      VALUES (?, ?, ?, ?)
+    `)
+
+		stmt.run(taskId, toolName, callCount, failureCount)
+	}
+
+	/**
+	 * Add a file record
+	 * @param taskId Task ID
+	 * @param filePath File path
+	 * @param status File status (created, modified, deleted)
+	 */
+	addFile(taskId: string, filePath: string, status: "created" | "modified" | "deleted"): void {
+		const stmt = this.db.prepare(`
+      INSERT INTO files (task_id, path, status)
+      VALUES (?, ?, ?)
+    `)
+
+		stmt.run(taskId, filePath, status)
+	}
+
+	/**
+	 * Get all runs
+	 * @returns Array of runs
+	 */
+	getRuns(): any[] {
+		const stmt = this.db.prepare(`
+      SELECT * FROM runs ORDER BY timestamp DESC
+    `)
+
+		return stmt.all()
+	}
+
+	/**
+	 * Get all tasks for a run
+	 * @param runId Run ID
+	 * @returns Array of tasks
+	 */
+	getRunTasks(runId: string): any[] {
+		const stmt = this.db.prepare(`
+      SELECT * FROM tasks WHERE run_id = ? ORDER BY timestamp ASC
+    `)
+
+		return stmt.all(runId)
+	}
+
+	/**
+	 * Get all metrics for a task
+	 * @param taskId Task ID
+	 * @returns Array of metrics
+	 */
+	getTaskMetrics(taskId: string): any[] {
+		const stmt = this.db.prepare(`
+      SELECT name, value FROM metrics WHERE task_id = ?
+    `)
+
+		return stmt.all(taskId)
+	}
+
+	/**
+	 * Get all tool calls for a task
+	 * @param taskId Task ID
+	 * @returns Array of tool calls
+	 */
+	getTaskToolCalls(taskId: string): any[] {
+		const stmt = this.db.prepare(`
+      SELECT tool_name, call_count, failure_count 
+      FROM tool_calls 
+      WHERE task_id = ?
+    `)
+
+		return stmt.all(taskId)
+	}
+
+	/**
+	 * Get all files for a task
+	 * @param taskId Task ID
+	 * @returns Array of files
+	 */
+	getTaskFiles(taskId: string): any[] {
+		const stmt = this.db.prepare(`
+      SELECT path, status FROM files WHERE task_id = ?
+    `)
+
+		return stmt.all(taskId)
+	}
+
+	/**
+	 * Close the database connection
+	 */
+	close(): void {
+		this.db.close()
+	}
+}

evals/cli/src/db/schema.ts

@@ -0,0 +1,48 @@
+/**
+ * SQL schema for the evaluation database
+ */
+export const SCHEMA = `
+CREATE TABLE IF NOT EXISTS runs (
+  id TEXT PRIMARY KEY,
+  timestamp INTEGER NOT NULL,
+  model TEXT NOT NULL,
+  benchmark TEXT NOT NULL,
+  completed INTEGER NOT NULL DEFAULT 0
+);
+
+CREATE TABLE IF NOT EXISTS tasks (
+  id TEXT PRIMARY KEY,
+  run_id TEXT NOT NULL,
+  task_id TEXT NOT NULL,
+  timestamp INTEGER NOT NULL,
+  success INTEGER NOT NULL DEFAULT 0,
+  total_tool_calls INTEGER DEFAULT 0,
+  total_tool_failures INTEGER DEFAULT 0,
+  FOREIGN KEY (run_id) REFERENCES runs(id)
+);
+
+CREATE TABLE IF NOT EXISTS metrics (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  task_id TEXT NOT NULL,
+  name TEXT NOT NULL,
+  value REAL NOT NULL,
+  FOREIGN KEY (task_id) REFERENCES tasks(id)
+);
+
+CREATE TABLE IF NOT EXISTS tool_calls (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  task_id TEXT NOT NULL,
+  tool_name TEXT NOT NULL,
+  call_count INTEGER NOT NULL,
+  failure_count INTEGER NOT NULL,
+  FOREIGN KEY (task_id) REFERENCES tasks(id)
+);
+
+CREATE TABLE IF NOT EXISTS files (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  task_id TEXT NOT NULL,
+  path TEXT NOT NULL,
+  status TEXT NOT NULL,
+  FOREIGN KEY (task_id) REFERENCES tasks(id)
+);
+`

evals/cli/src/index.ts

@@ -0,0 +1,70 @@
+#!/usr/bin/env node
+import { Command } from "commander"
+import chalk from "chalk"
+import { setupHandler } from "./commands/setup"
+import { runHandler } from "./commands/run"
+import { reportHandler } from "./commands/report"
+
+// Create the CLI program
+const program = new Command()
+
+// Set up CLI metadata
+program.name("cline-eval").description("CLI tool for orchestrating Cline evaluations across multiple benchmarks").version("0.1.0")
+
+// Setup command
+program
+	.command("setup")
+	.description("Clone and set up benchmark repositories")
+	.option(
+		"-b, --benchmarks <benchmarks>",
+		"Comma-separated list of benchmarks to set up",
+		"exercism,swe-bench,swelancer,multi-swe",
+	)
+	.action(async (options) => {
+		try {
+			await setupHandler(options)
+		} catch (error) {
+			console.error(chalk.red(`Error during setup: ${error instanceof Error ? error.message : String(error)}`))
+			process.exit(1)
+		}
+	})
+
+// Run command
+program
+	.command("run")
+	.description("Run evaluations")
+	.option("-b, --benchmark <benchmark>", "Specific benchmark to run")
+	.option("-m, --model <model>", "Model to evaluate", "claude-3-opus-20240229")
+	.option("-c, --count <count>", "Number of tasks to run", parseInt)
+	.option("-k, --api-key <apiKey>", "Cline API key to use for evaluations")
+	.action(async (options) => {
+		try {
+			await runHandler(options)
+		} catch (error) {
+			console.error(chalk.red(`Error during run: ${error instanceof Error ? error.message : String(error)}`))
+			process.exit(1)
+		}
+	})
+
+// Report command
+program
+	.command("report")
+	.description("Generate reports")
+	.option("-f, --format <format>", "Report format (json, markdown)", "markdown")
+	.option("-o, --output <path>", "Output path for the report")
+	.action(async (options) => {
+		try {
+			await reportHandler(options)
+		} catch (error) {
+			console.error(chalk.red(`Error generating report: ${error instanceof Error ? error.message : String(error)}`))
+			process.exit(1)
+		}
+	})
+
+// Parse command line arguments
+program.parse(process.argv)
+
+// If no arguments provided, show help
+if (process.argv.length === 2) {
+	program.help()
+}

evals/cli/src/utils/extensions.ts

@@ -0,0 +1,131 @@
+import execa from "execa"
+import * as fs from "fs"
+import * as path from "path"
+import * as os from "os"
+
+/**
+ * List of VSCode extensions to install for evaluation environments
+ * These extensions provide language support and other useful features
+ */
+export const REQUIRED_EXTENSIONS = [
+	"golang.go", // Go language support
+	"dbaeumer.vscode-eslint", // ESLint support
+	"redhat.java", // Java support
+	"ms-python.python", // Python support
+	"rust-lang.rust-analyzer", // Rust support
+	"ms-vscode.cpptools", // C/C++ support
+]
+
+/**
+ * Install required VSCode extensions in the specified extensions directory
+ * @param extensionsDir The directory where extensions should be installed
+ * @returns Promise that resolves when all extensions are installed
+ */
+export async function installRequiredExtensions(extensionsDir: string): Promise<void> {
+	console.log("Installing required VSCode extensions...")
+
+	// Create the extensions directory if it doesn't exist
+	if (!fs.existsSync(extensionsDir)) {
+		fs.mkdirSync(extensionsDir, { recursive: true })
+	}
+
+	// Install each extension
+	for (const extension of REQUIRED_EXTENSIONS) {
+		try {
+			console.log(`Installing extension: ${extension}...`)
+			await execa("code", ["--extensions-dir", extensionsDir, "--install-extension", extension, "--force"])
+			console.log(`✅ Extension ${extension} installed successfully`)
+		} catch (error: any) {
+			console.warn(`⚠️ Failed to install extension ${extension}: ${error.message}`)
+			// Continue with other extensions even if one fails
+		}
+	}
+
+	console.log("✅ All required extensions installed")
+}
+
+/**
+ * Check if a VSCode extension is installed in the specified directory
+ * @param extensionsDir The directory to check for installed extensions
+ * @param extensionId The ID of the extension to check
+ * @returns True if the extension is installed, false otherwise
+ */
+export function isExtensionInstalled(extensionsDir: string, extensionId: string): boolean {
+	// Extensions are installed in directories named publisher.name-version
+	// We need to check if any directory starts with the extensionId
+	const extensionPrefix = extensionId.toLowerCase() + "-"
+
+	try {
+		const files = fs.readdirSync(extensionsDir)
+		return files.some((file) => {
+			const lowerCaseFile = file.toLowerCase()
+			return lowerCaseFile === extensionId.toLowerCase() || lowerCaseFile.startsWith(extensionPrefix)
+		})
+	} catch (error) {
+		return false
+	}
+}
+
+/**
+ * Get the path to the VSCode settings file in the specified user data directory
+ * @param userDataDir The VSCode user data directory
+ * @returns The path to the settings.json file
+ */
+export function getSettingsPath(userDataDir: string): string {
+	const settingsDir = path.join(userDataDir, "User")
+	fs.mkdirSync(settingsDir, { recursive: true })
+	return path.join(settingsDir, "settings.json")
+}
+
+/**
+ * Configure extension settings in the VSCode user data directory
+ * @param userDataDir The VSCode user data directory
+ */
+export function configureExtensionSettings(userDataDir: string): void {
+	const settingsPath = getSettingsPath(userDataDir)
+
+	// Read existing settings if they exist
+	let settings = {}
+	if (fs.existsSync(settingsPath)) {
+		try {
+			settings = JSON.parse(fs.readFileSync(settingsPath, "utf8"))
+		} catch (error) {
+			console.warn(`Error reading settings file: ${error}`)
+		}
+	}
+
+	// Add or update extension-specific settings
+	const updatedSettings = {
+		...settings,
+		// Go extension settings
+		"go.toolsManagement.autoUpdate": false,
+		"go.survey.prompt": false,
+
+		// ESLint settings
+		"eslint.enable": true,
+		"eslint.run": "onSave",
+
+		// Java settings
+		"java.configuration.checkProjectSettingsExclusions": false,
+		"java.configure.checkForOutdatedExtensions": false,
+		"java.help.firstView": false,
+
+		// Python settings
+		"python.experiments.enabled": false,
+		"python.showStartPage": false,
+
+		// Rust settings
+		"rust-analyzer.checkOnSave.command": "check",
+
+		// C/C++ settings
+		"C_Cpp.intelliSenseEngine": "default",
+
+		// General extension settings
+		"extensions.autoUpdate": false,
+		"extensions.ignoreRecommendations": true,
+	}
+
+	// Write updated settings
+	fs.writeFileSync(settingsPath, JSON.stringify(updatedSettings, null, 2))
+	console.log("✅ Extension settings configured")
+}

evals/cli/src/utils/markdown.ts

@@ -0,0 +1,109 @@
+import * as fs from "fs"
+import * as path from "path"
+
+/**
+ * Generate a markdown report from evaluation results
+ * @param summary Overall summary
+ * @param benchmarkReports Benchmark-specific reports
+ * @param modelReports Model-specific reports
+ * @param outputPath Output file path
+ */
+export function generateMarkdownReport(
+	summary: any,
+	benchmarkReports: Record<string, any>,
+	modelReports: Record<string, any>,
+	outputPath: string,
+): void {
+	let markdown = `# Cline Evaluation Report\n\n`
+
+	// Generate summary section
+	markdown += `## Summary\n\n`
+	markdown += `- **Total Runs:** ${summary.runs}\n`
+	markdown += `- **Models:** ${summary.models.join(", ")}\n`
+	markdown += `- **Benchmarks:** ${summary.benchmarks.join(", ")}\n`
+	markdown += `- **Total Tasks:** ${summary.tasks}\n`
+	markdown += `- **Success Rate:** ${(summary.successRate * 100).toFixed(2)}%\n`
+	markdown += `- **Average Tokens:** ${Math.round(summary.averageTokens)}\n`
+	markdown += `- **Average Cost:** $${summary.averageCost.toFixed(4)}\n`
+	markdown += `- **Average Duration:** ${(summary.averageDuration / 1000).toFixed(2)}s\n`
+	markdown += `- **Total Tool Calls:** ${summary.totalToolCalls}\n`
+	markdown += `- **Tool Success Rate:** ${(summary.toolSuccessRate * 100).toFixed(2)}%\n\n`
+
+	// Generate tool usage section
+	markdown += `## Tool Usage\n\n`
+	markdown += `| Tool | Calls | Failures | Success Rate |\n`
+	markdown += `| ---- | ----- | -------- | ------------ |\n`
+
+	for (const [toolName, metrics] of Object.entries(summary.toolUsage)) {
+		const calls = (metrics as any).calls
+		const failures = (metrics as any).failures
+		const successRate = calls > 0 ? (1 - failures / calls) * 100 : 100
+
+		markdown += `| ${toolName} | ${calls} | ${failures} | ${successRate.toFixed(2)}% |\n`
+	}
+
+	// Generate benchmark results section
+	markdown += `\n## Benchmark Results\n\n`
+
+	for (const [benchmark, report] of Object.entries(benchmarkReports)) {
+		markdown += `### ${benchmark}\n\n`
+		markdown += `- **Runs:** ${report.runs}\n`
+		markdown += `- **Models:** ${report.models.join(", ")}\n`
+		markdown += `- **Tasks:** ${report.tasks}\n`
+		markdown += `- **Success Rate:** ${(report.successRate * 100).toFixed(2)}%\n`
+		markdown += `- **Average Tokens:** ${Math.round(report.averageTokens)}\n`
+		markdown += `- **Average Cost:** $${report.averageCost.toFixed(4)}\n`
+		markdown += `- **Average Duration:** ${(report.averageDuration / 1000).toFixed(2)}s\n\n`
+	}
+
+	// Generate model results section
+	markdown += `## Model Results\n\n`
+
+	for (const [model, report] of Object.entries(modelReports)) {
+		markdown += `### ${model}\n\n`
+		markdown += `- **Runs:** ${report.runs}\n`
+		markdown += `- **Benchmarks:** ${report.benchmarks.join(", ")}\n`
+		markdown += `- **Tasks:** ${report.tasks}\n`
+		markdown += `- **Success Rate:** ${(report.successRate * 100).toFixed(2)}%\n`
+		markdown += `- **Average Tokens:** ${Math.round(report.averageTokens)}\n`
+		markdown += `- **Average Cost:** $${report.averageCost.toFixed(4)}\n`
+		markdown += `- **Average Duration:** ${(report.averageDuration / 1000).toFixed(2)}s\n\n`
+	}
+
+	// Add charts using Mermaid
+	markdown += `## Charts\n\n`
+
+	// Success rate by benchmark chart
+	markdown += `### Success Rate by Benchmark\n\n`
+	markdown += "```mermaid\n"
+	markdown += "graph TD\n"
+	markdown += "  title[Success Rate by Benchmark]\n"
+	markdown += "  style title fill:none,stroke:none\n\n"
+
+	for (const [benchmark, report] of Object.entries(benchmarkReports)) {
+		const successRate = (report.successRate * 100).toFixed(2)
+		markdown += `  ${benchmark}[${benchmark}: ${successRate}%]\n`
+	}
+
+	markdown += "```\n\n"
+
+	// Success rate by model chart
+	markdown += `### Success Rate by Model\n\n`
+	markdown += "```mermaid\n"
+	markdown += "graph TD\n"
+	markdown += "  title[Success Rate by Model]\n"
+	markdown += "  style title fill:none,stroke:none\n\n"
+
+	for (const [model, report] of Object.entries(modelReports)) {
+		const successRate = (report.successRate * 100).toFixed(2)
+		markdown += `  ${model.replace(/[-\.]/g, "_")}[${model}: ${successRate}%]\n`
+	}
+
+	markdown += "```\n\n"
+
+	// Add timestamp
+	markdown += `\n\n---\n\nReport generated on ${new Date().toISOString()}\n`
+
+	// Write markdown to file
+	fs.writeFileSync(outputPath, markdown)
+}

evals/cli/src/utils/results.ts

@@ -0,0 +1,79 @@
+import { v4 as uuidv4 } from "uuid"
+import { ResultsDatabase } from "../db"
+import { Task } from "../adapters/types"
+
+/**
+ * Store task result in the database
+ * @param runId The run ID
+ * @param task The task that was executed
+ * @param result The result from the test server
+ * @param verification The verification result
+ */
+export async function storeTaskResult(runId: string, task: Task, result: any, verification: any): Promise<void> {
+	const db = new ResultsDatabase()
+	const taskId = uuidv4()
+
+	try {
+		// Extract metrics from the result
+		const { metrics } = result
+		const totalToolCalls = metrics?.totalToolCalls || 0
+		const totalToolFailures = metrics?.totalToolFailures || 0
+
+		// Create task with tool metrics
+		db.createTask(taskId, runId, task.id)
+		db.completeTask(taskId, verification.success, totalToolCalls, totalToolFailures)
+
+		// Store metrics
+		if (metrics) {
+			// Store token metrics
+			if (metrics.tokensIn) db.addMetric(taskId, "tokensIn", metrics.tokensIn)
+			if (metrics.tokensOut) db.addMetric(taskId, "tokensOut", metrics.tokensOut)
+			if (metrics.cost) db.addMetric(taskId, "cost", metrics.cost)
+			if (metrics.duration) db.addMetric(taskId, "duration", metrics.duration)
+
+			// Store tool call metrics
+			if (metrics.toolCalls) {
+				for (const [toolName, callCount] of Object.entries(metrics.toolCalls)) {
+					const failureCount = metrics.toolFailures?.[toolName] || 0
+					db.addToolCall(taskId, toolName, callCount as number, failureCount)
+				}
+			}
+		}
+
+		// Store verification metrics
+		if (verification.metrics) {
+			for (const [key, value] of Object.entries(verification.metrics)) {
+				if (typeof value === "number") {
+					db.addMetric(taskId, key, value)
+				}
+			}
+		}
+
+		// Store file changes
+		if (result.files) {
+			// Store created files
+			if (result.files.created) {
+				for (const file of result.files.created) {
+					db.addFile(taskId, file, "created")
+				}
+			}
+
+			// Store modified files
+			if (result.files.modified) {
+				for (const file of result.files.modified) {
+					db.addFile(taskId, file, "modified")
+				}
+			}
+
+			// Store deleted files
+			if (result.files.deleted) {
+				for (const file of result.files.deleted) {
+					db.addFile(taskId, file, "deleted")
+				}
+			}
+		}
+	} finally {
+		// Close the database connection
+		db.close()
+	}
+}

evals/cli/src/utils/task.ts

@@ -0,0 +1,52 @@
+import fetch from "node-fetch"
+import chalk from "chalk"
+
+/**
+ * Send a task to the Cline test server
+ * @param task The task description to send
+ * @param apiKey Optional Cline API key to use for the task
+ * @returns The result of the task execution
+ */
+export async function sendTaskToServer(task: string, apiKey?: string): Promise<any> {
+	const SERVER_URL = "http://localhost:9876/task"
+
+	try {
+		console.log(chalk.blue(`Sending task to server: ${task.substring(0, 100)}${task.length > 100 ? "..." : ""}`))
+
+		const response = await fetch(SERVER_URL, {
+			method: "POST",
+			headers: {
+				"Content-Type": "application/json",
+			},
+			body: JSON.stringify({
+				task,
+				apiKey,
+			}),
+		})
+
+		if (!response.ok) {
+			const errorText = await response.text()
+			throw new Error(`Server responded with status ${response.status}: ${errorText}`)
+		}
+
+		const result = await response.json()
+
+		if (!result.success) {
+			throw new Error(`Task execution failed: ${result.error || "Unknown error"}`)
+		}
+
+		if (result.timeout) {
+			throw new Error("Task execution timed out")
+		}
+
+		return result
+	} catch (error: any) {
+		if (error.code === "ECONNREFUSED") {
+			throw new Error(
+				"Could not connect to the test server. Make sure VSCode is running with the Cline extension and the test server is active.",
+			)
+		}
+
+		throw error
+	}
+}

evals/cli/src/utils/vscode.ts

@@ -0,0 +1,570 @@
+import execa from "execa"
+import * as path from "path"
+import * as fs from "fs"
+import fetch from "node-fetch"
+import * as os from "os"
+import * as child_process from "child_process"
+import { installRequiredExtensions, configureExtensionSettings } from "./extensions"
+
+// Store temporary directories for cleanup
+interface VSCodeResources {
+	tempUserDataDir: string
+	tempExtensionsDir: string
+	vscodePid?: number
+}
+
+// Global map to track resources for each workspace
+const workspaceResources = new Map<string, VSCodeResources>()
+
+/**
+ * Spawn a VSCode instance with the Cline extension
+ * @param workspacePath The workspace path to open
+ * @param vsixPath Optional path to a VSIX file to install
+ * @returns The resources created for this VS Code instance
+ */
+export async function spawnVSCode(workspacePath: string, vsixPath?: string): Promise<VSCodeResources> {
+	// Ensure the workspace path exists
+	if (!fs.existsSync(workspacePath)) {
+		throw new Error(`Workspace path does not exist: ${workspacePath}`)
+	}
+
+	// If no VSIX path is provided, build one with IS_TEST=true
+	if (!vsixPath) {
+		try {
+			// Build the VSIX with IS_TEST=true
+			console.log("Building test VSIX...")
+			const clineRoot = path.resolve(process.cwd(), "..", "..")
+			await execa("npx", ["vsce", "package"], {
+				cwd: clineRoot,
+				env: {
+					IS_TEST: "true",
+				},
+				stdio: "inherit",
+			})
+
+			// Find the generated VSIX file
+			const files = fs.readdirSync(clineRoot)
+			const vsixFile = files.find((file) => file.endsWith(".vsix"))
+			if (vsixFile) {
+				vsixPath = path.join(clineRoot, vsixFile)
+				console.log(`Using built VSIX: ${vsixPath}`)
+			} else {
+				console.warn("Could not find generated VSIX file")
+			}
+		} catch (error) {
+			console.warn("Failed to build test VSIX:", error)
+		}
+	}
+
+	// Create a temporary user data directory for this VS Code instance
+	const tempUserDataDir = path.join(os.tmpdir(), `vscode-cline-eval-${Date.now()}`)
+	fs.mkdirSync(tempUserDataDir, { recursive: true })
+	console.log(`Created temporary user data directory: ${tempUserDataDir}`)
+
+	// Create a temporary extensions directory to ensure no other extensions are loaded
+	const tempExtensionsDir = path.join(os.tmpdir(), `vscode-cline-eval-ext-${Date.now()}`)
+	fs.mkdirSync(tempExtensionsDir, { recursive: true })
+	console.log(`Created temporary extensions directory: ${tempExtensionsDir}`)
+
+	// Create settings.json in the temporary user data directory to disable workspace trust
+	// and configure Cline to auto-open on startup
+	const settingsDir = path.join(tempUserDataDir, "User")
+	fs.mkdirSync(settingsDir, { recursive: true })
+	const settingsPath = path.join(settingsDir, "settings.json")
+	const settings = {
+		// Disable workspace trust
+		"security.workspace.trust.enabled": false,
+		"security.workspace.trust.startupPrompt": "never",
+		"security.workspace.trust.banner": "never",
+		"security.workspace.trust.emptyWindow": true,
+
+		// Configure startup behavior
+		"workbench.startupEditor": "none",
+
+		// Auto-open Cline on startup
+		"cline.autoOpenOnStartup": true,
+
+		// Show the activity bar and sidebar
+		"workbench.activityBar.visible": true,
+		"workbench.sideBar.visible": true,
+		"workbench.view.extension.saoudrizwan.claude-dev-ActivityBar.visible": true,
+		"workbench.view.alwaysShowHeaderActions": true,
+		"workbench.editor.openSideBySideDirection": "right",
+
+		// Disable GitLens from opening automatically
+		"gitlens.views.repositories.autoReveal": false,
+		"gitlens.views.fileHistory.autoReveal": false,
+		"gitlens.views.lineHistory.autoReveal": false,
+		"gitlens.views.compare.autoReveal": false,
+		"gitlens.views.search.autoReveal": false,
+		"gitlens.showWelcomeOnInstall": false,
+		"gitlens.showWhatsNewAfterUpgrades": false,
+
+		// Disable other extensions that might compete for startup focus
+		"extensions.autoUpdate": false,
+	}
+	fs.writeFileSync(settingsPath, JSON.stringify(settings, null, 2))
+	console.log(`Created settings.json to disable workspace trust and auto-open Cline`)
+
+	// Create keybindings.json to automatically open Cline on startup
+	const keybindingsPath = path.join(settingsDir, "keybindings.json")
+	const keybindings = [
+		{
+			key: "alt+c",
+			command: "workbench.view.extension.saoudrizwan.claude-dev-ActivityBar",
+			when: "viewContainer.workbench.view.extension.saoudrizwan.claude-dev-ActivityBar.enabled",
+		},
+		{
+			key: "alt+shift+c",
+			command: "cline.openInNewTab",
+			when: "viewContainer.workbench.view.extension.saoudrizwan.claude-dev-ActivityBar.enabled",
+		},
+	]
+	fs.writeFileSync(keybindingsPath, JSON.stringify(keybindings, null, 2))
+	console.log(`Created keybindings.json to help with Cline activation`)
+
+	// Build the command arguments with custom user data directory
+	const args = [
+		// Use a custom user data directory to isolate this instance
+		"--user-data-dir",
+		tempUserDataDir,
+		// Use a custom extensions directory to ensure only our extension is loaded
+		"--extensions-dir",
+		tempExtensionsDir,
+		// Disable workspace trust
+		"--disable-workspace-trust",
+		"-n",
+		workspacePath,
+		// Force the extension to be activated on startup
+		"--start-up-extension",
+		"saoudrizwan.claude-dev",
+		// Run a command on startup to open Cline
+		"--command",
+		"workbench.view.extension.saoudrizwan.claude-dev-ActivityBar",
+		// Additional flags to help with extension activation
+		"--disable-gpu=false",
+		"--max-memory=4096",
+	]
+
+	// Create a startup script to run commands after VS Code launches
+	const startupScriptPath = path.join(settingsDir, "startup.js")
+	const startupScript = `
+		// This script will be executed when VS Code starts
+		setTimeout(() => {
+			// Try to open Cline in the sidebar
+			require('vscode').commands.executeCommand('workbench.view.extension.saoudrizwan.claude-dev-ActivityBar');
+			
+			// Also try to open Cline in a tab as a fallback
+			setTimeout(() => {
+				require('vscode').commands.executeCommand('cline.openInNewTab');
+			}, 5000);
+		}, 5000);
+	`
+	fs.writeFileSync(startupScriptPath, startupScript)
+	console.log(`Created startup script to activate Cline`)
+
+	// If a VSIX is provided, install it
+	if (vsixPath) {
+		if (!fs.existsSync(vsixPath)) {
+			throw new Error(`VSIX file does not exist: ${vsixPath}`)
+		}
+		args.unshift("--install-extension", vsixPath)
+	}
+
+	// Install required extensions
+	console.log("Installing required VSCode extensions...")
+	await installRequiredExtensions(tempExtensionsDir)
+
+	// Configure extension settings
+	console.log("Configuring extension settings...")
+	configureExtensionSettings(tempUserDataDir)
+
+	// Execute the command
+	try {
+		// We don't need to install extensions globally anymore since we're using a custom user data directory
+		// The VSIX will be installed in the isolated environment if provided in the args
+
+		// Launch VS Code
+		console.log("Launching VS Code...")
+		await execa("code", args, {
+			stdio: "inherit",
+		})
+
+		// Wait longer for VSCode to initialize and extension to load
+		console.log("Waiting for VS Code to initialize...")
+		await new Promise((resolve) => setTimeout(resolve, 30000))
+
+		// Create a JavaScript file that will be loaded as a VS Code extension
+		const extensionDir = path.join(tempExtensionsDir, "cline-activator")
+		fs.mkdirSync(extensionDir, { recursive: true })
+
+		// Create package.json for the extension
+		const packageJsonPath = path.join(extensionDir, "package.json")
+		const packageJson = {
+			name: "cline-activator",
+			displayName: "Cline Activator",
+			description: "Activates Cline and starts the test server",
+			version: "0.0.1",
+			engines: {
+				vscode: "^1.60.0",
+			},
+			main: "./extension.js",
+			activationEvents: ["*"],
+			contributes: {
+				commands: [
+					{
+						command: "cline-activator.activate",
+						title: "Activate Cline",
+					},
+				],
+			},
+		}
+		fs.writeFileSync(packageJsonPath, JSON.stringify(packageJson, null, 2))
+
+		// Create extension.js
+		const extensionJsPath = path.join(extensionDir, "extension.js")
+		const extensionJs = `
+			const vscode = require('vscode');
+			
+			/**
+			 * @param {vscode.ExtensionContext} context
+			 */
+			function activate(context) {
+				console.log('Cline Activator is now active!');
+				
+				// Register the command to activate Cline
+				let disposable = vscode.commands.registerCommand('cline-activator.activate', async function () {
+					try {
+						// Make sure the Cline extension is activated
+						const extension = vscode.extensions.getExtension('saoudrizwan.claude-dev');
+						if (!extension) {
+							console.error('Cline extension not found');
+							return;
+						}
+						
+						if (!extension.isActive) {
+							console.log('Activating Cline extension...');
+							await extension.activate();
+						}
+						
+						// Show the Cline sidebar
+						console.log('Opening Cline sidebar...');
+						await vscode.commands.executeCommand('workbench.view.extension.saoudrizwan.claude-dev-ActivityBar');
+						
+						// Wait a moment for the sidebar to initialize
+						await new Promise(resolve => setTimeout(resolve, 2000));
+						
+						// Also open Cline in a tab as a fallback
+						console.log('Opening Cline in a tab...');
+						await vscode.commands.executeCommand('cline.openInNewTab');
+						
+						// Wait a moment for the tab to initialize
+						await new Promise(resolve => setTimeout(resolve, 2000));
+						
+						// Create the test server if it doesn't exist
+						console.log('Creating test server...');
+						
+						// Get the visible webview instance
+						const clineRootPath = '${path.resolve(process.cwd(), "..", "..")}';
+						const visibleWebview = require(path.join(clineRootPath, 'src', 'core', 'webview')).WebviewProvider.getVisibleInstance();
+						if (visibleWebview) {
+							require(path.join(clineRootPath, 'src', 'services', 'test', 'TestServer')).createTestServer(visibleWebview);
+							console.log('Test server created successfully');
+						} else {
+							console.error('No visible webview instance found');
+						}
+					} catch (error) {
+						console.error('Error activating Cline:', error);
+					}
+				});
+				
+				context.subscriptions.push(disposable);
+				
+				// Automatically run the command after a delay
+				setTimeout(() => {
+					vscode.commands.executeCommand('cline-activator.activate');
+				}, 5000);
+			}
+			
+			function deactivate() {}
+			
+			module.exports = {
+				activate,
+				deactivate
+			}
+		`
+		fs.writeFileSync(extensionJsPath, extensionJs)
+		console.log(`Created Cline Activator extension`)
+
+		// Try multiple approaches to activate the extension
+		let serverStarted = false
+
+		// Create an activation script to run in VS Code
+		const activationScriptPath = path.join(settingsDir, "activate-cline.js")
+		const activationScript = `
+			// This script will be executed to activate Cline and start the test server
+			const vscode = require('vscode');
+			
+			// Execute the cline-activator.activate command
+			vscode.commands.executeCommand('cline-activator.activate');
+		`
+		fs.writeFileSync(activationScriptPath, activationScript)
+		console.log(`Created activation script to run in VS Code`)
+
+		// Execute the activation script
+		try {
+			console.log("Executing activation script to start Cline and test server...")
+			await execa(
+				"code",
+				[
+					"--user-data-dir",
+					tempUserDataDir,
+					"--extensions-dir",
+					tempExtensionsDir,
+					"--folder-uri",
+					`file://${workspacePath}`,
+					"--execute",
+					activationScriptPath,
+				],
+				{
+					stdio: "inherit",
+				},
+			)
+
+			// Wait for the test server to start
+			console.log("Waiting for test server to start...")
+			for (let i = 0; i < 30; i++) {
+				try {
+					// Try to connect to the test server
+					const response = await fetch("http://localhost:9876/task", {
+						method: "OPTIONS",
+						headers: {
+							"Content-Type": "application/json",
+						},
+					})
+
+					if (response.status === 204) {
+						console.log("Test server is running!")
+						serverStarted = true
+						break
+					}
+				} catch (error) {
+					// Server not started yet, wait and try again
+					await new Promise((resolve) => setTimeout(resolve, 1000))
+				}
+			}
+		} catch (error) {
+			console.warn("Failed to execute activation script:", error)
+		}
+
+		if (!serverStarted) {
+			console.warn("Test server did not start after multiple attempts")
+			console.log("You may need to manually open the Cline extension in VS Code")
+		}
+
+		// Store the resources for this workspace
+		const resources: VSCodeResources = {
+			tempUserDataDir,
+			tempExtensionsDir,
+		}
+
+		// Store in the global map
+		workspaceResources.set(workspacePath, resources)
+
+		// Return the resources
+		return resources
+	} catch (error: any) {
+		throw new Error(`Failed to spawn VSCode: ${error.message}`)
+	}
+}
+
+/**
+ * Clean up VS Code resources and shut down the test server
+ * @param workspacePath The workspace path to clean up resources for
+ */
+export async function cleanupVSCode(workspacePath: string): Promise<void> {
+	console.log(`Cleaning up VS Code resources for workspace: ${workspacePath}`)
+
+	// Get the resources for this workspace
+	const resources = workspaceResources.get(workspacePath)
+	if (!resources) {
+		console.log(`No resources found for workspace: ${workspacePath}`)
+		return
+	}
+
+	// Try to shut down the test server
+	try {
+		console.log("Shutting down test server...")
+		await fetch("http://localhost:9876/shutdown", {
+			method: "POST",
+			headers: {
+				"Content-Type": "application/json",
+			},
+		}).catch(() => {
+			// Ignore errors, the server might already be down
+		})
+	} catch (error) {
+		console.warn(`Error shutting down test server: ${error}`)
+	}
+
+	// Try to gracefully close VS Code instead of killing it
+	try {
+		console.log("Attempting to gracefully close VS Code...")
+
+		// Create a settings file that will disable the crash reporter and the exit confirmation dialog
+		const settingsDir = path.join(resources.tempUserDataDir, "User")
+		const settingsPath = path.join(settingsDir, "settings.json")
+
+		// Read existing settings if they exist
+		let settings = {}
+		if (fs.existsSync(settingsPath)) {
+			try {
+				settings = JSON.parse(fs.readFileSync(settingsPath, "utf8"))
+			} catch (error) {
+				console.warn(`Error reading settings file: ${error}`)
+			}
+		}
+
+		// Update settings to disable crash reporter and exit confirmation
+		settings = {
+			...settings,
+			"window.confirmBeforeClose": "never",
+			"telemetry.enableCrashReporter": false,
+			"window.restoreWindows": "none",
+			"window.newWindowDimensions": "default",
+		}
+
+		// Write updated settings
+		fs.writeFileSync(settingsPath, JSON.stringify(settings, null, 2))
+
+		// On macOS, use AppleScript to quit VS Code gracefully
+		if (process.platform === "darwin") {
+			try {
+				// First try AppleScript to quit VS Code gracefully
+				await execa("osascript", ["-e", 'tell application "Visual Studio Code" to quit'])
+
+				// Wait a moment for VS Code to close
+				await new Promise((resolve) => setTimeout(resolve, 2000))
+			} catch (appleScriptError) {
+				console.warn(`Error using AppleScript to quit VS Code: ${appleScriptError}`)
+			}
+		} else if (process.platform === "win32") {
+			// On Windows, try to use taskkill without /F first
+			try {
+				await execa("taskkill", ["/IM", "code.exe"])
+
+				// Wait a moment for VS Code to close
+				await new Promise((resolve) => setTimeout(resolve, 2000))
+			} catch (taskkillError) {
+				console.warn(`Error using taskkill to quit VS Code: ${taskkillError}`)
+			}
+		} else {
+			// On Linux, try to use SIGTERM first
+			try {
+				// Find VS Code processes
+				const { stdout } = await execa("ps", ["aux"])
+				const lines = stdout.split("\n")
+
+				for (const line of lines) {
+					if (line.includes(resources.tempUserDataDir)) {
+						const parts = line.trim().split(/\s+/)
+						const pid = parseInt(parts[1])
+
+						if (pid && !isNaN(pid)) {
+							console.log(`Sending SIGTERM to VS Code process with PID: ${pid}`)
+							try {
+								// Use SIGTERM instead of SIGKILL for a graceful shutdown
+								process.kill(pid, "SIGTERM")
+							} catch (killError) {
+								console.warn(`Failed to terminate process ${pid}: ${killError}`)
+							}
+						}
+					}
+				}
+
+				// Wait a moment for VS Code to close
+				await new Promise((resolve) => setTimeout(resolve, 2000))
+			} catch (psError) {
+				console.warn(`Error listing processes: ${psError}`)
+			}
+		}
+
+		// If graceful methods failed, fall back to forceful termination as a last resort
+		// Check if VS Code is still running with the temp user data dir
+		let vsCodeStillRunning = false
+
+		if (process.platform !== "win32") {
+			try {
+				const { stdout } = await execa("ps", ["aux"])
+				vsCodeStillRunning = stdout.split("\n").some((line) => line.includes(resources.tempUserDataDir))
+			} catch (error) {
+				console.warn(`Error checking if VS Code is still running: ${error}`)
+			}
+		} else {
+			try {
+				const { stdout } = await execa("tasklist", ["/FI", `IMAGENAME eq code.exe`])
+				vsCodeStillRunning = stdout.includes("code.exe")
+			} catch (error) {
+				console.warn(`Error checking if VS Code is still running: ${error}`)
+			}
+		}
+
+		// If VS Code is still running, use forceful termination as a last resort
+		if (vsCodeStillRunning) {
+			console.log("Graceful shutdown failed, falling back to forceful termination...")
+
+			if (process.platform === "win32") {
+				try {
+					await execa("taskkill", ["/IM", "code.exe", "/F"])
+				} catch (error) {
+					console.warn(`Error forcefully terminating VS Code: ${error}`)
+				}
+			} else {
+				try {
+					const { stdout } = await execa("ps", ["aux"])
+					const lines = stdout.split("\n")
+
+					for (const line of lines) {
+						if (line.includes(resources.tempUserDataDir)) {
+							const parts = line.trim().split(/\s+/)
+							const pid = parseInt(parts[1])
+
+							if (pid && !isNaN(pid)) {
+								console.log(`Forcefully killing VS Code process with PID: ${pid}`)
+								try {
+									process.kill(pid, "SIGKILL")
+								} catch (killError) {
+									console.warn(`Failed to kill process ${pid}: ${killError}`)
+								}
+							}
+						}
+					}
+				} catch (error) {
+					console.warn(`Error forcefully terminating VS Code: ${error}`)
+				}
+			}
+		}
+	} catch (error) {
+		console.warn(`Error closing VS Code: ${error}`)
+	}
+
+	// Clean up temporary directories
+	try {
+		console.log(`Removing temporary user data directory: ${resources.tempUserDataDir}`)
+		fs.rmSync(resources.tempUserDataDir, { recursive: true, force: true })
+	} catch (error) {
+		console.warn(`Error removing temporary user data directory: ${error}`)
+	}
+
+	try {
+		console.log(`Removing temporary extensions directory: ${resources.tempExtensionsDir}`)
+		fs.rmSync(resources.tempExtensionsDir, { recursive: true, force: true })
+	} catch (error) {
+		console.warn(`Error removing temporary extensions directory: ${error}`)
+	}
+
+	// Remove from the global map
+	workspaceResources.delete(workspacePath)
+
+	console.log("Cleanup completed")
+}

evals/cli/tsconfig.json

@@ -0,0 +1,17 @@
+{
+	"compilerOptions": {
+		"target": "ES2020",
+		"module": "commonjs",
+		"lib": ["ES2020"],
+		"declaration": true,
+		"outDir": "dist",
+		"rootDir": "src",
+		"strict": true,
+		"esModuleInterop": true,
+		"skipLibCheck": true,
+		"forceConsistentCasingInFileNames": true,
+		"resolveJsonModule": true
+	},
+	"include": ["src/**/*"],
+	"exclude": ["node_modules", "dist"]
+}

src/services/test/GitHelper.ts

@@ -0,0 +1,245 @@
+import * as path from "path"
+import * as vscode from "vscode"
+import { execa } from "execa"
+import { Logger } from "../../services/logging/Logger"
+import { WebviewProvider } from "../../core/webview"
+
+/**
+ * Gets a valid workspace path for Git operations
+ * @param visibleWebview The visible webview instance
+ * @returns A valid workspace path
+ */
+export function getWorkspacePath(visibleWebview: WebviewProvider): string {
+	// First try to get the path from the controller's state
+	let workspacePath = visibleWebview.controller.context.workspaceState.get<string>("cwd") || ""
+
+	// If workspace path is empty, try to get it from the active workspace folder
+	if (!workspacePath) {
+		const workspaceFolders = vscode.workspace.workspaceFolders
+		if (workspaceFolders && workspaceFolders.length > 0) {
+			workspacePath = workspaceFolders[0].uri.fsPath
+			Logger.log(`Using workspace folder path: ${workspacePath}`)
+		} else {
+			// If no workspace folder is open, use the extension directory as a fallback
+			workspacePath = path.join(__dirname, "..", "..", "..")
+			Logger.log(`No workspace folder found, using extension directory: ${workspacePath}`)
+		}
+	}
+
+	return workspacePath
+}
+
+/**
+ * Validates that the workspace path is valid and writable for Git operations
+ * @param workspacePath The workspace path to validate
+ * @throws Error if the workspace path is invalid or not writable
+ */
+export async function validateWorkspacePath(workspacePath: string): Promise<void> {
+	// Check if workspace path is valid
+	if (!workspacePath || workspacePath === "/") {
+		throw new Error(`Invalid workspace path: ${workspacePath}. Cannot initialize Git repository.`)
+	}
+
+	// Check if the directory exists
+	try {
+		const { stdout } = await execa("test", ["-d", workspacePath])
+	} catch (error) {
+		throw new Error(`Workspace path does not exist or is not a directory: ${workspacePath}`)
+	}
+
+	// Check if the directory is writable
+	try {
+		const testFile = path.join(workspacePath, ".cline_write_test")
+		await execa("touch", [testFile])
+		await execa("rm", [testFile])
+	} catch (error) {
+		throw new Error(`Workspace path is not writable: ${workspacePath}. Error: ${error.message}`)
+	}
+
+	Logger.log(`Validated workspace path: ${workspacePath}`)
+}
+
+/**
+ * Cleans up any existing Git repository in the specified workspace path
+ * @param workspacePath The workspace path to clean up
+ */
+export async function cleanupPreviousGit(workspacePath: string): Promise<void> {
+	const gitDir = path.join(workspacePath, ".git")
+
+	try {
+		// Check if .git directory exists using execa since we're already using it
+		try {
+			await execa("test", ["-d", gitDir])
+			// If we get here, the directory exists
+			Logger.log(`Removing existing Git repository in ${workspacePath}`)
+
+			// Use rm -rf to remove the directory
+			await execa("rm", ["-rf", gitDir])
+			Logger.log(`Removed existing Git repository`)
+		} catch (error) {
+			// Directory doesn't exist, which is fine
+			Logger.log(`No existing Git repository found in ${workspacePath}`)
+		}
+	} catch (error) {
+		Logger.log(`Warning: Failed to remove existing Git repository: ${error.message}`)
+	}
+}
+
+/**
+ * Initializes a Git repository in the specified workspace path
+ * @param workspacePath The workspace path to initialize Git in
+ * @returns True if the repository was newly initialized
+ */
+export async function initializeGitRepository(workspacePath: string): Promise<boolean> {
+	// Validate workspace path before proceeding
+	await validateWorkspacePath(workspacePath)
+
+	// Clean up any existing Git repository
+	await cleanupPreviousGit(workspacePath)
+
+	// Initialize a new Git repository
+	Logger.log(`Initializing Git repository in ${workspacePath}`)
+	try {
+		await execa("git", ["init"], { cwd: workspacePath })
+		await execa("git", ["config", "user.name", "Cline Evaluation"], { cwd: workspacePath })
+		await execa("git", ["config", "user.email", "cline@example.com"], { cwd: workspacePath })
+
+		// Try to create an initial commit, but don't fail if there are no files to commit
+		try {
+			// Check if there are any files to commit
+			const { stdout: statusOutput } = await execa("git", ["status", "--porcelain"], { cwd: workspacePath })
+
+			if (statusOutput.trim()) {
+				// There are files to commit
+				await execa("git", ["add", "."], { cwd: workspacePath })
+				await execa("git", ["commit", "-m", "Initial commit for evaluation"], { cwd: workspacePath })
+				Logger.log("Created initial Git commit in " + workspacePath)
+			} else {
+				// No files to commit, create an empty commit
+				Logger.log("No files to commit, creating empty initial commit")
+				try {
+					// Create an empty commit with --allow-empty
+					await execa("git", ["commit", "--allow-empty", "-m", "Initial empty commit for evaluation"], {
+						cwd: workspacePath,
+					})
+					Logger.log("Created empty initial commit in " + workspacePath)
+				} catch (emptyCommitError) {
+					// Even empty commit failed, but we'll continue anyway
+					Logger.log(`Warning: Failed to create empty commit: ${emptyCommitError.message}`)
+				}
+			}
+		} catch (commitError) {
+			// Initial commit failed, but Git is still initialized
+			Logger.log(`Warning: Failed to create initial commit: ${commitError.message}`)
+			Logger.log("Continuing without initial commit")
+		}
+
+		return true
+	} catch (gitError) {
+		// Only throw if Git initialization itself failed
+		const errorMessage = `Failed to initialize Git repository: ${gitError.message}`
+		Logger.log(errorMessage)
+		throw new Error(errorMessage)
+	}
+}
+
+/**
+ * Gets the file changes between the current state and the initial state
+ * @param workspacePath The workspace path to check for changes
+ * @returns Object containing lists of created, modified, and deleted files, plus the full diff
+ */
+export async function getFileChanges(workspacePath: string): Promise<{
+	created: string[]
+	modified: string[]
+	deleted: string[]
+	diff: string
+}> {
+	// Validate workspace path before proceeding
+	await validateWorkspacePath(workspacePath)
+
+	// Make sure all changes are staged so they appear in the diff
+	Logger.log(`Staging all changes in ${workspacePath} for diff`)
+	try {
+		// First check if there are any untracked files
+		const { stdout: untrackedOutput } = await execa("git", ["ls-files", "--others", "--exclude-standard"], {
+			cwd: workspacePath,
+		})
+		if (untrackedOutput.trim()) {
+			Logger.log(`Found untracked files: ${untrackedOutput}`)
+		}
+
+		// Stage all changes including untracked files
+		await execa("git", ["add", "-A"], { cwd: workspacePath })
+		Logger.log("Staged all changes for diff")
+	} catch (error) {
+		Logger.log(`Warning: Failed to stage changes: ${error.message}`)
+	}
+
+	try {
+		// Get list of changed files
+		const { stdout: statusOutput } = await execa("git", ["status", "--porcelain"], { cwd: workspacePath })
+		Logger.log(`Git status output: ${statusOutput || "(empty)"}`)
+
+		const created: string[] = []
+		const modified: string[] = []
+		const deleted: string[] = []
+
+		// Parse git status output
+		statusOutput
+			.split("\n")
+			.filter(Boolean)
+			.forEach((line) => {
+				const status = line.substring(0, 2).trim()
+				const file = line.substring(3)
+
+				if (status === "A" || status === "??") {
+					created.push(file)
+				} else if (status === "M") {
+					modified.push(file)
+				} else if (status === "D") {
+					deleted.push(file)
+				}
+			})
+
+		// Get the full diff - include both staged and unstaged changes
+		const { stdout: diffOutput } = await execa("git", ["diff", "--staged"], { cwd: workspacePath })
+		Logger.log(`Git diff output length: ${diffOutput.length} characters`)
+
+		// If there's no diff, try getting the diff of unstaged changes
+		let finalDiff = diffOutput
+		if (!finalDiff) {
+			const { stdout: unstaged } = await execa("git", ["diff"], { cwd: workspacePath })
+			finalDiff = unstaged
+			Logger.log(`Unstaged git diff output length: ${unstaged.length} characters`)
+		}
+
+		return {
+			created,
+			modified,
+			deleted,
+			diff: finalDiff,
+		}
+	} catch (error) {
+		// Throw the error instead of returning a fallback
+		const errorMessage = `Error getting file changes: ${error.message}`
+		Logger.log(errorMessage)
+		throw new Error(errorMessage)
+	}
+}
+
+/**
+ * Calculates the tool success rate based on calls and failures
+ * @param toolCalls Record of tool calls by name
+ * @param toolFailures Record of tool failures by name
+ * @returns The success rate as a number between 0 and 1
+ */
+export function calculateToolSuccessRate(toolCalls: Record<string, number>, toolFailures: Record<string, number>): number {
+	const totalCalls = Object.values(toolCalls).reduce((a, b) => a + b, 0)
+	const totalFailures = Object.values(toolFailures).reduce((a, b) => a + b, 0)
+
+	if (totalCalls === 0) {
+		return 1.0 // No calls means no failures
+	}
+
+	return 1.0 - totalFailures / totalCalls
+}

src/services/test/TestServer.ts

@@ -1,11 +1,66 @@
 import * as http from "http"
 import * as vscode from "vscode"
+import * as path from "path"
+import { execa } from "execa"
 import { Logger } from "../../services/logging/Logger"
 import { WebviewProvider } from "../../core/webview"
 import { AutoApprovalSettings } from "../../shared/AutoApprovalSettings"
-import { updateGlobalState, getAllExtensionState } from "../../core/storage/state"
+import {
+	getWorkspacePath,
+	validateWorkspacePath,
+	initializeGitRepository,
+	getFileChanges,
+	calculateToolSuccessRate,
+} from "./GitHelper"
+import { updateGlobalState, getAllExtensionState, updateApiConfiguration, storeSecret } from "../../core/storage/state"
 import { ClineAsk, ExtensionMessage } from "../../shared/ExtensionMessage"
+import { ApiProvider } from "../../shared/api"
 import { WebviewMessage } from "../../shared/WebviewMessage"
+import { HistoryItem } from "../../shared/HistoryItem"
+import { getSavedClineMessages, getSavedApiConversationHistory } from "../../core/storage/disk"
+
+/**
+ * Creates a tracker to monitor tool calls and failures during task execution
+ * @param webviewProvider The webview provider instance
+ * @returns Object tracking tool calls and failures
+ */
+function createToolCallTracker(webviewProvider: WebviewProvider): {
+	toolCalls: Record<string, number>
+	toolFailures: Record<string, number>
+} {
+	const tracker = {
+		toolCalls: {} as Record<string, number>,
+		toolFailures: {} as Record<string, number>,
+	}
+
+	// Intercept messages to track tool usage
+	const originalPostMessageToWebview = webviewProvider.controller.postMessageToWebview
+	webviewProvider.controller.postMessageToWebview = async (message: ExtensionMessage) => {
+		// Track tool calls
+		if (message.type === "partialMessage" && message.partialMessage?.say === "tool") {
+			const toolName = (message.partialMessage.text as any)?.tool
+			if (toolName) {
+				tracker.toolCalls[toolName] = (tracker.toolCalls[toolName] || 0) + 1
+			}
+		}
+
+		// Track tool failures
+		if (message.type === "partialMessage" && message.partialMessage?.say === "error") {
+			const errorText = message.partialMessage.text
+			if (errorText && errorText.includes("Error executing tool")) {
+				const match = errorText.match(/Error executing tool: (\w+)/)
+				if (match && match[1]) {
+					const toolName = match[1]
+					tracker.toolFailures[toolName] = (tracker.toolFailures[toolName] || 0) + 1
+				}
+			}
+		}
+
+		return originalPostMessageToWebview.call(webviewProvider.controller, message)
+	}
+
+	return tracker
+}
 
 // Task completion tracking
 let taskCompletionResolver: (() => void) | null = null
@@ -74,6 +129,13 @@ async function updateAutoApprovalSettings(context: vscode.ExtensionContext, prov
  * @returns The created HTTP server instance
  */
 export function createTestServer(webviewProvider?: WebviewProvider): http.Server {
+	// Try to show the Cline sidebar
+	Logger.log("[createTestServer] Opening Cline in sidebar...")
+	vscode.commands.executeCommand("workbench.view.claude-dev-ActivityBar")
+
+	// Then ensure the webview is focused/loaded
+	vscode.commands.executeCommand("claude-dev.SidebarProvider.focus")
+
 	// Update auto approval settings if webviewProvider is available
 	if (webviewProvider?.controller?.context) {
 		updateAutoApprovalSettings(webviewProvider.controller.context, webviewProvider)
@@ -93,6 +155,19 @@ export function createTestServer(webviewProvider?: WebviewProvider): http.Server
 			return
 		}
 
+		// Handle shutdown request
+		if (req.method === "POST" && req.url === "/shutdown") {
+			res.writeHead(200)
+			res.end(JSON.stringify({ success: true, message: "Server shutting down" }))
+
+			// Shut down the server after sending the response
+			setTimeout(() => {
+				shutdownTestServer()
+			}, 100)
+
+			return
+		}
+
 		// Only handle POST requests to /task
 		if (req.method !== "POST" || req.url !== "/task") {
 			res.writeHead(404)
@@ -109,7 +184,7 @@ export function createTestServer(webviewProvider?: WebviewProvider): http.Server
 		req.on("end", async () => {
 			try {
 				// Parse the JSON body
-				const { task } = JSON.parse(body)
+				const { task, apiKey } = JSON.parse(body)
 
 				if (!task) {
 					res.writeHead(400)
@@ -129,9 +204,76 @@ export function createTestServer(webviewProvider?: WebviewProvider): http.Server
 				Logger.log(`Test server initiating task: ${task}`)
 
 				try {
+					// Get and validate the workspace path
+					const workspacePath = getWorkspacePath(visibleWebview)
+					Logger.log(`Using workspace path: ${workspacePath}`)
+
+					// Validate workspace path before proceeding with any operations
+					try {
+						await validateWorkspacePath(workspacePath)
+					} catch (error) {
+						Logger.log(`Workspace validation failed: ${error.message}`)
+						res.writeHead(500)
+						res.end(
+							JSON.stringify({
+								error: `Workspace validation failed: ${error.message}. Please open a workspace folder in VSCode before running the test.`,
+								workspacePath,
+							}),
+						)
+						return
+					}
+
+					// Initialize Git repository before starting the task
+					try {
+						const wasNewlyInitialized = await initializeGitRepository(workspacePath)
+						if (wasNewlyInitialized) {
+							Logger.log(`Initialized new Git repository in ${workspacePath} before task start`)
+						} else {
+							Logger.log(`Using existing Git repository in ${workspacePath} before task start`)
+						}
+
+						// Log directory contents before task start
+						try {
+							const { stdout: lsOutput } = await execa("ls", ["-la", workspacePath])
+							Logger.log(`Directory contents before task start:\n${lsOutput}`)
+						} catch (lsError) {
+							Logger.log(`Warning: Failed to list directory contents: ${lsError.message}`)
+						}
+					} catch (gitError) {
+						Logger.log(`Warning: Git initialization failed: ${gitError.message}`)
+						Logger.log("Continuing without Git initialization")
+					}
+
 					// Clear any existing task
 					await visibleWebview.controller.clearTask()
 
+					// If API key is provided, update the API configuration
+					if (apiKey) {
+						Logger.log("API key provided, updating API configuration")
+
+						// Get current API configuration
+						const { apiConfiguration } = await getAllExtensionState(visibleWebview.controller.context)
+
+						// Update API configuration with API key
+						const updatedConfig = {
+							...apiConfiguration,
+							apiProvider: "cline" as ApiProvider,
+							clineApiKey: apiKey,
+						}
+
+						// Store the API key securely
+						await storeSecret(visibleWebview.controller.context, "clineApiKey", apiKey)
+
+						// Update the API configuration
+						await updateApiConfiguration(visibleWebview.controller.context, updatedConfig)
+
+						// Update global state to use cline provider
+						await updateGlobalState(visibleWebview.controller.context, "apiProvider", "cline" as ApiProvider)
+
+						// Post state to webview to reflect changes
+						await visibleWebview.controller.postStateToWebview()
+					}
+
 					// Ensure we're in Act mode before initiating the task
 					const { chatSettings } = await visibleWebview.controller.getStateToPostToWebview()
 					if (chatSettings.mode === "plan") {
@@ -139,8 +281,47 @@ export function createTestServer(webviewProvider?: WebviewProvider): http.Server
 						await visibleWebview.controller.togglePlanActModeWithChatSettings({ mode: "act" })
 					}
 
+					// Initialize tool call tracker
+					const toolTracker = createToolCallTracker(visibleWebview)
+
+					// Record task start time
+					const taskStartTime = Date.now()
+
 					// Initiate the new task
-					const taskId = await visibleWebview.controller.initTask(task)
+					const result = await visibleWebview.controller.initTask(task)
+
+					// Try to get the task ID directly from the result or from the state
+					let taskId: string | undefined
+
+					if (typeof result === "string") {
+						// If initTask returns the task ID directly
+						taskId = result
+					} else {
+						// Wait a moment for the state to update
+						await new Promise((resolve) => setTimeout(resolve, 1000))
+
+						// Try to get the task ID from the controller's state
+						const state = await visibleWebview.controller.getStateToPostToWebview()
+						taskId = state.currentTaskItem?.id
+
+						// If still not found, try polling a few times
+						if (!taskId) {
+							for (let i = 0; i < 5; i++) {
+								await new Promise((resolve) => setTimeout(resolve, 500))
+								const updatedState = await visibleWebview.controller.getStateToPostToWebview()
+								taskId = updatedState.currentTaskItem?.id
+								if (taskId) {
+									break
+								}
+							}
+						}
+					}
+
+					if (!taskId) {
+						throw new Error("Failed to get task ID after initiating task")
+					}
+
+					Logger.log(`Task initiated with ID: ${taskId}`)
 
 					// Create a completion tracker for this task
 					const completionPromise = createTaskCompletionTracker()
@@ -154,13 +335,110 @@ export function createTestServer(webviewProvider?: WebviewProvider): http.Server
 						// Wait for either completion or timeout
 						await Promise.race([completionPromise, timeoutPromise])
 
-						// Return success response with the task ID
+						// Get task history and metrics
+						const taskHistory = await visibleWebview.controller.getStateToPostToWebview()
+						const taskData = taskHistory.taskHistory?.find((t: HistoryItem) => t.id === taskId)
+
+						// Get messages and API conversation history
+						let messages: any[] = []
+						let apiConversationHistory: any[] = []
+						try {
+							if (typeof taskId === "string") {
+								messages = await getSavedClineMessages(visibleWebview.controller.context, taskId)
+							}
+						} catch (error) {
+							Logger.log(`Error getting saved Cline messages: ${error}`)
+						}
+
+						try {
+							if (typeof taskId === "string") {
+								apiConversationHistory = await getSavedApiConversationHistory(
+									visibleWebview.controller.context,
+									taskId,
+								)
+							}
+						} catch (error) {
+							Logger.log(`Error getting saved API conversation history: ${error}`)
+						}
+
+						// Get file changes
+						let fileChanges
+						try {
+							// Get the workspace path using our helper function
+							const workspacePath = getWorkspacePath(visibleWebview)
+							Logger.log(`Getting file changes from workspace path: ${workspacePath}`)
+
+							// Log directory contents for debugging
+							try {
+								const { stdout: lsOutput } = await execa("ls", ["-la", workspacePath])
+								Logger.log(`Directory contents after task completion:\n${lsOutput}`)
+							} catch (lsError) {
+								Logger.log(`Warning: Failed to list directory contents: ${lsError.message}`)
+							}
+
+							// Get file changes using Git
+							fileChanges = await getFileChanges(workspacePath)
+
+							// If no changes were detected, use a fallback method
+							if (!fileChanges.created.length && !fileChanges.modified.length && !fileChanges.deleted.length) {
+								Logger.log("No changes detected by Git, using fallback directory scan")
+
+								// Try to get a list of all files in the directory
+								try {
+									const { stdout: findOutput } = await execa("find", [
+										workspacePath,
+										"-type",
+										"f",
+										"-not",
+										"-path",
+										"*/.*",
+										"-not",
+										"-path",
+										"*/node_modules/*",
+									])
+									const files = findOutput.split("\n").filter(Boolean)
+
+									// Add all files as "created" since we can't determine which ones are new
+									fileChanges.created = files.map((file) => path.relative(workspacePath, file))
+									Logger.log(`Fallback found ${fileChanges.created.length} files`)
+								} catch (findError) {
+									Logger.log(`Warning: Fallback directory scan failed: ${findError.message}`)
+								}
+							}
+						} catch (fileChangeError) {
+							Logger.log(`Error getting file changes: ${fileChangeError.message}`)
+							throw new Error(`Error getting file changes: ${fileChangeError.message}`)
+						}
+
+						// Get tool metrics
+						const toolMetrics = {
+							toolCalls: toolTracker.toolCalls,
+							toolFailures: toolTracker.toolFailures,
+							totalToolCalls: Object.values(toolTracker.toolCalls).reduce((a, b) => a + b, 0),
+							totalToolFailures: Object.values(toolTracker.toolFailures).reduce((a, b) => a + b, 0),
+							toolSuccessRate: calculateToolSuccessRate(toolTracker.toolCalls, toolTracker.toolFailures),
+						}
+
+						// Calculate task duration
+						const taskDuration = Date.now() - taskStartTime
+
+						// Return comprehensive response with all metrics and data
 						res.writeHead(200, { "Content-Type": "application/json" })
 						res.end(
 							JSON.stringify({
 								success: true,
 								taskId,
 								completed: true,
+								metrics: {
+									tokensIn: taskData?.tokensIn || 0,
+									tokensOut: taskData?.tokensOut || 0,
+									cost: taskData?.totalCost || 0,
+									duration: taskDuration,
+									...toolMetrics,
+								},
+								messages,
+								apiConversationHistory,
+								files: fileChanges,
 							}),
 						)
 					} catch (timeoutError) {