Merge branch 'bee/stream-text' of https://github.com/cline/cline into bee/stream-text

- Add `name` property to minimax, kat-coder-pro, and trinity-large-preview
  models that were previously missing it
- Move type annotation from `as FeaturedModel[]` casts to the variable
  declaration for proper type checking at assignment time
- Add test to verify all featured models include a display name

.changeset/dark-hotels-care.md

@@ -0,0 +1,5 @@
+---
+"claude-dev": patch
+---
+
+Add .agents/skills as default skill directory (global and local)

.changeset/evals-testing-framework.md

@@ -0,0 +1,10 @@
+---
+"claude-dev": patch
+---
+
+Add comprehensive LLM evaluation framework with CI integration
+
+- Smoke tests: 7 curated scenarios testing tools across providers (Claude, GPT-5, Gemini)
+- Analysis framework: pass@k metrics for measuring reliability
+- CI workflow: Parallel smoke tests on PRs with ~3min execution time
+- E2E runner: cline-bench integration for real-world task evaluation (local only for now)

.changeset/swift-comics-hammer.md

@@ -0,0 +1,5 @@
+---
+"cline": patch
+---
+
+Allows users to enter custom aws region when selecting bedrock as a provider

.changeset/wicked-tables-sneeze.md

@@ -0,0 +1,5 @@
+---
+"cline": patch
+---
+
+Prevent Parent Container Scrolling In Dropdowns

.codex/environments/environment.toml

@@ -14,14 +14,7 @@ fi
 [[actions]]
 name = "VS Code"
 icon = "run"
-command = '''
-npm run compile && IS_DEV=true DEV_WORKSPACE_FOLDER="$(pwd)" CLINE_ENVIRONMENT=production code \
-  --extensionDevelopmentPath="$(pwd)" \
-  --disable-workspace-trust \
-  --disable-extension saoudrizwan.claude-dev \
-  --disable-extension saoudrizwan.cline-nightly \
-  "$(pwd)"
-'''
+command = "chmod +x ./scripts/run-extension-host.sh && ./scripts/run-extension-host.sh production"
 
 [[actions]]
 name = "CLI"
@@ -38,5 +31,4 @@ command = '''
 rm node_modules
 rm webview-ui/node_modules
 npm run install:all
-git checkout package-lock.json webview-ui/package-lock.json
 '''

.github/workflows/cline-evals-regression.yml

@@ -0,0 +1,70 @@
+name: Smoke Tests
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'src/core/**'
+      - 'src/shared/**'
+      - 'proto/**'
+      - 'evals/**'
+      - '.github/workflows/cline-evals-regression.yml'
+  pull_request:
+    paths:
+      - 'src/core/**'
+      - 'src/shared/**'
+      - 'proto/**'
+      - 'evals/**'
+      - '.github/workflows/cline-evals-regression.yml'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: smoke-tests-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  smoke-tests:
+    name: Smoke Tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '22'
+          cache: 'npm'
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Build and install CLI
+        run: |
+          npm run protos
+          cd cli && npm install && npm run build && npm link
+          echo "$(npm config get prefix)/bin" >> $GITHUB_PATH
+
+      - name: Verify CLI
+        run: cline --version
+
+      - name: Run smoke tests
+        env:
+          CLINE_API_KEY: ${{ secrets.CLINE_API_KEY }}
+        run: |
+          cline auth -p cline -k "$CLINE_API_KEY" -m "anthropic/claude-sonnet-4.5"
+          npx tsx evals/smoke-tests/run-smoke-tests.ts --trials 1 --parallel
+
+      - name: Generate summary
+        if: always()
+        run: cat evals/smoke-tests/results/latest/summary.md >> $GITHUB_STEP_SUMMARY
+
+      - name: Upload results
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: smoke-test-results-${{ github.run_id }}
+          path: evals/smoke-tests/results/latest/
+          retention-days: 30

.gitignore

@@ -48,3 +48,6 @@ test-results
 .secrets
 
 *.tsbuildinfo
+
+# Smoke test results (generated)
+evals/smoke-tests/results/

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "evals/cline-bench"]
+	path = evals/cline-bench
+	url = https://github.com/cline/cline-bench.git

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## [3.59.0]
+
+- Added Minimax 2.5 Free Promo
+- Fixed Response chaining for OpenAI's Responses API 
+
 ## [3.58.0]
 
 ### Added

cli/CHANGELOG.md

@@ -1,8 +1,14 @@
 # cline
 
+## [2.2.1]
+
+- Added Minimax 2.5 Free Promo
+- Fixed Response chaining for OpenAI's Responses API 
+
 ## [2.2.0]
 
 ### Added
+
 - Subagent: replace legacy subagents with the native `use_subagents` tool
 - Bundle `endpoints.json` support so packaged distributions can ship required endpoints out-of-the-box
 - Amazon Bedrock: support parallel tool calling
@@ -13,6 +19,7 @@
 - ZAI/GLM: add GLM-5
 
 ### Fixed
+
 - CLI: handle stdin redirection correctly in CI/headless environments
 - CLI: preserve OAuth callback paths during auth redirects
 - VS Code Web: generate auth callback URLs via `vscode.env.asExternalUri` (OAuth callback reliability)
@@ -24,6 +31,7 @@
 - CI: increase Windows E2E test timeout to reduce flakiness
 
 ### Changed
+
 - Settings/model UX: move "reasoning effort" into model configuration and expose it in settings
 - CLI provider selection: limit provider list to those remotely configured
 - UI: consolidate ViewHeader component/styling across views

cli/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "cline",
-	"version": "2.2.0",
+	"version": "2.2.1",
 	"description": "Autonomous coding agent CLI - capable of creating/editing files, running commands, using the browser, and more",
 	"main": "dist/cli.mjs",
 	"bin": {

cli/src/components/AuthView.tsx

@@ -5,12 +5,13 @@
 
 import { Box, Text, useApp, useInput } from "ink"
 import Spinner from "ink-spinner"
-// biome-ignore lint/style/useImportType: React is used as a value by JSX (jsx: "react" in tsconfig)
 import React, { useCallback, useEffect, useMemo, useState } from "react"
+import { refreshOcaModels } from "@/core/controller/models/refreshOcaModels"
 import { StateManager } from "@/core/storage/StateManager"
 import { openAiCodexOAuthManager } from "@/integrations/openai-codex/oauth"
 import { AuthService } from "@/services/auth/AuthService"
-import { liteLlmDefaultModelId, openAiCodexDefaultModelId, openRouterDefaultModelId } from "@/shared/api"
+import { openAiCodexDefaultModelId, openRouterDefaultModelId } from "@/shared/api"
+import { StringRequest } from "@/shared/proto/cline/common"
 import { openExternal } from "@/utils/env"
 import { COLORS } from "../constants/colors"
 import { useStdinContext } from "../context/StdinContext"
@@ -31,6 +32,7 @@ import {
 } from "./FeaturedModelPicker"
 import { ImportView } from "./ImportView"
 import { getDefaultModelId, hasModelPicker, ModelPicker } from "./ModelPicker"
+import { OcaEmployeeCheck } from "./OcaEmployeeCheck"
 import { getProviderLabel } from "./ProviderPicker"
 
 type AuthStep =
@@ -43,6 +45,7 @@ type AuthStep =
 	| "success"
 	| "error"
 	| "cline_auth"
+	| "oca_employee_check"
 	| "oca_auth"
 	| "cline_model"
 	| "openai_codex_auth"
@@ -160,7 +163,6 @@ export const AuthView: React.FC<AuthViewProps> = ({ controller, onComplete, onEr
 	const [modelId, setModelId] = useState("")
 	const [baseUrl, setBaseUrl] = useState("")
 	const [errorMessage, setErrorMessage] = useState("")
-	const [authStatus, setAuthStatus] = useState<string>("")
 	const [providerSearch, setProviderSearch] = useState("")
 	const [providerIndex, setProviderIndex] = useState(0)
 	const [clineModelIndex, setClineModelIndex] = useState(0)
@@ -171,11 +173,14 @@ export const AuthView: React.FC<AuthViewProps> = ({ controller, onComplete, onEr
 	// OCA auth hook - enabled when step is oca_auth
 	const handleOcaAuthSuccess = useCallback(async () => {
 		await applyProviderConfig({ providerId: "oca", controller })
+		// Fetch OCA models from the API - this sets actModeOcaModelId/planModeOcaModelId in state
+		await refreshOcaModels(controller, StringRequest.create({ value: "" }))
 		const stateManager = StateManager.get()
 		stateManager.setGlobalState("welcomeViewCompleted", true)
 		await stateManager.flushPendingState()
 		setSelectedProvider("oca")
-		setModelId(liteLlmDefaultModelId)
+		const actModelId = stateManager.getGlobalSettingsKey("actModeOcaModelId") || ""
+		setModelId(actModelId)
 		setStep("success")
 	}, [controller])
 
@@ -317,7 +322,6 @@ export const AuthView: React.FC<AuthViewProps> = ({ controller, onComplete, onEr
 	const startClineAuth = useCallback(async () => {
 		try {
 			setStep("cline_auth")
-			setAuthStatus("Starting authentication...")
 			await AuthService.getInstance(controller).createAuthRequest()
 		} catch (error) {
 			setErrorMessage(error instanceof Error ? error.message : String(error))
@@ -327,7 +331,6 @@ export const AuthView: React.FC<AuthViewProps> = ({ controller, onComplete, onEr
 
 	const startOcaAuth = useCallback(() => {
 		setStep("oca_auth")
-		setAuthStatus("Starting authentication...")
 		initiateOcaAuth()
 	}, [initiateOcaAuth])
 
@@ -358,7 +361,8 @@ export const AuthView: React.FC<AuthViewProps> = ({ controller, onComplete, onEr
 		(value: string) => {
 			setSelectedProvider(value)
 			if (value === "oca") {
-				startOcaAuth()
+				// Show employee check screen before starting auth
+				setStep("oca_employee_check")
 			} else if (value === "openai-codex") {
 				setStep("openai_codex_auth")
 				startOpenAiCodexAuth()
@@ -534,9 +538,12 @@ export const AuthView: React.FC<AuthViewProps> = ({ controller, onComplete, onEr
 				setBaseUrl("")
 				setStep("modelid")
 				break
-			case "oca_auth":
+			case "oca_employee_check":
 				setStep("provider")
 				break
+			case "oca_auth":
+				setStep("oca_employee_check")
+				break
 			case "cline_auth":
 				setStep("menu")
 				break
@@ -675,6 +682,9 @@ export const AuthView: React.FC<AuthViewProps> = ({ controller, onComplete, onEr
 					</Box>
 				)
 
+			case "oca_employee_check":
+				return <OcaEmployeeCheck isActive={step === "oca_employee_check"} onCancel={goBack} onSignIn={startOcaAuth} />
+
 			case "oca_auth":
 			case "cline_auth":
 				return (
@@ -760,6 +770,7 @@ export const AuthView: React.FC<AuthViewProps> = ({ controller, onComplete, onEr
 	const [menuIndex, setMenuIndex] = useState(0)
 
 	// Steps that allow going back with escape (apikey handled by ApiKeyInput component)
+	// OcaEmployeeCheck handles its own escape key, so oca_employee_check is not in this list
 	const canGoBack = [
 		"provider",
 		"modelid",

cli/src/components/ModelPicker.tsx

@@ -6,6 +6,7 @@
 import { Box, Text } from "ink"
 import Spinner from "ink-spinner"
 import React, { useEffect, useMemo, useState } from "react"
+import { refreshOcaModels } from "@/core/controller/models/refreshOcaModels"
 import { refreshOpenRouterModels } from "@/core/controller/models/refreshOpenRouterModels"
 import {
 	type ApiProvider,
@@ -64,6 +65,7 @@ import {
 	xaiDefaultModelId,
 	xaiModels,
 } from "@/shared/api"
+import { StringRequest } from "@/shared/proto/cline/common"
 import { filterOpenRouterModelIds } from "@/shared/utils/model-filters"
 import { COLORS } from "../constants/colors"
 import { getOpenRouterDefaultModelId, usesOpenRouterModels } from "../utils/openrouter-models"
@@ -105,7 +107,7 @@ export function hasStaticModels(provider: string): boolean {
 }
 
 export function hasModelPicker(provider: string): boolean {
-	return hasStaticModels(provider) || usesOpenRouterModels(provider)
+	return hasStaticModels(provider) || usesOpenRouterModels(provider) || provider === "oca"
 }
 
 export function getDefaultModelId(provider: string): string {
@@ -132,7 +134,7 @@ export const ModelPicker: React.FC<ModelPickerProps> = ({ provider, controller,
 	const [isLoading, setIsLoading] = useState(false)
 	const [asyncModels, setAsyncModels] = useState<string[]>([])
 
-	// Fetch OpenRouter models when needed using shared core function
+	// Fetch async models (OpenRouter or OCA) when needed
 	useEffect(() => {
 		if (usesOpenRouterModels(provider)) {
 			setIsLoading(true)
@@ -145,11 +147,23 @@ export const ModelPicker: React.FC<ModelPickerProps> = ({ provider, controller,
 				.finally(() => {
 					setIsLoading(false)
 				})
+		} else if (provider === "oca") {
+			setIsLoading(true)
+			refreshOcaModels(controller, StringRequest.create({ value: "" }))
+				.then((result) => {
+					if (result.models) {
+						const modelIds = Object.keys(result.models).sort((a, b) => a.localeCompare(b))
+						setAsyncModels(modelIds)
+					}
+				})
+				.finally(() => {
+					setIsLoading(false)
+				})
 		}
 	}, [provider, controller])
 
 	const modelList = useMemo(() => {
-		if (usesOpenRouterModels(provider)) {
+		if (usesOpenRouterModels(provider) || provider === "oca") {
 			return asyncModels
 		}
 		return getModelList(provider)
@@ -180,7 +194,7 @@ export const ModelPicker: React.FC<ModelPickerProps> = ({ provider, controller,
 	}
 
 	// If async fetch returned no models, render nothing
-	if (usesOpenRouterModels(provider) && modelList.length === 0) {
+	if ((usesOpenRouterModels(provider) || provider === "oca") && modelList.length === 0) {
 		return null
 	}
 

cli/src/components/OcaEmployeeCheck.tsx

@@ -0,0 +1,88 @@
+/**
+ * OCA (Oracle Cloud Assist) employee check component.
+ * Shows a checkbox for "I'm an Oracle Employee" and a sign-in button.
+ * Sets ocaMode in state before triggering the OAuth flow.
+ */
+
+import { Box, Text, useInput } from "ink"
+// biome-ignore lint/style/useImportType: React is used as a value by JSX (jsx: "react" in tsconfig)
+import React, { useCallback, useState } from "react"
+import { StateManager } from "@/core/storage/StateManager"
+import { COLORS } from "../constants/colors"
+import { useStdinContext } from "../context/StdinContext"
+
+interface OcaEmployeeCheckProps {
+	/** Whether this component is active and should handle input */
+	isActive: boolean
+	/** Called when user confirms and wants to proceed with sign-in */
+	onSignIn: () => void
+	/** Called when user presses Escape to go back */
+	onCancel: () => void
+}
+
+export const OcaEmployeeCheck: React.FC<OcaEmployeeCheckProps> = ({ isActive, onSignIn, onCancel }) => {
+	const { isRawModeSupported } = useStdinContext()
+	const [isEmployee, setIsEmployee] = useState(true) // Default to checked (internal), matching extension behavior
+	const [selectedIndex, setSelectedIndex] = useState(0) // 0 = checkbox, 1 = sign in button
+
+	const ITEM_COUNT = 2
+
+	const handleSignIn = useCallback(async () => {
+		// Persist ocaMode to state before starting auth
+		const stateManager = StateManager.get()
+		stateManager.setGlobalState("ocaMode", isEmployee ? "internal" : "external")
+		await stateManager.flushPendingState()
+		onSignIn()
+	}, [isEmployee, onSignIn])
+
+	useInput(
+		(_input, key) => {
+			if (key.escape) {
+				onCancel()
+				return
+			}
+			if (key.upArrow) {
+				setSelectedIndex((prev) => (prev > 0 ? prev - 1 : ITEM_COUNT - 1))
+			} else if (key.downArrow) {
+				setSelectedIndex((prev) => (prev < ITEM_COUNT - 1 ? prev + 1 : 0))
+			} else if (key.tab || (key.return && selectedIndex === 0)) {
+				// Toggle checkbox when Tab is pressed or Enter on checkbox item
+				if (selectedIndex === 0) {
+					setIsEmployee((prev) => !prev)
+				}
+			} else if (key.return && selectedIndex === 1) {
+				// Sign in button
+				handleSignIn()
+			}
+		},
+		{ isActive: isRawModeSupported && isActive },
+	)
+
+	return (
+		<Box flexDirection="column">
+			<Text color="white">Oracle Code Assist</Text>
+			<Text> </Text>
+			{/* Checkbox: I'm an Oracle Employee */}
+			<Text>
+				<Text bold color={selectedIndex === 0 ? COLORS.primaryBlue : undefined}>
+					{selectedIndex === 0 ? "❯" : " "}{" "}
+				</Text>
+				<Text color={selectedIndex === 0 || isEmployee ? COLORS.primaryBlue : "gray"}>{isEmployee ? "[✓]" : "[ ]"}</Text>
+				<Text color={selectedIndex === 0 ? COLORS.primaryBlue : "white"}> I'm an Oracle Employee</Text>
+				{selectedIndex === 0 && <Text color="gray"> (Tab to toggle)</Text>}
+			</Text>
+			{/* Sign in button */}
+			<Text>
+				<Text bold color={selectedIndex === 1 ? COLORS.primaryBlue : undefined}>
+					{selectedIndex === 1 ? "❯" : " "}{" "}
+				</Text>
+				<Text color={selectedIndex === 1 ? COLORS.primaryBlue : "white"}>Sign in with Oracle Code Assist</Text>
+				{selectedIndex === 1 && <Text color="gray"> (Enter)</Text>}
+			</Text>
+			<Text> </Text>
+			<Text color="gray">Please ask your IT administrator to set up Oracle Code Assist as a model provider.</Text>
+			<Text> </Text>
+			<Text color="gray">Arrows to navigate, Tab to toggle, Enter to continue, Esc to go back</Text>
+		</Box>
+	)
+}

cli/src/components/SettingsPanelContent.tsx

@@ -14,10 +14,12 @@ import Spinner from "ink-spinner"
 import React, { useCallback, useEffect, useMemo, useState } from "react"
 import { buildApiHandler } from "@/core/api"
 import type { Controller } from "@/core/controller"
+import { refreshOcaModels } from "@/core/controller/models/refreshOcaModels"
 import { StateManager } from "@/core/storage/StateManager"
 import { openAiCodexOAuthManager } from "@/integrations/openai-codex/oauth"
 import { ClineAccountService } from "@/services/account/ClineAccountService"
 import { AuthService, ClineAccountOrganization } from "@/services/auth/AuthService"
+import { StringRequest } from "@/shared/proto/cline/common"
 import { openExternal } from "@/utils/env"
 import { supportsReasoningEffortForModel } from "@/utils/model-utils"
 import { version as CLI_VERSION } from "../../package.json"
@@ -37,6 +39,7 @@ import {
 } from "./FeaturedModelPicker"
 import { LanguagePicker } from "./LanguagePicker"
 import { hasModelPicker, ModelPicker } from "./ModelPicker"
+import { OcaEmployeeCheck } from "./OcaEmployeeCheck"
 import { OrganizationPicker } from "./OrganizationPicker"
 import { Panel, PanelTab } from "./Panel"
 import { getProviderLabel, ProviderPicker } from "./ProviderPicker"
@@ -162,6 +165,7 @@ export const SettingsPanelContent: React.FC<SettingsPanelContentProps> = ({
 	const [isEnteringApiKey, setIsEnteringApiKey] = useState(false)
 	const [isConfiguringBedrock, setIsConfiguringBedrock] = useState(false)
 	const [isWaitingForCodexAuth, setIsWaitingForCodexAuth] = useState(false)
+	const [isShowingOcaEmployeeCheck, setIsShowingOcaEmployeeCheck] = useState(false)
 	const [codexAuthError, setCodexAuthError] = useState<string | null>(null)
 	const [pendingProvider, setPendingProvider] = useState<string | null>(null)
 	const [apiKeyValue, setApiKeyValue] = useState("")
@@ -235,6 +239,8 @@ export const SettingsPanelContent: React.FC<SettingsPanelContentProps> = ({
 	// OCA auth hook
 	const handleOcaAuthSuccess = useCallback(async () => {
 		await applyProviderConfig({ providerId: "oca", controller })
+		// Fetch OCA models from the API - this sets actModeOcaModelId/planModeOcaModelId in state
+		await refreshOcaModels(controller!, StringRequest.create({ value: "" }))
 		setProvider("oca")
 		refreshModelIds()
 	}, [controller, refreshModelIds])
@@ -1078,8 +1084,8 @@ export const SettingsPanelContent: React.FC<SettingsPanelContentProps> = ({
 					setProvider("oca")
 					refreshModelIds()
 				} else {
-					// Not logged in - trigger OAuth
-					startOcaAuth()
+					// Not logged in - show employee check before auth
+					setIsShowingOcaEmployeeCheck(true)
 				}
 				return
 			}
@@ -1370,7 +1376,7 @@ export const SettingsPanelContent: React.FC<SettingsPanelContentProps> = ({
 				return
 			}
 		},
-		{ isActive: isRawModeSupported && !isEnteringApiKey && !isConfiguringBedrock },
+		{ isActive: isRawModeSupported && !isEnteringApiKey && !isConfiguringBedrock && !isShowingOcaEmployeeCheck },
 	)
 
 	// Render content
@@ -1546,6 +1552,19 @@ export const SettingsPanelContent: React.FC<SettingsPanelContentProps> = ({
 			)
 		}
 
+		if (isShowingOcaEmployeeCheck) {
+			return (
+				<OcaEmployeeCheck
+					isActive={isShowingOcaEmployeeCheck}
+					onCancel={() => setIsShowingOcaEmployeeCheck(false)}
+					onSignIn={() => {
+						setIsShowingOcaEmployeeCheck(false)
+						startOcaAuth()
+					}}
+				/>
+			)
+		}
+
 		if (isWaitingForOcaAuth) {
 			return (
 				<Box flexDirection="column">
@@ -1727,6 +1746,7 @@ export const SettingsPanelContent: React.FC<SettingsPanelContentProps> = ({
 		!!codexAuthError ||
 		isPickingOrganization ||
 		isWaitingForClineAuth ||
+		isShowingOcaEmployeeCheck ||
 		isWaitingForOcaAuth ||
 		isEditing
 

cli/src/constants/featured-models.test.ts

@@ -0,0 +1,12 @@
+import { describe, expect, it } from "vitest"
+import { getAllFeaturedModels } from "./featured-models"
+
+describe("featured models", () => {
+	it("includes display names for all featured models", () => {
+		const models = getAllFeaturedModels()
+
+		for (const model of models) {
+			expect(model.name).toBeTruthy()
+		}
+	})
+})

cli/src/constants/featured-models.ts

@@ -10,7 +10,7 @@ export interface FeaturedModel {
 	labels: string[]
 }
 
-export const FEATURED_MODELS = {
+export const FEATURED_MODELS: { recommended: FeaturedModel[]; free: FeaturedModel[] } = {
 	recommended: [
 		{
 			id: "anthropic/claude-opus-4.6",
@@ -30,33 +30,27 @@ export const FEATURED_MODELS = {
 			description: "1M context window for large codebases",
 			labels: ["TRENDING"],
 		},
-	] as FeaturedModel[],
+	],
 	free: [
 		{
-			id: "minimax/minimax-m2.1",
-			name: "MiniMax M2.1",
-			description: "Exceptional Multi-Programming Language Capabilities",
-			labels: ["FREE"],
-		},
-		{
-			id: "moonshotai/kimi-k2.5",
-			name: "Kimi K2.5",
-			description: "State-of-the-art model topping benchmarks",
+			id: "minimax/minimax-m2.5",
+			name: "MiniMax M2.5",
+			description: "MiniMax-M2.5 is a lightweight, state-of-the-art LLM optimized for coding and agentic workflows",
 			labels: ["FREE"],
 		},
 		{
 			id: "kwaipilot/kat-coder-pro",
 			name: "KAT Coder Pro",
-			description: "Advanced agentic coding model",
+			description: "KwaiKAT's most advanced agentic coding model in the KAT-Coder series",
 			labels: ["FREE"],
 		},
 		{
 			id: "arcee-ai/trinity-large-preview:free",
 			name: "Trinity Large Preview",
-			description: "US built open source coding model",
+			description: "Arcee AI's advanced large preview model in the Trinity series",
 			labels: ["FREE"],
 		},
-	] as FeaturedModel[],
+	],
 }
 
 export function getAllFeaturedModels(): FeaturedModel[] {

cli/src/index.ts

@@ -15,9 +15,7 @@ import { HostProvider } from "@/hosts/host-provider"
 import { FileEditProvider } from "@/integrations/editor/FileEditProvider"
 import { openAiCodexOAuthManager } from "@/integrations/openai-codex/oauth"
 import { StandaloneTerminalManager } from "@/integrations/terminal/standalone/StandaloneTerminalManager"
-import { BannerService } from "@/services/banner/BannerService"
 import { ErrorService } from "@/services/error/ErrorService"
-import { initializeDistinctId } from "@/services/logging/distinctId"
 import { telemetryService } from "@/services/telemetry"
 import { PostHogClientProvider } from "@/services/telemetry/providers/posthog/PostHogClientProvider"
 import { HistoryItem } from "@/shared/HistoryItem"
@@ -76,7 +74,24 @@ async function disposeTelemetryServices(): Promise<void> {
 	await Promise.allSettled([telemetryService.dispose(), PostHogClientProvider.getInstance().dispose()])
 }
 
+/**
+ * Restore yoloModeToggled to its original value from before this CLI session.
+ * This ensures the --yolo flag is session-only and doesn't leak into future runs.
+ * Must be called before flushPendingState so the restored value gets persisted.
+ */
+function restoreYoloState(): void {
+	if (savedYoloModeToggled !== null) {
+		try {
+			StateManager.get().setGlobalState("yoloModeToggled", savedYoloModeToggled)
+			savedYoloModeToggled = null
+		} catch {
+			// StateManager may not be initialized (e.g., early exit before init)
+		}
+	}
+}
+
 async function disposeCliContext(ctx: CliContext): Promise<void> {
+	restoreYoloState()
 	await ctx.controller.stateManager.flushPendingState()
 	await ctx.controller.dispose()
 	await ErrorService.get().dispose()
@@ -189,9 +204,12 @@ function applyTaskOptions(options: TaskOptions): void {
 		telemetryService.captureHostEvent("max_consecutive_mistakes_flag", String(maxConsecutiveMistakes))
 	}
 
-	// Set yolo mode based on --yolo flag
+	// Override yolo mode only if --yolo flag is explicitly passed.
+	// The original value is saved in initializeCli and restored on exit.
 	if (options.yolo) {
-		StateManager.get().setGlobalState("yoloModeToggled", true)
+		const state = StateManager.get()
+		savedYoloModeToggled = state.getGlobalSettingsKey("yoloModeToggled") ?? false
+		state.setGlobalState("yoloModeToggled", true)
 		telemetryService.captureHostEvent("yolo_flag", "true")
 	}
 
@@ -296,6 +314,9 @@ let activeContext: CliContext | null = null
 let isShuttingDown = false
 // Track if we're in plain text mode (no Ink UI) - set by runTask when piped stdin detected
 let isPlainTextMode = false
+// Track the original yoloModeToggled value from before this CLI session so we can restore it on exit.
+// The --yolo flag should only affect the current invocation, not persist across runs.
+let savedYoloModeToggled: boolean | null = null
 
 /**
  * Wait for stdout to fully drain before exiting.
@@ -337,6 +358,10 @@ function setupSignalHandlers() {
 		printWarning(`${signal} received, shutting down...`)
 
 		try {
+			// Restore yolo state before any cleanup - this is idempotent and safe
+			// even if disposeCliContext also calls it (restoreYoloState checks savedYoloModeToggled !== null)
+			restoreYoloState()
+
 			if (activeContext) {
 				const task = activeContext.controller.task
 				if (task) {
@@ -344,6 +369,12 @@ function setupSignalHandlers() {
 				}
 				await disposeCliContext(activeContext)
 			} else {
+				// Best-effort flush of restored yolo state when no active context
+				try {
+					await StateManager.get().flushPendingState()
+				} catch {
+					// StateManager may not be initialized yet
+				}
 				await ErrorService.get().dispose()
 				await disposeTelemetryServices()
 			}
@@ -407,7 +438,6 @@ async function initializeCli(options: InitOptions): Promise<CliContext> {
 	Logger.subscribe(logToChannel)
 
 	await ClineEndpoint.initialize(EXTENSION_DIR)
-	await initializeDistinctId(extensionContext)
 
 	// Auto-update check (after endpoints initialized, so we can detect bundled configs)
 	autoUpdateOnStartup(CLI_VERSION)
@@ -437,6 +467,7 @@ async function initializeCli(options: InitOptions): Promise<CliContext> {
 	)
 
 	await StateManager.initialize(extensionContext as any)
+
 	await ErrorService.initialize()
 
 	// Initialize OpenAI Codex OAuth manager with extension context for secrets storage
@@ -445,8 +476,6 @@ async function initializeCli(options: InitOptions): Promise<CliContext> {
 	const webview = HostProvider.get().createWebviewProvider() as CliWebviewProvider
 	const controller = webview.controller
 
-	BannerService.initialize(webview.controller)
-
 	await telemetryService.captureExtensionActivated()
 	await telemetryService.captureHostEvent("cline_cli", "initialized")
 

cli/vitest.config.ts

@@ -13,6 +13,7 @@ export default defineConfig({
 	},
 	resolve: {
 		alias: {
+			vscode: path.resolve(__dirname, "src/vscode-shim.ts"),
 			// Match tsconfig paths - baseUrl is parent directory
 			"@": path.resolve(__dirname, "../src"),
 			"@api": path.resolve(__dirname, "../src/core/api"),

docs/features/skills.mdx

@@ -62,15 +62,16 @@ The description is critical because it's how Cline decides whether to activate a
 Skills can be stored in two locations:
 
 **Global Skills** apply to all your projects:
-- **macOS/Linux:** `~/.cline/skills/`
-- **Windows:** `C:\Users\USERNAME\.cline\skills\`
+- **macOS/Linux:** `~/.agents/skills/` (recommended) or `~/.cline/skills/`
+- **Windows:** `C:\Users\USERNAME\.agents\skills\` (recommended) or `C:\Users\USERNAME\.cline\skills\`
 
 **Project Skills** apply only to the current workspace:
-- `.cline/skills/` (recommended)
+- `.agents/skills/` (recommended)
+- `.cline/skills/`
 - `.clinerules/skills/`
 - `.claude/skills/` (for Claude Code compatibility)
 
-When a global skill and project skill have the same name, the global skill takes precedence. This lets you customize skills for your personal workflow while still using project defaults.
+When a global skill and project skill have the same name, the global skill takes precedence. Skills in `.agents/skills` directories take precedence over other locations with the same name, letting you customize skills for your personal workflow while still using project defaults.
 
 ## Managing Skills
 

docs/features/subagents.mdx

@@ -18,7 +18,7 @@ When Cline uses the `use_subagents` tool, it launches independent agents simulta
 - Runs with a separate context window and token budget
 - Can read files, search code, list directories, run read-only commands, and use skills
 - Cannot edit files, use the browser, access MCP servers, or spawn nested subagents
-- Returns a result that includes file paths, line numbers, and recommended files for the main agent to read next
+- Returns a result focused on the most relevant file paths for the main agent to read next
 
 Subagent costs (tokens and API spend) are tracked separately per subagent and rolled into the task's total cost. You can see per-subagent stats (tool calls, tokens, cost) in the chat UI as they run.
 
@@ -43,6 +43,7 @@ Example prompts:
 - "I'm new to this codebase. Use subagents to map out the main entry points, the routing layer, and the data access patterns"
 
 Each subagent prompt should describe a focused research question. Cline will run them in parallel and synthesize the results.
+You can also run only one subagent when the task is small enough that parallel discovery would be unnecessary overhead.
 
 ## Auto-Approve Behavior
 
@@ -69,6 +70,7 @@ Subagents cannot write files, apply patches, use the browser, access MCP servers
 
 <Note>
   Commands run by subagents execute in the background and are restricted to read-only operations. Subagents will not run commands that modify files or system state.
+  Subagents also benefit from command pipelines and filters to narrow output quickly before reading files, for example `rg ... | sort | uniq`.
 </Note>
 
 ## When to Use Subagents

evals/.gitignore

@@ -2,23 +2,28 @@ repositories
 temp-files
 results
 
+# Tool precision - results and databases
+benchmarks/tool-precision/replace-in-file/results/
+benchmarks/tool-precision/replace-in-file/*.db
+benchmarks/tool-precision/replace-in-file/*.db-wal
+benchmarks/tool-precision/replace-in-file/*.db-shm
+
+# Tool precision - private test cases (from real sessions)
+# Public/synthetic cases (example-*.json) ARE committed
+benchmarks/tool-precision/replace-in-file/cases/private-*.json
+benchmarks/tool-precision/replace-in-file/cases/session-*.json
+
+# Legacy paths (kept for backwards compatibility)
 diff-edits/cases/
 diff-edits/results/
+diff_editing/test_cases/
+diff_editing/test_outputs/
+diff-edits/cases.zip
 
 # Environment variables
 .env
 
-# backwards compatible
-diff_editing/test_cases/
-diff_editing/test_outputs/
-
-*.db
-*.db-wal
-*.db-shm
-
-.cache
-
 # Python bytecode cache
 *__pycache__/
 
-diff-edits/cases.zip
+.cache

evals/ARCHITECTURE.md

@@ -0,0 +1,288 @@
+# Cline Evals Architecture
+
+## Overview
+
+The evals system provides multi-layered testing for Cline's AI capabilities.
+
+```
+┌─────────────────────────────────────────────────────────────────────────────┐
+│                           TESTING PYRAMID                                   │
+├─────────────────────────────────────────────────────────────────────────────┤
+│                                                                             │
+│                              ┌─────────┐                                    │
+│                             /   E2E    \         Layer 3: Full Agent        │
+│                            /  cline-   \         - Real coding tasks        │
+│                           /   bench     \        - Harbor execution         │
+│                          /_______________\       - Nightly runs             │
+│                                                                             │
+│                        ┌───────────────────┐                                │
+│                       /    Smoke Tests     \     Layer 2: Provider          │
+│                      /   run-smoke-tests    \    - 5 curated scenarios      │
+│                     /    (cline provider)    \   - 3 models via Vercel      │
+│                    /_________________________\   - pass@k metrics           │
+│                                                                             │
+│              ┌─────────────────────────────────┐                            │
+│             /        Contract Tests            \  Layer 1: Unit             │
+│            /   thinking-traces.test.ts          \ - No LLM calls            │
+│           /    tool-parsing.test.ts              \ - Fast, deterministic    │
+│          /______________________________________ \ - API format validation  │
+│                                                                             │
+└─────────────────────────────────────────────────────────────────────────────┘
+```
+
+## Directory Structure
+
+```
+evals/
+├── ARCHITECTURE.md          # This file
+├── README.md                # Quick start guide
+│
+├── analysis/                # Shared metrics & reporting
+│   └── src/
+│       ├── metrics.ts       # pass@k, pass^k, flakiness calculations
+│       └── cli.ts           # Analysis CLI
+│
+├── smoke-tests/             # Layer 2: Provider smoke tests
+│   ├── run-smoke-tests.ts   # Main runner
+│   ├── README.md            # Usage docs
+│   ├── scenarios/           # Test definitions
+│   │   ├── 01-create-file/
+│   │   │   ├── config.json  # Prompt, expected files/content
+│   │   │   ├── template/    # Initial files (if any)
+│   │   │   └── workspace/   # Working dir (cleaned each run)
+│   │   ├── 02-edit-file/
+│   │   ├── 03-read-summarize/
+│   │   ├── 04-multi-file/
+│   │   └── 05-typescript-function/
+│   └── results/             # Generated outputs
+│       ├── latest -> 2026-01-27T.../  # Symlink to most recent
+│       └── 2026-01-27T19-50-54-391Z/
+│           ├── report.json  # Full results
+│           ├── summary.md   # CI-friendly markdown
+│           └── 01-create-file/
+│               └── claude-sonnet/
+│                   ├── trial-1.log           # CLI stdout/stderr
+│                   └── workspace-trial-1/    # Kept for failures only
+│
+├── e2e/                     # Layer 3: Full agent E2E
+│   ├── run-cline-bench.ts   # Harbor runner
+│   └── README.md
+│
+└── cline-bench/             # Git submodule with real coding tasks
+    └── tasks/               # SWE-bench style problems
+```
+
+## Smoke Test Workflow
+
+```
+┌──────────────────────────────────────────────────────────────────────────────┐
+│                         SMOKE TEST EXECUTION FLOW                            │
+└──────────────────────────────────────────────────────────────────────────────┘
+
+ npm run eval:smoke
+        │
+        ▼
+┌───────────────────┐
+│ Load scenarios    │  Read config.json from each scenarios/* dir
+│ from disk         │
+└────────┬──────────┘
+         │
+         ▼
+┌───────────────────┐
+│ Create results    │  evals/smoke-tests/results/2026-01-27T.../
+│ directory         │
+└────────┬──────────┘
+         │
+         ▼
+┌───────────────────────────────────────────────────────────────┐
+│                    FOR EACH SCENARIO                          │
+│  ┌─────────────────────────────────────────────────────────┐  │
+│  │                  FOR EACH MODEL                         │  │
+│  │  ┌───────────────────────────────────────────────────┐  │  │
+│  │  │           RUN 3 TRIALS SEQUENTIALLY               │  │  │
+│  │  │                                                   │  │  │
+│  │  │  Trial 1 ──► Trial 2 ──► Trial 3 ──► Results      │  │  │
+│  │  │  (Sequential - Cline instance handles one at a time)  │  │
+│  │  │                                                   │  │  │
+│  │  │  Each trial:                                      │  │  │
+│  │  │  1. Create workspace-trial-N/                     │  │  │
+│  │  │  2. Copy template files (if any)                  │  │  │
+│  │  │  3. Run: cline -y -o "prompt"                     │  │  │
+│  │  │  4. Verify expected files exist                   │  │  │
+│  │  │  5. Verify expected content                       │  │  │
+│  │  │  6. Save trial-N.log                              │  │  │
+│  │  │  7. If failed, copy workspace to results/         │  │  │
+│  │  └───────────────────────────────────────────────────┘  │  │
+│  │                         │                               │  │
+│  │                         ▼                               │  │
+│  │  ┌───────────────────────────────────────────────────┐  │  │
+│  │  │ Calculate metrics: pass@1, pass@3, pass^3, flaky  │  │  │
+│  │  └───────────────────────────────────────────────────┘  │  │
+│  └─────────────────────────────────────────────────────────┘  │
+└───────────────────────────────────────────────────────────────┘
+         │
+         ▼
+┌───────────────────┐
+│ Generate outputs  │
+│ - report.json     │
+│ - summary.md      │
+│ - latest symlink  │
+└───────────────────┘
+```
+
+## Models Tested
+
+```
+┌─────────────────────────────────────────────────────────────────┐
+│                    CLINE PROVIDER ROUTING                       │
+├─────────────────────────────────────────────────────────────────┤
+│                                                                 │
+│   ┌─────────────┐                                               │
+│   │ Smoke Test  │                                               │
+│   │   Runner    │                                               │
+│   └──────┬──────┘                                               │
+│          │                                                      │
+│          │  cline -y -o "prompt" --model <model>                │
+│          │                                                      │
+│          ▼                                                      │
+│   ┌─────────────┐                                               │
+│   │   Cline     │                                               │
+│   │  Provider   │ ◄─── Uses your Cline auth (cline auth)        │
+│   └──────┬──────┘                                               │
+│          │                                                      │
+│          │ Routes to backend                                    │
+│          ▼                                                      │
+│   ┌─────────────────────────────────────────────────────────┐   │
+│   │                   Default Models                        │   │
+│   ├─────────────────────────────────────────────────────────┤   │
+│   │  claude-sonnet-4-20250514                               │   │
+│   │  gpt-4o                                                 │   │
+│   │  gemini-2.5-pro-preview-06-05                           │   │
+│   └─────────────────────────────────────────────────────────┘   │
+│                                                                 │
+└─────────────────────────────────────────────────────────────────┘
+```
+
+## Metrics Explained
+
+```
+┌─────────────────────────────────────────────────────────────────┐
+│                        METRICS                                  │
+├─────────────────────────────────────────────────────────────────┤
+│                                                                 │
+│  pass@k   "What's the probability of getting at least one      │
+│            success if I run k trials?"                          │
+│                                                                 │
+│            Example: 2/3 trials pass → pass@3 ≈ 96%              │
+│            (Very likely to pass if you run 3 times)             │
+│                                                                 │
+├─────────────────────────────────────────────────────────────────┤
+│                                                                 │
+│  pass^k   "What's the probability of ALL k trials succeeding?" │
+│                                                                 │
+│            Example: 2/3 trials pass → pass^3 ≈ 30%              │
+│            (Only 30% chance all 3 would pass)                   │
+│                                                                 │
+├─────────────────────────────────────────────────────────────────┤
+│                                                                 │
+│  Status   PASS  = All trials passed                             │
+│           FLAKY = Some passed, some failed                      │
+│           FAIL  = All trials failed                             │
+│                                                                 │
+└─────────────────────────────────────────────────────────────────┘
+```
+
+## Quick Commands
+
+```bash
+# Run all smoke tests (all models, 3 trials each)
+npm run eval:smoke
+
+# Run single model (use exact model ID for reproducibility)
+npm run eval:smoke -- --model claude-sonnet-4-20250514
+
+# Run single scenario
+npm run eval:smoke -- --scenario 01-create-file
+
+# Quick check (1 trial)
+npm run eval:smoke -- --trials 1
+
+# CI-like run (builds CLI from source, single trial)
+npm run eval:smoke:ci
+
+# View latest results
+cat evals/smoke-tests/results/latest/summary.md
+
+# Debug a failure
+cat evals/smoke-tests/results/latest/<scenario>/<model>/trial-1.log
+ls evals/smoke-tests/results/latest/<scenario>/<model>/workspace-trial-1/
+```
+
+## CI Integration
+
+Smoke tests run automatically on merge to `main` via `.github/workflows/cline-evals-regression.yml`.
+
+**Triggers:**
+- Push to `main` branch (paths: `src/core/**`, `src/shared/**`, `proto/**`)
+- Manual dispatch via `workflow_dispatch`
+
+**What it does:**
+1. Builds the Go CLI from source via `scripts/run-smoke-tests.sh`
+2. Runs all 5 scenarios × 3 models × 1 trial
+3. Uploads results as artifact
+4. Posts summary to GitHub Actions job summary
+
+```bash
+# The CI runs this script which handles proto generation + CLI build:
+bash scripts/run-smoke-tests.sh --trials 1
+```
+
+### Viewing CI Results
+
+1. **Job Summary**: Each run posts results to the Actions tab
+2. **Artifacts**: Full results downloadable as `smoke-test-results-<run_id>`
+
+### Running CI-like Tests Locally
+
+```bash
+# One command - builds CLI from source and runs tests
+npm run eval:smoke:ci
+
+# Or manually:
+npm run protos-go
+cd cli && go build -o cline ./cmd/cline
+export PATH="$(pwd)/cli:$PATH"
+npx tsx evals/smoke-tests/run-smoke-tests.ts --trials 1
+```
+
+### Why Build CLI in CI?
+
+We build the Go CLI from source rather than using a pre-built release because:
+- Tests actual CLI code from the commit (catches CLI regressions)
+- Proto definitions may have changed
+- No dependency on external releases
+
+## Contract Tests (Layer 1)
+
+> **Note**: The old `evals/benchmarks/tool-precision/` tests have been removed. Their functionality is now covered by contract tests in `src/core/**/__tests__/` and the 52 system prompt snapshot tests that run with `npm run test:unit`.
+
+Located in `src/core/api/transform/__tests__/`:
+
+```
+thinking-traces.test.ts
+├── convertToOpenAiMessages preserves reasoning_details
+├── convertToAnthropicMessage preserves thinking blocks
+└── sanitizeGeminiMessages handles provider-specific cleaning
+
+tool-parsing.test.ts
+├── Anthropic tool_use → OpenAI tool_calls conversion
+├── Tool call ID truncation (>40 chars)
+├── OpenAI Responses API ID transformation
+└── Tool result matching
+```
+
+Run with:
+```bash
+npm run test:unit -- --grep "Thinking Trace"  # 9 tests
+npm run test:unit -- --grep "Tool Call"       # 11 tests
+```

evals/README.md

@@ -1,341 +1,149 @@
-# Cline Evaluation System
+# Cline Evaluation Framework
 
-This directory contains the evaluation system for benchmarking Cline against various coding evaluation frameworks.
-
-## Overview
-
-The Cline Evaluation System allows you to:
-
-1. Run Cline against standardized coding benchmarks
-2. Collect comprehensive metrics on performance
-3. Generate detailed reports on evaluation results
-4. Compare performance across different models and benchmarks
-
-## Architecture
-
-The evaluation system consists of two main components:
-
-1. **CLI Tool**: Command-line interface in `evals/cli/` for orchestrating evaluations
-2. **Diff Edit Benchmark**: Separate command using the CLI tool that runs a comprehensive diff editing benchmark suite on real world cases, along with a streamlit dashboard displaying the results. For more details, see the Diff Edit Benchmark [README](./diff-edits/README.md). Make sure you add a `evals/diff-edits/cases` folder with all the conversation jsons.
+A layered testing system for measuring Cline's performance at different levels.
 
 ## Directory Structure
 
 ```
-evals/                            # Main directory for evaluation system
-├── cli/                          # CLI tool for orchestrating evaluations
-│   └── src/
-│       ├── index.ts              # CLI entry point
-│       ├── commands/             # CLI commands (setup, run, report)
-│       ├── adapters/             # Benchmark adapters
-│       ├── db/                   # Database management
-│       └── utils/                # Utility functions
-├── diff-edits/                   # Diff editing evaluation suite
-│   ├── cases/                    # Test case JSON files
-│   ├── results/                  # Evaluation results
-│   ├── diff-apply/               # Diff application logic
-│   ├── parsing/                  # Assistant message parsing
-│   └── prompts/                  # System prompts
-├── repositories/                 # Cloned benchmark repositories
-│   └── exercism/                 # Exercism (Aider Polyglot)
-├── results/                      # Evaluation results storage
-│   ├── runs/                     # Individual run results
-│   └── reports/                  # Generated reports
-└── README.md                     # This file
+evals/
+├── smoke-tests/           # Quick provider validation (minutes)
+│   ├── run-smoke-tests.ts
+│   └── scenarios/         # 5 curated test scenarios
+│
+├── e2e/                   # Full E2E with cline-bench (hours)
+│   └── run-cline-bench.ts
+│
+├── cline-bench/           # Real-world tasks (git submodule)
+│   └── tasks/             # 12 production bug fixes
+│
+├── analysis/              # Metrics and reporting framework
+│   ├── src/
+│   │   ├── metrics.ts     # pass@k, pass^k calculations
+│   │   ├── classifier.ts  # Failure pattern matching
+│   │   └── reporters/     # Markdown, JSON output
+│   └── patterns/
+│       └── cline-failures.yaml
+│
+└── baselines/             # Performance baselines for regression detection
 ```
 
-## Getting Started
+## Test Layers
 
-### Prerequisites
+### Layer 1: Contract Tests (Unit)
 
-- Node.js 16+
-- VSCode with Cline extension installed
-- Git
+Location: `src/core/api/transform/__tests__/`
 
-### Installation
-
-1. Build the CLI tool:
+Tests API transform logic without LLM calls:
+- Thinking trace preservation
+- Tool call parsing (XML, native formats)
+- Provider format conversions
 
 ```bash
-cd evals
-npm install
-npm run build:cli
+npm run test:unit -- --grep "Thinking\|Tool Call"
 ```
 
-### Usage
+### Layer 2: Smoke Tests (Minutes)
 
-#### Setting Up Benchmarks
+Location: `evals/smoke-tests/`
+
+Quick validation across providers with real LLM calls:
+- 5 curated scenarios
+- 3 trials per test for pass@k metrics
+- Runs via cline CLI with `-s` flags
 
 ```bash
-cd evals/cli
-node dist/index.js setup
+# Set API key (Cline provider)
+export CLINE_API_KEY=sk-...
+
+# Run smoke tests
+npm run eval:smoke
+
+# Run specific scenario
+npm run eval:smoke -- --scenario 01-create-file
+
+# Run with specific model (overrides per-scenario models)
+npm run eval:smoke -- --model anthropic/claude-sonnet-4.5
 ```
 
-This will clone and set up all benchmark repositories. You can specify specific benchmarks:
+### Layer 3: E2E Tests (Hours)
+
+Location: `evals/e2e/` + `evals/cline-bench/`
+
+Full agent tests on production-grade tasks:
+- 12 real-world coding problems
+- Docker/Daytona execution via Harbor
+- Nightly CI runs
 
 ```bash
-node dist/index.js setup --benchmarks exercism
+# Prerequisites: Python 3.13, Harbor, Docker
+npm run eval:e2e
+
+# Specific task
+npm run eval:e2e -- --tasks discord
+
+# Different provider
+npm run eval:e2e -- --provider openai --model gpt-4o
 ```
 
-#### Running Evaluations
-
-```bash
-node dist/index.js run --benchmark exercism --count 10
-```
-
-Options:
-- `--benchmark`: Specific benchmark to run (default: exercism)
-- `--count`: Number of tasks to run (default: all available tasks)
-
-**Note:** Model selection is currently configured through the Cline CLI itself, not through evaluation flags.
-
-#### Generating Reports
-
-```bash
-node dist/index.js report
-```
-
-Options:
-- `--format`: Report format (json, markdown) (default: markdown)
-- `--output`: Output path for the report
-
-## Benchmarks
-
-### Exercism
-
-Modified Exercism exercises from the [polyglot-benchmark](https://github.com/Aider-AI/polyglot-benchmark) repository. These are small, focused programming exercises in various languages.
-
-### SWE-Bench (Coming Soon)
-
-Real-world software engineering tasks from the [SWE-bench](https://github.com/SWE-bench/SWE-bench) repository.
-
-### SWELancer (Coming Soon)
-
-Freelance-style programming tasks from the SWELancer benchmark.
-
-### Multi-SWE-Bench (Coming Soon)
-
-Multi-file software engineering tasks from the Multi-SWE-Bench repository.
-
-## Diff Edit Evaluations
-
-The Cline Evaluation System includes a specialized suite for evaluating how well models can make precise edits to files using the `replace_in_file` tool.
-
-### Overview
-
-Diff edit evaluations test a model's ability to:
-
-1. Understand file content and identify specific sections to modify
-2. Generate correct SEARCH/REPLACE blocks for targeted edits
-3. Successfully apply changes without introducing errors
-
-### Directory Structure
-
-```
-diff-edits/
-├── cases/                  # Test case JSON files
-├── results/                # Evaluation results
-├── ClineWrapper.ts         # Wrapper for model interaction
-├── TestRunner.ts           # Main test execution logic
-├── types.ts                # Type definitions
-├── diff-apply/             # Diff application logic
-├── parsing/                # Assistant message parsing
-└── prompts/                # System prompts
-```
-
-### Creating Test Cases
-
-Test cases are defined as JSON files in the `diff-edits/cases/` directory. Each test case should include:
-
-```json
-{
-  "test_id": "example_test_1",
-  "messages": [
-    {
-      "role": "user",
-      "text": "Please fix the bug in this code...",
-      "images": []
-    },
-    {
-      "role": "assistant",
-      "text": "I'll help you fix that bug..."
-    }
-  ],
-  "file_contents": "// Original file content here\nfunction example() {\n  // Code with bug\n}",
-  "file_path": "src/example.js",
-  "system_prompt_details": {
-    "mcp_string": "",
-    "cwd_value": "/path/to/working/directory",
-    "browser_use": false,
-    "width": 900,
-    "height": 600,
-    "os_value": "macOS",
-    "shell_value": "/bin/zsh",
-    "home_value": "/Users/username",
-    "user_custom_instructions": ""
-  },
-  "original_diff_edit_tool_call_message": ""
-}
-```
-
-### Running Diff Edit Evaluations
-
-#### Single Model Evaluation
-
-```bash
-cd evals/cli
-node dist/index.js run-diff-eval --model-ids "anthropic/claude-3-5-sonnet-20241022"
-```
-
-#### Multi-Model Evaluation
-
-Compare multiple models in a single evaluation run:
-
-```bash
-# Compare Claude and Grok models
-node dist/index.js run-diff-eval \
-  --model-ids "anthropic/claude-3-5-sonnet-20241022,x-ai/grok-beta" \
-  --max-cases 10 \
-  --valid-attempts-per-case 3 \
-  --verbose
-
-# Compare multiple Claude variants
-node dist/index.js run-diff-eval \
-  --model-ids "anthropic/claude-3-5-sonnet-20241022,anthropic/claude-3-5-haiku-20241022,anthropic/claude-3-opus-20240229" \
-  --max-cases 5 \
-  --valid-attempts-per-case 2 \
-  --parallel
-```
-
-#### Options
-
-- `--model-ids`: Comma-separated list of model IDs to evaluate (required)
-- `--system-prompt-name`: System prompt to use (default: "basicSystemPrompt")
-- `--valid-attempts-per-case`: Number of attempts per test case per model (default: 1)
-- `--max-cases`: Maximum number of test cases to run (default: all available)
-- `--parsing-function`: Function to parse assistant messages (default: "parseAssistantMessageV2")
-- `--diff-edit-function`: Function to apply diffs (default: "constructNewFileContentV2")
-- `--test-path`: Path to test cases (default: diff-edits/cases)
-- `--thinking-budget`: Tokens allocated for thinking (default: 0)
-- `--parallel`: Run tests in parallel (flag)
-- `--replay`: Use pre-recorded LLM output (flag)
-- `--verbose`: Enable detailed logging (flag)
-
-#### Examples
-
-```bash
-# Quick test with 2 models, 4 cases, 2 attempts each
-node dist/index.js run-diff-eval \
-  --model-ids "anthropic/claude-3-5-sonnet-20241022,x-ai/grok-beta" \
-  --max-cases 4 \
-  --valid-attempts-per-case 2 \
-  --verbose
-
-# Comprehensive evaluation with parallel execution
-node dist/index.js run-diff-eval \
-  --model-ids "anthropic/claude-3-5-sonnet-20241022,anthropic/claude-3-5-haiku-20241022" \
-  --system-prompt-name claude4SystemPrompt \
-  --valid-attempts-per-case 5 \
-  --max-cases 20 \
-  --parallel \
-  --verbose
-```
-
-### Database Storage & Analytics
-
-All evaluation results are automatically stored in a SQLite database (`diff-edits/evals.db`) for advanced analytics and comparison. The database includes:
-
-- **System Prompts**: Versioned system prompt content with hashing for deduplication
-- **Processing Functions**: Versioned parsing and diff-edit function configurations
-- **Files**: Original and edited file content with content-based hashing
-- **Runs**: Evaluation run metadata and configuration
-- **Cases**: Individual test case information with context tokens
-- **Results**: Detailed results with timing, cost, and success metrics
-
-### Interactive Dashboard
-
-Launch the Streamlit dashboard to visualize and analyze evaluation results:
-
-```bash
-cd diff-edits/dashboard
-streamlit run app.py
-```
-
-The dashboard provides:
-
-- **Model Performance Comparison**: Side-by-side comparison of success rates, latency, and costs
-- **Interactive Charts**: Success rate trends, latency vs cost analysis, and performance metrics
-- **Detailed Drill-Down**: Individual result analysis with file content viewing
-- **Run Selection**: Browse and compare different evaluation runs
-- **Real-time Updates**: Automatically refreshes with new evaluation data
-
-#### Dashboard Features
-
-1. **Hero Section**: Overview of current run with key metrics
-2. **Model Cards**: Performance cards with grades and detailed metrics
-3. **Comparison Charts**: Interactive Plotly charts for visual analysis
-4. **Result Explorer**: Detailed view of individual test results including:
-   - Original and edited file content
-   - Raw model output
-   - Parsed tool calls
-   - Timing and cost metrics
-   - Error analysis
-
-#### Quick Start Dashboard
-
-```bash
-# Run a quick evaluation
-node cli/dist/index.js run-diff-eval \
-  --model-ids "anthropic/claude-3-5-sonnet-20241022,x-ai/grok-beta" \
-  --max-cases 4 \
-  --valid-attempts-per-case 2 \
-  --verbose
-
-# Launch dashboard to view results
-cd diff-edits/dashboard && streamlit run app.py
-```
-
-### Legacy Results
-
-For backward compatibility, results are also saved as JSON files in the `diff-edits/results/` directory. The JSON results include:
-- Success/failure status
-- Extracted tool calls
-- Diff edit content
-- Token usage and cost metrics
-
 ## Metrics
 
-The evaluation system collects the following metrics:
+The framework calculates:
 
-- **Token Usage**: Input and output tokens
-- **Cost**: Estimated cost of API calls
-- **Duration**: Time taken to complete tasks
-- **Tool Usage**: Number of tool calls and failures
-- **Success Rate**: Percentage of tasks completed successfully
-- **Test Success Rate**: Percentage of tests passed
-- **Functional Correctness**: Ratio of tests passed to total tests
+| Metric | Formula | Interpretation |
+|--------|---------|----------------|
+| **pass@k** | P(≥1 of k passes) | Solution finding capability |
+| **pass^k** | P(all k pass) | Reliability |
+| **Flakiness** | Entropy of pass rate | Consistency |
 
-## Reports
+With 3 trials:
+- All pass → `pass` (reliable)
+- All fail → `fail` (broken)
+- Mixed → `flaky` (needs investigation)
 
-Reports are generated in Markdown or JSON format and include:
+## CI Integration
 
-- Overall summary
-- Benchmark-specific results
-- Model-specific results
-- Tool usage statistics
-- Charts and visualizations
+- **PR Gate**: Contract tests + smoke tests (fast, ~3min)
+- **Nightly**: E2E tests with cline-bench (not yet implemented, see TODO)
 
-## Development
+## Quick Start
 
-### Adding a New Benchmark
+```bash
+# Run all fast tests
+npm run test:unit
+npm run eval:smoke
 
-1. Create a new adapter in `evals/cli/src/adapters/`
-2. Implement the `BenchmarkAdapter` interface
-3. Register the adapter in `evals/cli/src/adapters/index.ts`
+# Run E2E (requires setup)
+cd evals/cline-bench
+# Follow README.md for Harbor setup
+npm run eval:e2e
+```
 
-### Extending Metrics
+## Adding Tests
 
-To add new metrics:
+### Smoke Test Scenario
 
-1. Update the database schema in `evals/cli/src/db/schema.ts`
-2. Add collection logic in `evals/cli/src/utils/results.ts`
-3. Update report generation in `evals/cli/src/commands/report.ts`
+1. Create `evals/smoke-tests/scenarios/<name>/config.json`
+2. Add optional `template/` directory with starting files
+3. Run to verify: `npm run eval:smoke -- --scenario <name>`
+
+### Contract Test
+
+1. Add to `src/core/api/transform/__tests__/`
+2. Run: `npm run test:unit -- --grep "YourTest"`
+
+### E2E Task
+
+Contribute to [cline/cline-bench](https://github.com/cline/cline-bench)
+
+## Resources
+
+- [cline-bench tasks](evals/cline-bench/README.md)
+- [Smoke test scenarios](evals/smoke-tests/README.md)
+
+## TODO
+
+- [ ] **Nightly E2E CI**: Add scheduled workflow for cline-bench tests
+  - Requires: Docker runner, Harbor setup, ~1-2 hour timeout
+  - Should run on schedule (e.g., nightly) not per-PR
+  - Separate secrets for E2E environment
+- [ ] **Native tool calling smoke tests**: Add CLI support for `native_tool_call_enabled` setting to test Claude 4 with native tools

evals/analysis/package.json

@@ -0,0 +1,34 @@
+{
+	"name": "@cline/analysis",
+	"version": "1.0.0",
+	"description": "Analysis framework for Cline evaluations with failure classification and metrics",
+	"type": "module",
+	"main": "dist/index.js",
+	"types": "dist/index.d.ts",
+	"scripts": {
+		"start": "tsx src/cli.ts",
+		"build": "tsc",
+		"test": "vitest run",
+		"test:watch": "vitest",
+		"test:ui": "vitest --ui"
+	},
+	"keywords": [
+		"cline",
+		"evaluation",
+		"benchmarking",
+		"ai-testing",
+		"metrics"
+	],
+	"dependencies": {
+		"commander": "^12.0.0",
+		"js-yaml": "^4.1.0",
+		"chalk": "^5.0.0"
+	},
+	"devDependencies": {
+		"@types/node": "^20.0.0",
+		"@types/js-yaml": "^4.0.0",
+		"tsx": "^4.0.0",
+		"typescript": "^5.0.0",
+		"vitest": "^1.0.0"
+	}
+}

evals/analysis/patterns/cline-failures.yaml

@@ -0,0 +1,57 @@
+# Cline-specific failure patterns for classification
+# Version 1.0
+
+version: "1.0"
+
+patterns:
+  # Provider-specific bugs (Cline integration issues)
+  - name: "gemini_signature"
+    pattern: "missing.?signature|thoughtSignature"
+    category: "provider_bug"
+    issue: "https://github.com/cline/cline/issues/7974"
+    description: "Gemini 3 Pro requires thoughtSignature for native tool calls"
+
+  - name: "claude_tool_format"
+    pattern: "write_to_file.*missing.*content|content.*parameter.*required"
+    category: "provider_bug"
+    issue: "https://github.com/cline/cline/issues/7998"
+    description: "Claude tool parameter extraction failure"
+
+  # Transient failures (retriable)
+  - name: "rate_limit"
+    pattern: "429|rate.?limit|too.?many.?requests|quota.?exceeded"
+    category: "transient"
+    description: "API rate limiting"
+
+  - name: "network_timeout"
+    pattern: "ECONNREFUSED|ETIMEDOUT|ENOTFOUND|timed.?out"
+    category: "transient"
+    description: "Network connectivity issues"
+
+  - name: "model_overloaded"
+    pattern: "503|service.?unavailable|overloaded"
+    category: "transient"
+    description: "Provider service unavailable"
+
+  # Infrastructure/harness failures
+  - name: "harness_error"
+    pattern: "verifier.*failed|test.*harness.*error|missing.*test.*file"
+    category: "harness"
+    description: "Test harness or verification script failure"
+
+  - name: "environment_failure"
+    pattern: "docker.*failed|container.*exit|OCI.*runtime|pod.*error"
+    category: "environment"
+    description: "Docker/Daytona environment setup failure"
+
+  # Policy/safety failures
+  - name: "safety_refusal"
+    pattern: "content.*policy|safety.*filter|inappropriate.*request"
+    category: "policy"
+    description: "Model refused due to safety/content policy"
+
+  # Auth issues (non-retriable)
+  - name: "auth_error"
+    pattern: "401|unauthorized|invalid.?api.?key"
+    category: "auth"
+    description: "Invalid API credentials"

evals/analysis/src/__tests__/classifier.test.ts

@@ -0,0 +1,180 @@
+import { describe, expect, it } from "vitest"
+import { FailureClassifier } from "../classifier"
+
+describe("FailureClassifier", () => {
+	const classifier = new FailureClassifier()
+
+	describe("Provider Bug Detection", () => {
+		it("detects Gemini signature issue", () => {
+			const logs = `
+				Error: Function call is missing a thought_signature in functionCall parts.
+				This is required for tools to work correctly with Gemini 3 Pro...
+			`
+			const failures = classifier.classify(logs)
+
+			expect(failures.length).toBeGreaterThan(0)
+			expect(failures[0].name).toBe("gemini_signature")
+			expect(failures[0].category).toBe("provider_bug")
+			expect(failures[0].issue_url).toBe("https://github.com/cline/cline/issues/7974")
+		})
+
+		it("detects Claude tool format issue", () => {
+			const logs = `Cline tried to use write_to_file without value for required parameter 'content'`
+			const failures = classifier.classify(logs)
+
+			expect(failures.length).toBeGreaterThan(0)
+			expect(failures[0].name).toBe("claude_tool_format")
+			expect(failures[0].category).toBe("provider_bug")
+			expect(failures[0].issue_url).toBe("https://github.com/cline/cline/issues/7998")
+		})
+	})
+
+	describe("Transient Failure Detection", () => {
+		it("detects rate limiting", () => {
+			const logs = "Error: 429 Too Many Requests - Rate limit exceeded"
+			const failures = classifier.classify(logs)
+
+			expect(failures.length).toBeGreaterThan(0)
+			expect(failures[0].name).toBe("rate_limit")
+			expect(failures[0].category).toBe("transient")
+		})
+
+		it("detects network timeout", () => {
+			const logs = "Error: ETIMEDOUT - Connection timed out"
+			const failures = classifier.classify(logs)
+
+			expect(failures.length).toBeGreaterThan(0)
+			expect(failures[0].name).toBe("network_timeout")
+			expect(failures[0].category).toBe("transient")
+		})
+
+		it("detects service unavailable", () => {
+			const logs = "503 Service Unavailable - Model is currently overloaded"
+			const failures = classifier.classify(logs)
+
+			expect(failures.length).toBeGreaterThan(0)
+			expect(failures[0].name).toBe("model_overloaded")
+			expect(failures[0].category).toBe("transient")
+		})
+	})
+
+	describe("Infrastructure Failure Detection", () => {
+		it("detects harness errors", () => {
+			const logs = "verifier script failed with exit code 1"
+			const failures = classifier.classify(logs)
+
+			expect(failures.length).toBeGreaterThan(0)
+			expect(failures[0].name).toBe("harness_error")
+			expect(failures[0].category).toBe("harness")
+		})
+
+		it("detects environment failures", () => {
+			const logs = "Error: docker container exit code 137"
+			const failures = classifier.classify(logs)
+
+			expect(failures.length).toBeGreaterThan(0)
+			expect(failures[0].name).toBe("environment_failure")
+			expect(failures[0].category).toBe("environment")
+		})
+	})
+
+	describe("Policy and Auth Failures", () => {
+		it("detects safety refusals", () => {
+			const logs = "Request blocked: Content policy violation"
+			const failures = classifier.classify(logs)
+
+			expect(failures.length).toBeGreaterThan(0)
+			expect(failures[0].name).toBe("safety_refusal")
+			expect(failures[0].category).toBe("policy")
+		})
+
+		it("detects auth errors", () => {
+			const logs = "401 Unauthorized: Invalid API key"
+			const failures = classifier.classify(logs)
+
+			expect(failures.length).toBeGreaterThan(0)
+			expect(failures[0].name).toBe("auth_error")
+			expect(failures[0].category).toBe("auth")
+		})
+	})
+
+	describe("Excerpt Extraction", () => {
+		it("extracts context around the matched pattern", () => {
+			const logs = `
+				This is some context before the error.
+				Error: 429 Too Many Requests - Rate limit exceeded
+				This is some context after the error.
+			`
+			const failures = classifier.classify(logs)
+
+			expect(failures[0].excerpt).toContain("Rate limit exceeded")
+			expect(failures[0].excerpt.length).toBeLessThan(500)
+		})
+	})
+
+	describe("Helper Methods", () => {
+		it("hasProviderBug returns true for provider bugs", () => {
+			const logs = "Error: missing thoughtSignature in function call"
+			expect(classifier.hasProviderBug(logs)).toBe(true)
+		})
+
+		it("hasProviderBug returns false for non-provider bugs", () => {
+			const logs = "Error: 429 Too Many Requests"
+			expect(classifier.hasProviderBug(logs)).toBe(false)
+		})
+
+		it("hasTransientFailure returns true for transient errors", () => {
+			const logs = "Error: ETIMEDOUT"
+			expect(classifier.hasTransientFailure(logs)).toBe(true)
+		})
+
+		it("hasTransientFailure returns false for non-transient errors", () => {
+			const logs = "Error: missing thoughtSignature"
+			expect(classifier.hasTransientFailure(logs)).toBe(false)
+		})
+
+		it("getPatternsByCategory returns correct patterns", () => {
+			const providerBugs = classifier.getPatternsByCategory("provider_bug")
+			expect(providerBugs).toContain("gemini_signature")
+			expect(providerBugs).toContain("claude_tool_format")
+
+			const transient = classifier.getPatternsByCategory("transient")
+			expect(transient).toContain("rate_limit")
+			expect(transient).toContain("network_timeout")
+			expect(transient).toContain("model_overloaded")
+		})
+	})
+
+	describe("Multiple Pattern Matching", () => {
+		it("detects multiple failures in same log", () => {
+			const logs = `
+				Error: 429 Too Many Requests
+				Later: Error: ETIMEDOUT
+			`
+			const failures = classifier.classify(logs)
+
+			expect(failures.length).toBe(2)
+			expect(failures.map((f) => f.name)).toContain("rate_limit")
+			expect(failures.map((f) => f.name)).toContain("network_timeout")
+		})
+	})
+
+	describe("Case Insensitivity", () => {
+		it("matches patterns case-insensitively", () => {
+			const logs = "error: RATE LIMIT exceeded"
+			const failures = classifier.classify(logs)
+
+			expect(failures.length).toBeGreaterThan(0)
+			expect(failures[0].name).toBe("rate_limit")
+		})
+	})
+
+	describe("No Match", () => {
+		it("returns empty array when no patterns match", () => {
+			const logs = "Everything completed successfully"
+			const failures = classifier.classify(logs)
+
+			expect(failures).toEqual([])
+		})
+	})
+})

evals/analysis/src/__tests__/metrics.test.ts

@@ -0,0 +1,249 @@
+import { describe, expect, it } from "vitest"
+import { MetricsCalculator } from "../metrics"
+
+describe("MetricsCalculator", () => {
+	const calc = new MetricsCalculator()
+
+	describe("pass@k (solution finding)", () => {
+		it("calculates 100% when at least k trials pass", () => {
+			expect(calc.passAtK([true, true, true], 1)).toBe(1.0)
+			expect(calc.passAtK([true, true, false], 1)).toBe(1.0)
+			expect(calc.passAtK([true, true, true], 3)).toBe(1.0)
+		})
+
+		it("calculates 0% when fewer than k trials pass", () => {
+			expect(calc.passAtK([false, false, false], 1)).toBe(0.0)
+		})
+
+		it("calculates correct probability for mixed results", () => {
+			// With n=3, c=2, k=2: 1 - C(1,2)/C(3,2) = 1 - 0/3 = 1.0
+			expect(calc.passAtK([true, true, false], 2)).toBe(1.0)
+
+			// With n=3, c=1, k=2: 1 - C(2,2)/C(3,2) = 1 - 1/3 = 2/3
+			expect(calc.passAtK([true, false, false], 2)).toBeCloseTo(0.6667, 4)
+		})
+
+		it("throws error when k > n", () => {
+			expect(() => calc.passAtK([true, false], 3)).toThrow()
+		})
+
+		it("handles k=1 correctly (most common case)", () => {
+			expect(calc.passAtK([true, false, false], 1)).toBe(1.0)
+			expect(calc.passAtK([false, false, false], 1)).toBe(0.0)
+		})
+
+		it("handles all-pass scenarios", () => {
+			expect(calc.passAtK([true, true, true, true, true], 3)).toBe(1.0)
+			expect(calc.passAtK([true, true, true, true, true], 5)).toBe(1.0)
+		})
+
+		it("handles all-fail scenarios", () => {
+			expect(calc.passAtK([false, false, false], 1)).toBe(0.0)
+			expect(calc.passAtK([false, false, false], 3)).toBe(0.0)
+		})
+	})
+
+	describe("pass^k (reliability)", () => {
+		it("calculates 100% when all k trials must and do pass", () => {
+			expect(calc.passCaretK([true, true, true], 3)).toBe(1.0)
+			expect(calc.passCaretK([true, true, true, true], 3)).toBeCloseTo(1.0, 4)
+		})
+
+		it("calculates 0% when fewer than k trials pass", () => {
+			expect(calc.passCaretK([true, true, false], 3)).toBe(0.0)
+			expect(calc.passCaretK([true, false, false], 2)).toBe(0.0)
+			expect(calc.passCaretK([false, false, false], 1)).toBe(0.0)
+		})
+
+		it("calculates correct probability for sufficient passes", () => {
+			// With n=4, c=3, k=2: C(3,2)/C(4,2) = 3/6 = 0.5
+			expect(calc.passCaretK([true, true, true, false], 2)).toBeCloseTo(0.5, 4)
+
+			// With n=5, c=3, k=2: C(3,2)/C(5,2) = 3/10 = 0.3
+			expect(calc.passCaretK([true, true, true, false, false], 2)).toBeCloseTo(0.3, 4)
+		})
+
+		it("throws error when k > n", () => {
+			expect(() => calc.passCaretK([true, false], 3)).toThrow()
+		})
+
+		it("diverges from pass@k as trials increase", () => {
+			const trials = [true, true, false, false, false]
+
+			// pass@k increases (eventually finds solution)
+			const passAt1 = calc.passAtK(trials, 1)
+			const passAt3 = calc.passAtK(trials, 3)
+			expect(passAt3).toBeGreaterThanOrEqual(passAt1)
+
+			// pass^k decreases (reliability drops)
+			const passCaret1 = calc.passCaretK(trials, 1)
+			const passCaret3 = calc.passCaretK(trials, 3)
+			expect(passCaret3).toBeLessThanOrEqual(passCaret1)
+		})
+	})
+
+	describe("flakinessScore (variance)", () => {
+		it("returns 0 for all-pass scenarios", () => {
+			expect(calc.flakinessScore([true, true, true])).toBe(0)
+		})
+
+		it("returns 0 for all-fail scenarios", () => {
+			expect(calc.flakinessScore([false, false, false])).toBe(0)
+		})
+
+		it("returns 1 for maximum variance (50% pass rate)", () => {
+			expect(calc.flakinessScore([true, false])).toBe(1)
+			expect(calc.flakinessScore([true, true, false, false])).toBe(1)
+		})
+
+		it("returns values between 0 and 1 for partial variance", () => {
+			const score1 = calc.flakinessScore([true, true, true, false])
+			expect(score1).toBeGreaterThan(0)
+			expect(score1).toBeLessThan(1)
+
+			const score2 = calc.flakinessScore([true, false, false, false])
+			expect(score2).toBeGreaterThan(0)
+			expect(score2).toBeLessThan(1)
+		})
+
+		it("symmetric around 50% pass rate", () => {
+			const score25 = calc.flakinessScore([true, false, false, false])
+			const score75 = calc.flakinessScore([true, true, true, false])
+			expect(score25).toBeCloseTo(score75, 4)
+		})
+
+		it("higher variance for rates closer to 50%", () => {
+			const score25 = calc.flakinessScore([true, false, false, false])
+			const score50 = calc.flakinessScore([true, true, false, false])
+			expect(score50).toBeGreaterThan(score25)
+		})
+	})
+
+	describe("binomial coefficient", () => {
+		it("calculates C(n, 0) = 1", () => {
+			expect(calc["binomial"](5, 0)).toBe(1)
+		})
+
+		it("calculates C(n, n) = 1", () => {
+			expect(calc["binomial"](5, 5)).toBe(1)
+		})
+
+		it("calculates C(n, 1) = n", () => {
+			expect(calc["binomial"](5, 1)).toBe(5)
+		})
+
+		it("calculates C(n, k) correctly", () => {
+			expect(calc["binomial"](5, 2)).toBe(10)
+			expect(calc["binomial"](6, 3)).toBe(20)
+			expect(calc["binomial"](10, 3)).toBe(120)
+		})
+
+		it("returns 0 when k > n", () => {
+			expect(calc["binomial"](3, 5)).toBe(0)
+		})
+
+		it("optimizes by using smaller k", () => {
+			// C(10, 8) = C(10, 2) = 45
+			expect(calc["binomial"](10, 8)).toBe(45)
+			expect(calc["binomial"](10, 2)).toBe(45)
+		})
+	})
+
+	describe("calculateTaskMetrics", () => {
+		it("calculates all metrics for 3 trials", () => {
+			const metrics = calc.calculateTaskMetrics([true, true, false])
+
+			expect(metrics.passAt1).toBe(1.0)
+			expect(metrics.passAt3).toBeGreaterThan(0)
+			expect(metrics.passCaret3).toBe(0.0)
+			expect(metrics.flakinessScore).toBeGreaterThan(0)
+		})
+
+		it("calculates all metrics for perfect pass", () => {
+			const metrics = calc.calculateTaskMetrics([true, true, true])
+
+			expect(metrics.passAt1).toBe(1.0)
+			expect(metrics.passAt3).toBe(1.0)
+			expect(metrics.passCaret3).toBe(1.0)
+			expect(metrics.flakinessScore).toBe(0)
+		})
+
+		it("calculates all metrics for perfect fail", () => {
+			const metrics = calc.calculateTaskMetrics([false, false, false])
+
+			expect(metrics.passAt1).toBe(0.0)
+			expect(metrics.passAt3).toBe(0.0)
+			expect(metrics.passCaret3).toBe(0.0)
+			expect(metrics.flakinessScore).toBe(0)
+		})
+
+		it("throws error for empty trials", () => {
+			expect(() => calc.calculateTaskMetrics([])).toThrow()
+		})
+
+		it("handles fewer than 3 trials gracefully", () => {
+			const metrics = calc.calculateTaskMetrics([true, false])
+
+			expect(metrics.passAt1).toBe(1.0)
+			expect(metrics.passAt3).toBe(0) // Not enough trials
+			expect(metrics.passCaret3).toBe(0)
+			expect(metrics.flakinessScore).toBe(1)
+		})
+	})
+
+	describe("getTaskStatus", () => {
+		it("returns 'pass' when all trials pass", () => {
+			expect(calc.getTaskStatus([true, true, true])).toBe("pass")
+		})
+
+		it("returns 'fail' when all trials fail", () => {
+			expect(calc.getTaskStatus([false, false, false])).toBe("fail")
+		})
+
+		it("returns 'flaky' when some trials pass and some fail", () => {
+			expect(calc.getTaskStatus([true, false, false])).toBe("flaky")
+			expect(calc.getTaskStatus([true, true, false])).toBe("flaky")
+		})
+
+		it("handles single trial", () => {
+			expect(calc.getTaskStatus([true])).toBe("pass")
+			expect(calc.getTaskStatus([false])).toBe("fail")
+		})
+	})
+
+	describe("Real-world scenarios", () => {
+		it("handles typical cline-bench results", () => {
+			// Scenario: Task passed 2/3 times
+			const trials = [true, true, false]
+			const metrics = calc.calculateTaskMetrics(trials)
+
+			expect(metrics.passAt1).toBe(1.0) // Found solution
+			expect(metrics.passAt3).toBeGreaterThan(0.5) // Likely to solve
+			expect(metrics.passCaret3).toBe(0) // Not reliable
+			expect(metrics.flakinessScore).toBeGreaterThan(0) // Has variance
+			expect(calc.getTaskStatus(trials)).toBe("flaky")
+		})
+
+		it("handles consistent success", () => {
+			const trials = [true, true, true]
+			const metrics = calc.calculateTaskMetrics(trials)
+
+			expect(metrics.passAt1).toBe(1.0)
+			expect(metrics.passAt3).toBe(1.0)
+			expect(metrics.passCaret3).toBe(1.0)
+			expect(metrics.flakinessScore).toBe(0)
+			expect(calc.getTaskStatus(trials)).toBe("pass")
+		})
+
+		it("handles consistent failure", () => {
+			const trials = [false, false, false]
+			const metrics = calc.calculateTaskMetrics(trials)
+
+			expect(metrics.passAt1).toBe(0)
+			expect(metrics.passAt3).toBe(0)
+			expect(metrics.passCaret3).toBe(0)
+			expect(metrics.flakinessScore).toBe(0)
+			expect(calc.getTaskStatus(trials)).toBe("fail")
+		})
+	})
+})

evals/analysis/src/classifier.ts

@@ -0,0 +1,124 @@
+/**
+ * Failure classification system for Cline evaluations
+ *
+ * Classifies failures by matching log patterns against known issues:
+ * - Provider bugs (Gemini #7974, Claude #7998)
+ * - Transient failures (rate limits, timeouts)
+ * - Infrastructure issues (harness, environment)
+ * - Policy/safety refusals
+ * - Auth errors
+ */
+
+import * as fs from "fs"
+import * as yaml from "js-yaml"
+import * as path from "path"
+import type { FailureCategory, FailureInfo } from "./schemas"
+
+export interface FailurePattern {
+	name: string
+	pattern: string // Regex pattern as string
+	category: FailureCategory
+	issue?: string // GitHub issue URL
+	description: string
+}
+
+export interface FailurePatternsConfig {
+	version: string
+	patterns: FailurePattern[]
+}
+
+export class FailureClassifier {
+	private patterns: Array<FailurePattern & { regex: RegExp }>
+
+	constructor(patternsPath?: string) {
+		const defaultPath = path.join(__dirname, "../patterns/cline-failures.yaml")
+		const configPath = patternsPath || defaultPath
+
+		const config = this.loadPatternsFromYaml(configPath)
+		this.patterns = config.patterns.map((p) => ({
+			...p,
+			regex: new RegExp(p.pattern, "i"), // Case-insensitive matching
+		}))
+	}
+
+	private loadPatternsFromYaml(filePath: string): FailurePatternsConfig {
+		const content = fs.readFileSync(filePath, "utf-8")
+		const config = yaml.load(content) as FailurePatternsConfig
+
+		if (!config.version || !config.patterns) {
+			throw new Error("Invalid patterns YAML: missing version or patterns")
+		}
+
+		return config
+	}
+
+	/**
+	 * Classify failures in log text
+	 * @param logs Full log text (e.g., cline.txt content)
+	 * @returns Array of matched failure categories with excerpts
+	 */
+	classify(logs: string): FailureInfo[] {
+		const failures: FailureInfo[] = []
+
+		for (const pattern of this.patterns) {
+			const match = pattern.regex.exec(logs)
+			if (match) {
+				failures.push({
+					name: pattern.name,
+					category: pattern.category,
+					excerpt: this.extractExcerpt(logs, match.index, match[0].length),
+					issue_url: pattern.issue,
+				})
+			}
+		}
+
+		return failures
+	}
+
+	/**
+	 * Extract a context snippet around the matched pattern
+	 * @param logs Full log text
+	 * @param matchIndex Index where pattern matched
+	 * @param matchLength Length of the matched text
+	 * @returns Context snippet (up to 200 chars before/after match)
+	 */
+	private extractExcerpt(logs: string, matchIndex: number, matchLength: number): string {
+		const contextSize = 200
+		const start = Math.max(0, matchIndex - contextSize)
+		const end = Math.min(logs.length, matchIndex + matchLength + contextSize)
+
+		let excerpt = logs.slice(start, end)
+
+		// Trim to complete lines for readability
+		excerpt = excerpt.replace(/^\s*\S*\s*/, "") // Remove partial first line
+		excerpt = excerpt.replace(/\s*\S*\s*$/, "") // Remove partial last line
+
+		// Truncate if still too long
+		if (excerpt.length > 400) {
+			excerpt = excerpt.slice(0, 400) + "..."
+		}
+
+		return excerpt.trim()
+	}
+
+	/**
+	 * Check if logs contain any known provider bug patterns
+	 */
+	hasProviderBug(logs: string): boolean {
+		return this.classify(logs).some((f) => f.category === "provider_bug")
+	}
+
+	/**
+	 * Check if logs contain transient failure patterns (retriable)
+	 */
+	hasTransientFailure(logs: string): boolean {
+		return this.classify(logs).some((f) => f.category === "transient")
+	}
+
+	/**
+	 * Get all pattern names for a specific category
+	 */
+	getPatternsByCategory(category: FailureCategory): string[] {
+		return this.patterns.filter((p) => p.category === category).map((p) => p.name)
+	}
+}

evals/analysis/src/cli.ts

@@ -0,0 +1,240 @@
+#!/usr/bin/env node
+
+/**
+ * Cline Analysis Framework CLI
+ *
+ * Commands:
+ * - analyze: Parse Harbor job output and generate reports
+ * - compare: Compare baseline vs current results for regression detection
+ */
+
+import chalk from "chalk"
+import { Command } from "commander"
+import * as fs from "fs"
+import { HarborParser } from "./parsers"
+import { JsonReporter, MarkdownReporter } from "./reporters"
+import type { AnalysisOutputV1, ComparisonResult } from "./schemas"
+
+const program = new Command()
+
+program.name("cline-analysis").description("Analysis framework for Cline evaluations").version("1.0.0")
+
+// Analyze command
+program
+	.command("analyze <job-dir>")
+	.description("Parse Harbor job output and generate analysis report")
+	.option("-f, --format <format>", "Output format: markdown, json, or minimal", "markdown")
+	.option("-o, --output <file>", "Write report to file (default: stdout)")
+	.option("--no-color", "Disable colored output")
+	.action(async (jobDir: string, options: any) => {
+		try {
+			// Validate job directory
+			if (!fs.existsSync(jobDir)) {
+				console.error(chalk.red(`Error: Job directory not found: ${jobDir}`))
+				process.exit(1)
+			}
+
+			console.error(chalk.blue(`Analyzing Harbor job: ${jobDir}`))
+
+			// Parse Harbor output
+			const parser = new HarborParser()
+			const analysis = parser.parseJob(jobDir)
+
+			// Generate report
+			let report: string
+
+			if (options.format === "json") {
+				const jsonReporter = new JsonReporter()
+				report = jsonReporter.generate(analysis, true)
+			} else if (options.format === "minimal") {
+				const jsonReporter = new JsonReporter()
+				report = jsonReporter.generateMinimal(analysis)
+			} else {
+				const markdownReporter = new MarkdownReporter()
+				report = markdownReporter.generate(analysis, options.color)
+			}
+
+			// Output report
+			if (options.output) {
+				fs.writeFileSync(options.output, report)
+				console.error(chalk.green(`✓ Report written to: ${options.output}`))
+
+				// Also write full JSON for future reference
+				if (options.format === "markdown") {
+					const jsonPath = options.output.replace(/\.md$/, ".json")
+					const jsonReporter = new JsonReporter()
+					fs.writeFileSync(jsonPath, jsonReporter.generate(analysis, true))
+					console.error(chalk.gray(`  (Full JSON saved to: ${jsonPath})`))
+				}
+			} else {
+				console.log(report)
+			}
+
+			// Summary on stderr
+			const markdownReporter = new MarkdownReporter()
+			const summary = markdownReporter.generateCompactSummary(analysis)
+			console.error("\n" + chalk.bold("Summary:"))
+			console.error(summary)
+		} catch (error) {
+			console.error(chalk.red("Error during analysis:"))
+			console.error(error)
+			process.exit(1)
+		}
+	})
+
+// Compare command
+program
+	.command("compare <baseline> <current>")
+	.description("Compare baseline and current analysis results")
+	.option("-t, --threshold <number>", "Regression threshold (percentage points)", "10")
+	.option("--no-color", "Disable colored output")
+	.action(async (baselinePath: string, currentPath: string, options: any) => {
+		try {
+			// Load both analysis outputs
+			if (!fs.existsSync(baselinePath)) {
+				console.error(chalk.red(`Error: Baseline file not found: ${baselinePath}`))
+				process.exit(1)
+			}
+
+			if (!fs.existsSync(currentPath)) {
+				console.error(chalk.red(`Error: Current file not found: ${currentPath}`))
+				process.exit(1)
+			}
+
+			const baseline: AnalysisOutputV1 = JSON.parse(fs.readFileSync(baselinePath, "utf-8"))
+			const current: AnalysisOutputV1 = JSON.parse(fs.readFileSync(currentPath, "utf-8"))
+
+			const threshold = parseFloat(options.threshold)
+
+			// Compare results
+			const comparison = compareAnalyses(baseline, current, threshold)
+
+			// Display comparison
+			displayComparison(comparison, options.color)
+
+			// Exit with error if regression detected
+			if (comparison.regression_detected) {
+				console.error(chalk.red("\n✗ Regression detected! See details above."))
+				process.exit(1)
+			} else {
+				console.error(chalk.green("\n✓ No significant regression detected."))
+				process.exit(0)
+			}
+		} catch (error) {
+			console.error(chalk.red("Error during comparison:"))
+			console.error(error)
+			process.exit(1)
+		}
+	})
+
+program.parse()
+
+/**
+ * Compare two analysis outputs for regression detection
+ */
+function compareAnalyses(baseline: AnalysisOutputV1, current: AnalysisOutputV1, threshold: number): ComparisonResult {
+	const delta = {
+		pass_at_1: (current.summary.pass_at_1 - baseline.summary.pass_at_1) * 100,
+		pass_at_3: (current.summary.pass_at_3 - baseline.summary.pass_at_3) * 100,
+		pass_caret_3: (current.summary.pass_caret_3 - baseline.summary.pass_caret_3) * 100,
+		cost_usd: current.summary.total_cost_usd - baseline.summary.total_cost_usd,
+		duration_sec: current.summary.total_duration_sec - baseline.summary.total_duration_sec,
+	}
+
+	// Detect regression (drop in pass rates exceeding threshold)
+	const regression_detected = delta.pass_at_1 < -threshold || delta.pass_at_3 < -threshold
+
+	// Find tasks that regressed or improved
+	const tasks_regressed: string[] = []
+	const tasks_improved: string[] = []
+
+	const baselineTaskMap = new Map(baseline.tasks.map((t) => [t.task_id, t]))
+
+	for (const currentTask of current.tasks) {
+		const baselineTask = baselineTaskMap.get(currentTask.task_id)
+		if (!baselineTask) {
+			continue
+		}
+
+		const taskDelta = (currentTask.metrics.pass_at_3 - baselineTask.metrics.pass_at_3) * 100
+
+		if (taskDelta < -threshold) {
+			tasks_regressed.push(currentTask.task_name)
+		} else if (taskDelta > threshold) {
+			tasks_improved.push(currentTask.task_name)
+		}
+	}
+
+	return {
+		baseline: baseline.summary,
+		current: current.summary,
+		delta,
+		regression_detected,
+		tasks_regressed,
+		tasks_improved,
+	}
+}
+
+/**
+ * Display comparison results with color coding
+ */
+function displayComparison(comparison: ComparisonResult, useColor: boolean): void {
+	const separator = "━".repeat(79)
+
+	console.log(useColor ? chalk.bold(separator) : separator)
+	console.log(useColor ? chalk.bold.cyan("Baseline vs Current Comparison") : "Baseline vs Current Comparison")
+	console.log(useColor ? chalk.bold(separator) : separator)
+	console.log("")
+
+	// Pass rate changes
+	console.log(useColor ? chalk.bold("Pass Rate Changes:") : "Pass Rate Changes:")
+	console.log(`  pass@1: ${formatDelta(comparison.delta.pass_at_1, useColor)} percentage points`)
+	console.log(`  pass@3: ${formatDelta(comparison.delta.pass_at_3, useColor)} percentage points`)
+	console.log(`  pass^3: ${formatDelta(comparison.delta.pass_caret_3, useColor)} percentage points`)
+	console.log("")
+
+	// Cost and duration changes
+	console.log(useColor ? chalk.bold("Resource Changes:") : "Resource Changes:")
+	console.log(`  Cost: ${formatDelta(comparison.delta.cost_usd, useColor, true)} USD`)
+	console.log(`  Duration: ${formatDelta(comparison.delta.duration_sec, useColor)} seconds`)
+	console.log("")
+
+	// Tasks regressed
+	if (comparison.tasks_regressed.length > 0) {
+		console.log(useColor ? chalk.bold.red("Tasks Regressed:") : "Tasks Regressed:")
+		for (const task of comparison.tasks_regressed) {
+			console.log(`  • ${task}`)
+		}
+		console.log("")
+	}
+
+	// Tasks improved
+	if (comparison.tasks_improved.length > 0) {
+		console.log(useColor ? chalk.bold.green("Tasks Improved:") : "Tasks Improved:")
+		for (const task of comparison.tasks_improved) {
+			console.log(`  • ${task}`)
+		}
+		console.log("")
+	}
+}
+
+/**
+ * Format delta value with color coding
+ */
+function formatDelta(value: number, useColor: boolean, invertSign = false): string {
+	const sign = invertSign ? -Math.sign(value) : Math.sign(value)
+	const absValue = Math.abs(value).toFixed(2)
+	const signStr = sign > 0 ? "+" : sign < 0 ? "-" : " "
+
+	if (!useColor) {
+		return `${signStr}${absValue}`
+	}
+
+	if (sign > 0) {
+		return chalk.green(`${signStr}${absValue}`)
+	}
+	if (sign < 0) {
+		return chalk.red(`${signStr}${absValue}`)
+	}
+	return chalk.gray(`${signStr}${absValue}`)
+}

evals/analysis/src/metrics.ts

@@ -0,0 +1,186 @@
+/**
+ * Metrics calculation for nondeterministic AI testing
+ *
+ * Implements:
+ * - pass@k: P(at least 1 of k trials passes) - solution finding capability
+ * - pass^k: P(all k trials pass) - reliability measure
+ * - Flakiness score: Entropy-based variance measurement
+ *
+ * References:
+ * - HumanEval paper: https://arxiv.org/abs/2107.03374
+ * - pass@k methodology: https://github.com/openai/human-eval
+ */
+
+export class MetricsCalculator {
+	/**
+	 * Calculate pass@k: Probability that at least 1 of k trials succeeds
+	 *
+	 * Formula: 1 - C(n-c, k) / C(n, k)
+	 * where n = total trials, c = number of passes, k = sample size
+	 *
+	 * Interpretation: "Can this model solve the problem?"
+	 *
+	 * @param trials Array of boolean trial results (true = pass, false = fail)
+	 * @param k Number of trials to sample
+	 * @returns Probability [0, 1]
+	 */
+	passAtK(trials: boolean[], k: number): number {
+		const n = trials.length
+		const c = trials.filter(Boolean).length
+
+		if (n < k) {
+			throw new Error(`Cannot calculate pass@${k} with only ${n} trials`)
+		}
+
+		// If we have at least k passes, probability is 100%
+		if (c >= k) {
+			return 1.0
+		}
+
+		// Calculate: 1 - C(n-c, k) / C(n, k)
+		const numerator = this.binomial(n - c, k)
+		const denominator = this.binomial(n, k)
+
+		return 1 - numerator / denominator
+	}
+
+	/**
+	 * Calculate pass^k: Probability that ALL k trials succeed
+	 *
+	 * Formula: C(c, k) / C(n, k)
+	 * where n = total trials, c = number of passes, k = sample size
+	 *
+	 * Interpretation: "Can I rely on this model?" (reliability metric)
+	 *
+	 * @param trials Array of boolean trial results
+	 * @param k Number of trials that must all pass
+	 * @returns Probability [0, 1]
+	 */
+	passCaretK(trials: boolean[], k: number): number {
+		const n = trials.length
+		const c = trials.filter(Boolean).length
+
+		if (n < k) {
+			throw new Error(`Cannot calculate pass^${k} with only ${n} trials`)
+		}
+
+		// If we have fewer than k passes, probability is 0%
+		if (c < k) {
+			return 0.0
+		}
+
+		// Calculate: C(c, k) / C(n, k)
+		const numerator = this.binomial(c, k)
+		const denominator = this.binomial(n, k)
+
+		return numerator / denominator
+	}
+
+	/**
+	 * Calculate flakiness score: Entropy-based measure of variance
+	 *
+	 * Formula: -p*log2(p) - (1-p)*log2(1-p)
+	 * where p = pass rate
+	 *
+	 * Returns:
+	 * - 0.0: No variance (all pass or all fail)
+	 * - 1.0: Maximum variance (50% pass rate)
+	 *
+	 * Interpretation: How unpredictable/inconsistent is this task?
+	 *
+	 * @param trials Array of boolean trial results
+	 * @returns Flakiness score [0, 1]
+	 */
+	flakinessScore(trials: boolean[]): number {
+		const passRate = trials.filter(Boolean).length / trials.length
+
+		// No variance if all pass or all fail
+		if (passRate === 0 || passRate === 1) {
+			return 0
+		}
+
+		// Binary entropy
+		const entropy = -passRate * Math.log2(passRate) - (1 - passRate) * Math.log2(1 - passRate)
+
+		return entropy // Already in [0, 1] range
+	}
+
+	/**
+	 * Binomial coefficient C(n, k) = n! / (k! * (n-k)!)
+	 *
+	 * Uses iterative calculation to avoid factorial overflow
+	 *
+	 * @param n Total items
+	 * @param k Items to choose
+	 * @returns Number of ways to choose k items from n
+	 */
+	private binomial(n: number, k: number): number {
+		if (k > n) {
+			return 0
+		}
+		if (k === 0 || k === n) {
+			return 1
+		}
+
+		// Optimize by using smaller k
+		if (k > n - k) {
+			k = n - k
+		}
+
+		let result = 1
+		for (let i = 1; i <= k; i++) {
+			result *= n - i + 1
+			result /= i
+		}
+
+		return result
+	}
+
+	/**
+	 * Calculate all metrics for a task's trials
+	 *
+	 * @param trials Array of boolean trial results
+	 * @returns Object with pass@1, pass@3, pass^3, and flakiness scores
+	 */
+	calculateTaskMetrics(trials: boolean[]): {
+		passAt1: number
+		passAt3: number
+		passCaret3: number
+		flakinessScore: number
+	} {
+		if (trials.length === 0) {
+			throw new Error("Cannot calculate metrics with no trials")
+		}
+
+		// Calculate pass@k and pass^k for available trials
+		const passAt1 = trials.length >= 1 ? this.passAtK(trials, 1) : 0
+		const passAt3 = trials.length >= 3 ? this.passAtK(trials, 3) : 0
+		const passCaret3 = trials.length >= 3 ? this.passCaretK(trials, 3) : 0
+
+		return {
+			passAt1,
+			passAt3,
+			passCaret3,
+			flakinessScore: this.flakinessScore(trials),
+		}
+	}
+
+	/**
+	 * Determine task status based on trial results
+	 *
+	 * @param trials Array of boolean trial results
+	 * @returns "pass" | "fail" | "flaky"
+	 */
+	getTaskStatus(trials: boolean[]): "pass" | "fail" | "flaky" {
+		const passCount = trials.filter(Boolean).length
+		const totalCount = trials.length
+
+		if (passCount === totalCount) {
+			return "pass"
+		}
+		if (passCount === 0) {
+			return "fail"
+		}
+		return "flaky"
+	}
+}

evals/analysis/src/parsers/harbor.ts

@@ -0,0 +1,301 @@
+/**
+ * Parser for Harbor framework job output
+ *
+ * Parses jobs/ directory structure created by Harbor to extract:
+ * - Trial results (pass/fail, duration, cost, tokens)
+ * - Task groupings and metrics
+ * - Failure classifications
+ */
+
+import * as fs from "fs"
+import * as path from "path"
+import { FailureClassifier } from "../classifier"
+import { MetricsCalculator } from "../metrics"
+import type {
+	AnalysisMetadata,
+	AnalysisOutputV1,
+	AnalysisSummary,
+	FailureAnalysis,
+	TaskResultV1,
+	TrialResultV1,
+} from "../schemas"
+
+export interface HarborParserOptions {
+	patternsPath?: string
+}
+
+export class HarborParser {
+	private classifier: FailureClassifier
+	private metrics: MetricsCalculator
+
+	constructor(options: HarborParserOptions = {}) {
+		this.classifier = new FailureClassifier(options.patternsPath)
+		this.metrics = new MetricsCalculator()
+	}
+
+	/**
+	 * Parse a complete Harbor job directory
+	 *
+	 * @param jobDir Path to job directory (e.g., jobs/2025-01-25__10-30-00/)
+	 * @returns Structured analysis output with schema version 1.0
+	 */
+	parseJob(jobDir: string): AnalysisOutputV1 {
+		const configPath = path.join(jobDir, "config.json")
+		const resultPath = path.join(jobDir, "result.json")
+
+		if (!fs.existsSync(configPath) || !fs.existsSync(resultPath)) {
+			throw new Error(`Invalid Harbor job directory: ${jobDir}`)
+		}
+
+		const config = JSON.parse(fs.readFileSync(configPath, "utf-8"))
+		const result = JSON.parse(fs.readFileSync(resultPath, "utf-8"))
+
+		// Find all trial directories
+		const trialDirs = this.findTrialDirectories(jobDir)
+		const trials = trialDirs.map((dir) => this.parseTrialDirectory(dir))
+
+		// Group trials by task ID
+		const taskResults = this.groupTrialsByTask(trials)
+
+		// Calculate aggregate metrics
+		const summary = this.calculateSummary(taskResults)
+
+		// Analyze failures
+		const failures = this.analyzeFailures(taskResults)
+
+		const metadata: AnalysisMetadata = {
+			generated_at: new Date().toISOString(),
+			analysis_version: "1.0.0", // TODO: Get from package.json
+			job_id: path.basename(jobDir),
+			model: config.model,
+			agent: config.agent || "cline-cli",
+			environment: config.environment || "docker",
+		}
+
+		return {
+			schema_version: "1.0",
+			metadata,
+			summary,
+			tasks: taskResults,
+			failures,
+		}
+	}
+
+	/**
+	 * Find all trial directories in a job
+	 */
+	private findTrialDirectories(jobDir: string): string[] {
+		const entries = fs.readdirSync(jobDir, { withFileTypes: true })
+
+		return entries
+			.filter((entry) => entry.isDirectory())
+			.filter((entry) => {
+				// Trial dirs have format: 01k7a12s...disco__fhSEuhr
+				const configExists = fs.existsSync(path.join(jobDir, entry.name, "config.json"))
+				return configExists
+			})
+			.map((entry) => path.join(jobDir, entry.name))
+	}
+
+	/**
+	 * Parse a single trial directory
+	 */
+	private parseTrialDirectory(trialDir: string): ParsedTrial {
+		const configPath = path.join(trialDir, "config.json")
+		const resultPath = path.join(trialDir, "result.json")
+		const rewardPath = path.join(trialDir, "verifier", "reward.txt")
+		const logsPath = path.join(trialDir, "agent", "cline.txt")
+		const testOutputPath = path.join(trialDir, "verifier", "test-stdout.txt")
+
+		const config = JSON.parse(fs.readFileSync(configPath, "utf-8"))
+		const result = JSON.parse(fs.readFileSync(resultPath, "utf-8"))
+		const reward = fs.readFileSync(rewardPath, "utf-8").trim()
+		const logs = fs.existsSync(logsPath) ? fs.readFileSync(logsPath, "utf-8") : ""
+		const testOutput = fs.existsSync(testOutputPath) ? fs.readFileSync(testOutputPath, "utf-8") : ""
+
+		const passed = reward === "1"
+		const failures = passed ? [] : this.classifier.classify(logs)
+
+		return {
+			taskId: config.task_id,
+			trialHash: path.basename(trialDir).split("__")[1] || "",
+			passed,
+			duration: result.duration_sec || 0,
+			cost: result.cost_usd || 0,
+			tokensIn: result.tokens_in,
+			tokensOut: result.tokens_out,
+			logs,
+			testOutput,
+			failures,
+		}
+	}
+
+	/**
+	 * Group trials by task ID and calculate metrics
+	 */
+	private groupTrialsByTask(trials: ParsedTrial[]): TaskResultV1[] {
+		const taskMap = new Map<string, ParsedTrial[]>()
+
+		// Group trials by task ID
+		for (const trial of trials) {
+			const existing = taskMap.get(trial.taskId) || []
+			existing.push(trial)
+			taskMap.set(trial.taskId, existing)
+		}
+
+		// Convert to TaskResultV1 format
+		const taskResults: TaskResultV1[] = []
+
+		for (const [taskId, taskTrials] of taskMap.entries()) {
+			const trialResults: TrialResultV1[] = taskTrials.map((trial, index) => ({
+				trial_index: index,
+				trial_hash: trial.trialHash,
+				passed: trial.passed,
+				duration_sec: trial.duration,
+				cost_usd: trial.cost,
+				tokens_in: trial.tokensIn,
+				tokens_out: trial.tokensOut,
+				failures: trial.failures,
+			}))
+
+			const passResults = taskTrials.map((t) => t.passed)
+			const metrics = this.metrics.calculateTaskMetrics(passResults)
+			const status = this.metrics.getTaskStatus(passResults)
+
+			const totalCost = taskTrials.reduce((sum, t) => sum + t.cost, 0)
+			const avgDuration = taskTrials.reduce((sum, t) => sum + t.duration, 0) / taskTrials.length
+
+			// Extract readable task name from ID
+			const taskName = this.extractTaskName(taskId)
+
+			taskResults.push({
+				task_id: taskId,
+				task_name: taskName,
+				trials: trialResults,
+				metrics,
+				status,
+				total_cost_usd: totalCost,
+				avg_duration_sec: avgDuration,
+			})
+		}
+
+		return taskResults.sort((a, b) => a.task_name.localeCompare(b.task_name))
+	}
+
+	/**
+	 * Extract human-readable task name from task ID
+	 * Example: 01k7a12sd1nk15j08e6x0x7v9e-discord-trivia-approval-keyerror → discord-trivia
+	 */
+	private extractTaskName(taskId: string): string {
+		const parts = taskId.split("-")
+		if (parts.length > 1) {
+			// Remove the ID prefix and get first 2-3 meaningful words
+			const words = parts.slice(1, 4)
+			return words.join("-")
+		}
+		return taskId
+	}
+
+	/**
+	 * Calculate aggregate summary metrics
+	 */
+	private calculateSummary(taskResults: TaskResultV1[]): AnalysisSummary {
+		const totalTasks = taskResults.length
+		const totalTrials = taskResults.reduce((sum, task) => sum + task.trials.length, 0)
+
+		// Calculate overall pass@k metrics
+		const allTrials = taskResults.flatMap((task) => task.trials.map((t) => t.passed))
+		let passAt1 = 0
+		let passAt3 = 0
+		let passCaret3 = 0
+
+		if (allTrials.length >= 1) {
+			passAt1 = this.metrics.passAtK(allTrials, 1)
+		}
+		if (allTrials.length >= 3) {
+			passAt3 = this.metrics.passAtK(allTrials, 3)
+			passCaret3 = this.metrics.passCaretK(allTrials, 3)
+		}
+
+		const totalCost = taskResults.reduce((sum, task) => sum + task.total_cost_usd, 0)
+		const totalDuration = taskResults.reduce((sum, task) => sum + task.avg_duration_sec * task.trials.length, 0)
+
+		const flakyTaskCount = taskResults.filter((task) => task.status === "flaky").length
+
+		return {
+			total_tasks: totalTasks,
+			total_trials: totalTrials,
+			pass_at_1: passAt1,
+			pass_at_3: passAt3,
+			pass_caret_3: passCaret3,
+			total_cost_usd: totalCost,
+			total_duration_sec: totalDuration,
+			flaky_task_count: flakyTaskCount,
+		}
+	}
+
+	/**
+	 * Analyze failure patterns across all tasks
+	 */
+	private analyzeFailures(taskResults: TaskResultV1[]): FailureAnalysis {
+		const categoryCount = new Map<string, number>()
+		const patternCount = new Map<string, { count: number; issue_url?: string; examples: any[] }>()
+
+		for (const task of taskResults) {
+			for (const trial of task.trials) {
+				if (!trial.passed) {
+					for (const failure of trial.failures) {
+						// Count by category
+						categoryCount.set(failure.category, (categoryCount.get(failure.category) || 0) + 1)
+
+						// Count by pattern
+						const existing = patternCount.get(failure.name) || {
+							count: 0,
+							issue_url: failure.issue_url,
+							examples: [],
+						}
+						existing.count++
+
+						// Add example if not too many
+						if (existing.examples.length < 3) {
+							existing.examples.push({
+								task_id: task.task_id,
+								trial_index: trial.trial_index,
+								excerpt: failure.excerpt,
+							})
+						}
+
+						patternCount.set(failure.name, existing)
+					}
+				}
+			}
+		}
+
+		const byCategory: Record<string, number> = {}
+		for (const [category, count] of categoryCount.entries()) {
+			byCategory[category] = count
+		}
+
+		const byPattern = Array.from(patternCount.entries()).map(([name, data]) => ({
+			name,
+			count: data.count,
+			issue_url: data.issue_url,
+			examples: data.examples,
+		}))
+
+		return { by_category: byCategory as any, by_pattern: byPattern }
+	}
+}
+
+interface ParsedTrial {
+	taskId: string
+	trialHash: string
+	passed: boolean
+	duration: number
+	cost: number
+	tokensIn?: number
+	tokensOut?: number
+	logs: string
+	testOutput: string
+	failures: any[]
+}

evals/analysis/src/parsers/index.ts

@@ -0,0 +1,8 @@
+/**
+ * Parser exports for Cline Analysis Framework
+ *
+ * Parsers for different benchmark types:
+ * - Harbor: Real-world tasks via cline-bench
+ */
+
+export * from "./harbor"

evals/analysis/src/reporters/index.ts

@@ -0,0 +1,10 @@
+/**
+ * Reporter exports for Cline Analysis Framework
+ *
+ * Available reporters:
+ * - JsonReporter: Structured JSON output with schema validation
+ * - MarkdownReporter: Human-readable terminal reports
+ */
+
+export * from "./json"
+export * from "./markdown"

evals/analysis/src/reporters/json.ts

@@ -0,0 +1,96 @@
+/**
+ * JSON reporter for Cline analysis results
+ *
+ * Outputs structured JSON with schema versioning for:
+ * - CI integration (baseline diffing, regression detection)
+ * - Programmatic analysis
+ * - Data archival
+ */
+
+import type { AnalysisOutputV1 } from "../schemas"
+
+export class JsonReporter {
+	/**
+	 * Generate JSON report from analysis output
+	 *
+	 * @param output Analysis output (already structured)
+	 * @param pretty Whether to pretty-print the JSON
+	 * @returns JSON string
+	 */
+	generate(output: AnalysisOutputV1, pretty = true): string {
+		return JSON.stringify(output, null, pretty ? 2 : undefined)
+	}
+
+	/**
+	 * Validate that output conforms to AnalysisOutputV1 schema
+	 *
+	 * @param output Analysis output to validate
+	 * @throws Error if schema validation fails
+	 */
+	validate(output: any): asserts output is AnalysisOutputV1 {
+		if (output.schema_version !== "1.0") {
+			throw new Error(`Unsupported schema version: ${output.schema_version}`)
+		}
+
+		// Basic structure validation
+		const required = ["metadata", "summary", "tasks", "failures"]
+		for (const field of required) {
+			if (!(field in output)) {
+				throw new Error(`Missing required field: ${field}`)
+			}
+		}
+
+		// Validate metadata
+		if (!output.metadata.generated_at || !output.metadata.job_id || !output.metadata.model) {
+			throw new Error("Invalid metadata: missing required fields")
+		}
+
+		// Validate summary
+		if (typeof output.summary.total_tasks !== "number") {
+			throw new Error("Invalid summary: total_tasks must be a number")
+		}
+
+		// Validate tasks array
+		if (!Array.isArray(output.tasks)) {
+			throw new Error("Invalid tasks: must be an array")
+		}
+	}
+
+	/**
+	 * Generate a minimal JSON report (without full logs and excerpts)
+	 *
+	 * Useful for CI artifacts where space is limited
+	 *
+	 * @param output Analysis output
+	 * @returns Minified JSON string
+	 */
+	generateMinimal(output: AnalysisOutputV1): string {
+		const minimal = {
+			schema_version: output.schema_version,
+			metadata: {
+				job_id: output.metadata.job_id,
+				model: output.metadata.model,
+				generated_at: output.metadata.generated_at,
+			},
+			summary: output.summary,
+			tasks: output.tasks.map((task) => ({
+				task_id: task.task_id,
+				task_name: task.task_name,
+				status: task.status,
+				metrics: task.metrics,
+				total_cost_usd: task.total_cost_usd,
+				avg_duration_sec: task.avg_duration_sec,
+			})),
+			failures: {
+				by_category: output.failures.by_category,
+				by_pattern: output.failures.by_pattern.map((p) => ({
+					name: p.name,
+					count: p.count,
+					issue_url: p.issue_url,
+				})),
+			},
+		}
+
+		return JSON.stringify(minimal, null, 2)
+	}
+}

evals/analysis/src/reporters/markdown.ts

@@ -0,0 +1,236 @@
+/**
+ * Markdown reporter for Cline analysis results
+ *
+ * Generates human-readable reports with:
+ * - Summary metrics (pass@k, cost, duration)
+ * - Task-by-task results
+ * - Failure analysis with issue links
+ * - Terminal-friendly formatting
+ */
+
+import chalk from "chalk"
+import type { AnalysisOutputV1, TaskResultV1 } from "../schemas"
+
+export class MarkdownReporter {
+	/**
+	 * Generate markdown report from analysis output
+	 *
+	 * @param output Analysis output
+	 * @param useColor Whether to use terminal colors
+	 * @returns Markdown-formatted report string
+	 */
+	generate(output: AnalysisOutputV1, useColor = true): string {
+		const sections: string[] = []
+
+		sections.push(this.generateHeader(output, useColor))
+		sections.push(this.generateSummary(output, useColor))
+		sections.push(this.generateTaskResults(output, useColor))
+		sections.push(this.generateFailureAnalysis(output, useColor))
+		sections.push(this.generateCostPerformance(output, useColor))
+
+		return sections.join("\n\n")
+	}
+
+	private generateHeader(output: AnalysisOutputV1, useColor: boolean): string {
+		const separator = "━".repeat(79)
+		const title = "Cline Bench Analysis Report"
+
+		const lines = [
+			useColor ? chalk.bold(separator) : separator,
+			useColor ? chalk.bold.cyan(title) : title,
+			useColor ? chalk.bold(separator) : separator,
+			"",
+			`Job: ${output.metadata.job_id}`,
+			`Model: ${output.metadata.model}`,
+			`Tasks: ${output.summary.total_tasks} | Trials per task: ${Math.round(output.summary.total_trials / output.summary.total_tasks)}`,
+		]
+
+		return lines.join("\n")
+	}
+
+	private generateSummary(output: AnalysisOutputV1, useColor: boolean): string {
+		const { summary } = output
+		const separator = "━".repeat(79)
+
+		const passAt1Pct = (summary.pass_at_1 * 100).toFixed(1)
+		const passAt3Pct = (summary.pass_at_3 * 100).toFixed(1)
+		const passCaret3Pct = (summary.pass_caret_3 * 100).toFixed(1)
+
+		const lines = [
+			useColor ? chalk.bold(separator) : separator,
+			useColor ? chalk.bold("Results Summary") : "Results Summary",
+			useColor ? chalk.bold(separator) : separator,
+			"",
+			"Overall Metrics:",
+			`  pass@1: ${passAt1Pct}%  (solution finding)`,
+			`  pass@3: ${passAt3Pct}%  (with 3 attempts)`,
+			`  pass^3: ${passCaret3Pct}%  (reliability - all 3 pass)`,
+			"",
+		]
+
+		if (summary.flaky_task_count > 0) {
+			lines.push(
+				useColor
+					? chalk.yellow(`Flakiness: ${summary.flaky_task_count} tasks showed variance across trials`)
+					: `Flakiness: ${summary.flaky_task_count} tasks showed variance across trials`,
+			)
+		} else {
+			lines.push("Flakiness: No variance detected (all tasks consistent)")
+		}
+
+		return lines.join("\n")
+	}
+
+	private generateTaskResults(output: AnalysisOutputV1, useColor: boolean): string {
+		const separator = "━".repeat(79)
+
+		const lines = [
+			useColor ? chalk.bold(separator) : separator,
+			useColor ? chalk.bold("Task Results") : "Task Results",
+			useColor ? chalk.bold(separator) : separator,
+			"",
+		]
+
+		for (const task of output.tasks) {
+			const statusIcon = this.getStatusIcon(task, useColor)
+			const passAt1Pct = (task.metrics.pass_at_1 * 100).toFixed(0)
+			const passCaret3Pct = (task.metrics.pass_caret_3 * 100).toFixed(0)
+			const trialPattern = this.getTrialPattern(task, useColor)
+			const flakyWarning =
+				task.status === "flaky" && useColor ? chalk.yellow(" ⚠️  FLAKY") : task.status === "flaky" ? " ⚠️  FLAKY" : ""
+
+			const taskLine = `${statusIcon} ${task.task_name.padEnd(30)} | pass@1: ${passAt1Pct.padStart(3)}% | pass^3: ${passCaret3Pct.padStart(3)}% | ${trialPattern}${flakyWarning}`
+			lines.push(taskLine)
+		}
+
+		return lines.join("\n")
+	}
+
+	private getStatusIcon(task: TaskResultV1, useColor: boolean): string {
+		if (task.status === "pass") {
+			return useColor ? chalk.green("✓") : "✓"
+		}
+		if (task.status === "fail") {
+			return useColor ? chalk.red("✗") : "✗"
+		}
+		return useColor ? chalk.yellow("◐") : "◐"
+	}
+
+	private getTrialPattern(task: TaskResultV1, useColor: boolean): string {
+		const pattern = task.trials
+			.map((t) => {
+				if (t.passed) {
+					return useColor ? chalk.green("P") : "P"
+				}
+				return useColor ? chalk.red("F") : "F"
+			})
+			.join("")
+
+		return `[${pattern}]`
+	}
+
+	private generateFailureAnalysis(output: AnalysisOutputV1, useColor: boolean): string {
+		const separator = "━".repeat(79)
+
+		const lines = [
+			useColor ? chalk.bold(separator) : separator,
+			useColor ? chalk.bold("Failure Analysis") : "Failure Analysis",
+			useColor ? chalk.bold(separator) : separator,
+			"",
+		]
+
+		// Known issues (provider bugs)
+		const providerBugs = output.failures.by_pattern.filter((p) => p.issue_url)
+
+		if (providerBugs.length > 0) {
+			lines.push("Known Issues Detected:")
+			for (const bug of providerBugs) {
+				const line = `  • ${bug.name} (${bug.count} occurrence${bug.count > 1 ? "s" : ""}) - ${bug.issue_url}`
+				lines.push(useColor ? chalk.yellow(line) : line)
+
+				// Show first example
+				if (bug.examples.length > 0) {
+					const example = bug.examples[0]
+					lines.push(`    Task: ${example.task_id}, Trial ${example.trial_index}`)
+				}
+			}
+			lines.push("")
+		}
+
+		// Transient failures
+		const transient = Object.entries(output.failures.by_category).filter(([cat]) =>
+			["transient", "harness", "environment"].includes(cat),
+		)
+
+		if (transient.length > 0) {
+			lines.push("Infrastructure/Transient Failures:")
+			for (const [category, count] of transient) {
+				lines.push(`  • ${category}: ${count} occurrence${count > 1 ? "s" : ""}`)
+			}
+			lines.push("")
+		}
+
+		// Task failures (model couldn't solve)
+		const taskFailures = output.tasks.filter((t) => t.status === "fail")
+		if (taskFailures.length > 0) {
+			lines.push("Task Failures (Model couldn't solve):")
+			for (const task of taskFailures) {
+				const allFailed = task.trials.every((t) => !t.passed)
+				if (allFailed) {
+					lines.push(`  • ${task.task_name}: All ${task.trials.length} trials failed verification tests`)
+				}
+			}
+		}
+
+		return lines.join("\n")
+	}
+
+	private generateCostPerformance(output: AnalysisOutputV1, useColor: boolean): string {
+		const separator = "━".repeat(79)
+
+		const avgCostPerTask = output.summary.total_cost_usd / output.summary.total_tasks
+		const avgDurationPerTask = output.summary.total_duration_sec / output.summary.total_tasks
+
+		const formattedDuration = this.formatDuration(output.summary.total_duration_sec)
+		const avgFormattedDuration = this.formatDuration(avgDurationPerTask)
+
+		const lines = [
+			useColor ? chalk.bold(separator) : separator,
+			useColor ? chalk.bold("Cost & Performance") : "Cost & Performance",
+			useColor ? chalk.bold(separator) : separator,
+			"",
+			`Total Cost: $${output.summary.total_cost_usd.toFixed(2)}`,
+			`Avg per task: $${avgCostPerTask.toFixed(2)}`,
+			`Total Duration: ${formattedDuration}`,
+			`Avg per task: ${avgFormattedDuration}`,
+			"",
+			`Full report saved to: ${output.metadata.job_id}/analysis_report.md`,
+		]
+
+		return lines.join("\n")
+	}
+
+	private formatDuration(seconds: number): string {
+		if (seconds < 60) {
+			return `${seconds.toFixed(0)}s`
+		}
+		const minutes = Math.floor(seconds / 60)
+		const secs = Math.floor(seconds % 60)
+		return `${minutes}m ${secs}s`
+	}
+
+	/**
+	 * Generate a compact summary (for CI output)
+	 */
+	generateCompactSummary(output: AnalysisOutputV1): string {
+		const passAt1 = (output.summary.pass_at_1 * 100).toFixed(1)
+		const passAt3 = (output.summary.pass_at_3 * 100).toFixed(1)
+		const cost = output.summary.total_cost_usd.toFixed(2)
+
+		return [
+			`✓ pass@1: ${passAt1}% | pass@3: ${passAt3}%`,
+			`  Cost: $${cost} | Tasks: ${output.summary.total_tasks}`,
+			`  Flaky: ${output.summary.flaky_task_count}`,
+		].join("\n")
+	}
+}

evals/analysis/src/schemas/analysis-output.ts

@@ -0,0 +1,146 @@
+/**
+ * Type definitions for Cline Analysis Framework output
+ * These schemas define the structured JSON output from our analysis tools
+ */
+
+/**
+ * Versioned analysis output schema (V1)
+ * Breaking changes should increment the version number
+ */
+export interface AnalysisOutputV1 {
+	schema_version: "1.0"
+	metadata: AnalysisMetadata
+	summary: AnalysisSummary
+	tasks: TaskResultV1[]
+	failures: FailureAnalysis
+}
+
+export interface AnalysisMetadata {
+	generated_at: string // ISO 8601 timestamp
+	analysis_version: string // Package version (from package.json)
+	job_id: string // Job directory name (e.g., "2025-01-25__10-30-00")
+	model: string // Model used for the run
+	agent: string // Agent name (e.g., "cline-cli")
+	environment: string // "docker" | "daytona"
+}
+
+export interface AnalysisSummary {
+	total_tasks: number
+	total_trials: number
+	pass_at_1: number // Probability at least 1 of k=1 succeeds
+	pass_at_3: number // Probability at least 1 of k=3 succeeds
+	pass_caret_3: number // Probability ALL k=3 succeed (reliability)
+	total_cost_usd: number
+	total_duration_sec: number
+	flaky_task_count: number // Tasks with variance across trials
+}
+
+export interface TaskResultV1 {
+	task_id: string
+	task_name: string
+	trials: TrialResultV1[]
+	metrics: TaskMetrics
+	status: "pass" | "fail" | "flaky"
+	total_cost_usd: number
+	avg_duration_sec: number
+}
+
+export interface TrialResultV1 {
+	trial_index: number // 0-indexed trial number
+	trial_hash: string // Hash from trial directory name
+	passed: boolean
+	duration_sec: number
+	cost_usd: number
+	tokens_in?: number
+	tokens_out?: number
+	failures: FailureInfo[] // Classified failure patterns
+}
+
+export interface TaskMetrics {
+	pass_at_1: number // P(at least 1 of 1 succeeds)
+	pass_at_3: number // P(at least 1 of 3 succeeds)
+	pass_caret_3: number // P(all 3 succeed) - reliability metric
+	flakiness_score: number // Entropy-based variance measure (0-1)
+}
+
+export interface FailureInfo {
+	name: string // Pattern name (e.g., "gemini_signature")
+	category: FailureCategory
+	excerpt: string // Log excerpt showing the failure
+	issue_url?: string // GitHub issue link if applicable
+}
+
+export type FailureCategory =
+	| "provider_bug" // Cline integration bugs (Gemini #7974, Claude #7998)
+	| "transient" // Rate limits, network timeouts, service unavailable
+	| "harness" // Test harness or verification script failure
+	| "environment" // Docker/Daytona setup failure
+	| "policy" // Model safety/content policy refusal
+	| "auth" // Invalid API credentials
+	| "task_failure" // Model couldn't solve the task
+
+export interface FailureAnalysis {
+	by_category: Record<FailureCategory, number>
+	by_pattern: FailurePatternSummary[]
+}
+
+export interface FailurePatternSummary {
+	name: string
+	count: number
+	issue_url?: string
+	examples: FailureExample[]
+}
+
+export interface FailureExample {
+	task_id: string
+	trial_index: number
+	excerpt: string
+}
+
+/**
+ * Tool Precision Test Result (for replace_in_file benchmarks)
+ */
+export interface ToolPrecisionResult {
+	schema_version: "1.0"
+	benchmark: "tool-precision/replace-in-file"
+	total_cases: number
+	passed: number
+	failed: number
+	pass_rate: number
+	avg_latency_ms: number
+	known_failures: string[]
+	timestamp: string
+}
+
+/**
+ * Coding Exercises Result (for small task benchmarks)
+ */
+export interface CodingExercisesResult {
+	schema_version: "1.0"
+	benchmark: "coding-exercises"
+	total_exercises: number
+	passed: number
+	failed: number
+	pass_at_1: number
+	avg_duration_sec: number
+	known_failures: string[]
+	timestamp: string
+}
+
+/**
+ * Comparison result between two analysis outputs
+ */
+export interface ComparisonResult {
+	baseline: AnalysisSummary
+	current: AnalysisSummary
+	delta: {
+		pass_at_1: number // Percentage point change
+		pass_at_3: number
+		pass_caret_3: number
+		cost_usd: number
+		duration_sec: number
+	}
+	regression_detected: boolean
+	tasks_regressed: string[] // Task IDs that got worse
+	tasks_improved: string[] // Task IDs that got better
+}

evals/analysis/src/schemas/harbor-output.ts

@@ -0,0 +1,111 @@
+/**
+ * Type definitions for Harbor framework output structure
+ * These schemas document Harbor's jobs/ directory format (read-only, we validate against this)
+ *
+ * Harbor is the execution framework used by cline-bench.
+ * See: https://harborframework.com
+ */
+
+export interface HarborTrialConfig {
+	task_id: string
+	model: string
+	agent: string
+	environment?: string
+	retries?: number
+}
+
+export interface HarborTrialResult {
+	reward: 0 | 1 // Binary pass/fail from verifier
+	duration_sec: number
+	cost_usd: number
+	tokens_in?: number
+	tokens_out?: number
+	timestamp?: string
+}
+
+export interface HarborTrialFiles {
+	agent: {
+		"cline.txt": string // Full conversation log
+		"setup/stdout.txt": string
+		"setup/stderr.txt": string
+		"setup/return-code.txt": string
+		[key: string]: string // command-N/ directories with stdout, stderr, return-code.txt
+	}
+	verifier: {
+		"reward.txt": "0" | "1"
+		"test-stdout.txt": string
+		"test-stderr.txt": string
+	}
+}
+
+/**
+ * Structure of a single trial directory
+ * Example: jobs/2025-01-25__10-30-00/01k7a12s...disco__fhSEuhr/
+ */
+export interface HarborTrialDirectory {
+	"config.json": HarborTrialConfig
+	"result.json": HarborTrialResult
+	agent: HarborTrialFiles["agent"]
+	verifier: HarborTrialFiles["verifier"]
+}
+
+/**
+ * Structure of a job's config.json
+ */
+export interface HarborJobConfig {
+	model: string
+	agent: string
+	tasks: string[] // Task IDs
+	trials_per_task: number
+	environment: string // "docker" | "daytona"
+	created_at?: string
+}
+
+/**
+ * Structure of a job's result.json (aggregate)
+ */
+export interface HarborJobResult {
+	total_tasks: number
+	passed_tasks: number
+	failed_tasks: number
+	total_cost_usd: number
+	total_duration_sec: number
+	started_at?: string
+	completed_at?: string
+}
+
+/**
+ * Complete job directory structure
+ * Example: jobs/2025-01-25__10-30-00/
+ */
+export interface HarborJobDirectory {
+	"config.json": HarborJobConfig
+	"result.json": HarborJobResult
+	trials: Record<string, HarborTrialDirectory> // trial-hash -> trial data
+}
+
+/**
+ * Parsed trial result from Harbor output
+ */
+export interface ParsedHarborTrial {
+	taskId: string
+	trialHash: string
+	passed: boolean
+	duration: number
+	cost: number
+	tokensIn?: number
+	tokensOut?: number
+	logs: string // Full cline.txt content
+	testOutput: string // Test verification output
+	errors: string[] // Setup or command errors
+}
+
+/**
+ * Utility type for extracting trial directory paths
+ */
+export interface HarborTrialPath {
+	jobDir: string
+	trialDir: string
+	taskId: string
+	trialHash: string
+}

evals/analysis/src/schemas/index.ts

@@ -0,0 +1,11 @@
+/**
+ * Schema exports for Cline Analysis Framework
+ *
+ * This module provides TypeScript type definitions for:
+ * - Harbor framework output (read-only validation)
+ * - Analysis framework output (our structured results)
+ * - Comparison results (baseline vs current)
+ */
+
+export * from "./analysis-output"
+export * from "./harbor-output"

evals/analysis/tsconfig.json

@@ -0,0 +1,32 @@
+{
+	"compilerOptions": {
+		"target": "ES2022",
+		"module": "ESNext",
+		"moduleResolution": "bundler",
+		"lib": [
+			"ES2022"
+		],
+		"outDir": "./dist",
+		"rootDir": "./src",
+		"strict": true,
+		"esModuleInterop": true,
+		"skipLibCheck": true,
+		"forceConsistentCasingInFileNames": true,
+		"resolveJsonModule": true,
+		"declaration": true,
+		"declarationMap": true,
+		"sourceMap": true,
+		"types": [
+			"node",
+			"vitest/globals"
+		]
+	},
+	"include": [
+		"src/**/*"
+	],
+	"exclude": [
+		"node_modules",
+		"dist",
+		"src/**/*.test.ts"
+	]
+}

evals/benchmarks/tool-precision/DEPRECATED.md

@@ -0,0 +1,17 @@
+# DEPRECATED
+
+This framework is kept for backward compatibility but is scheduled for removal.
+
+## Why It's Deprecated
+
+1. **Narrow scope**: Tests diff schema compliance, not general model capability
+2. **Better alternatives**: `cline-bench` provides real-world task evaluation
+3. **Maintenance burden**: Requires manual test case generation from conversations
+
+## If You're Using This
+
+Contact @ara or @robin before this gets removed. We can help migrate your use case to the smoke tests or cline-bench framework.
+
+## Removal Timeline
+
+Target: Q2 2026 (or when cline-bench is fully operational for model comparison)

evals/cli/src/adapters/exercism.ts

@@ -1,625 +0,0 @@
-import chalk from "chalk"
-import execa from "execa"
-import * as fs from "fs"
-import * as path from "path"
-import { BenchmarkAdapter, Task, VerificationResult } from "./types"
-
-const EVALS_DIR = path.resolve(__dirname, "../../../")
-
-/**
- * Adapter for the modified Exercism benchmark
- */
-export class ExercismAdapter implements BenchmarkAdapter {
-	name = "exercism"
-
-	/**
-	 * Set up the Exercism benchmark repository
-	 */
-	async setup(): Promise<void> {
-		// Clone repository if needed
-		const exercismDir = path.join(EVALS_DIR, "repositories", "exercism")
-
-		if (!fs.existsSync(exercismDir)) {
-			console.log(`Cloning Exercism repository to ${exercismDir}...`)
-			await execa("git", ["clone", "https://github.com/Aider-AI/polyglot-benchmark.git", exercismDir])
-			console.log("Exercism repository cloned successfully")
-
-			// Unskip all JavaScript and Java tests after cloning
-			this.unskipAllJavaScriptTests(exercismDir)
-			this.unskipAllJavaTests(exercismDir)
-		} else {
-			console.log(`Exercism repository already exists at ${exercismDir}`)
-
-			// Pull latest changes
-			console.log("Pulling latest changes...")
-			await execa("git", ["pull"], { cwd: exercismDir })
-			console.log("Repository updated successfully")
-
-			// Unskip tests again after pulling
-			this.unskipAllJavaScriptTests(exercismDir)
-			this.unskipAllJavaTests(exercismDir)
-		}
-	}
-
-	/**
-	 * List all available tasks in the Exercism benchmark
-	 */
-	async listTasks(): Promise<Task[]> {
-		const tasks: Task[] = []
-		const exercisesDir = path.join(EVALS_DIR, "repositories", "exercism")
-
-		// Ensure the repository exists
-		if (!fs.existsSync(exercisesDir)) {
-			throw new Error(`Exercism repository not found at ${exercisesDir}. Run setup first.`)
-		}
-
-		// Read language directories
-		const languages = fs
-			.readdirSync(exercisesDir)
-			.filter((dir) => fs.statSync(path.join(exercisesDir, dir)).isDirectory())
-			.filter((dir) => !dir.startsWith(".") && !["node_modules", ".git"].includes(dir))
-
-		for (const language of languages) {
-			const languageDir = path.join(exercisesDir, language, "exercises", "practice")
-
-			// Read exercise directories
-			const exercises = fs.readdirSync(languageDir).filter((dir) => fs.statSync(path.join(languageDir, dir)).isDirectory())
-
-			for (const exercise of exercises) {
-				const exerciseDir = path.join(languageDir, exercise)
-
-				// Read instructions
-				let description = ""
-				const instructionsPath = path.join(exerciseDir, ".docs", "instructions.md")
-				if (fs.existsSync(instructionsPath)) {
-					description = fs.readFileSync(instructionsPath, "utf-8")
-				}
-
-				// Determine test commands based on language
-				let testCommands: string[] = []
-				switch (language) {
-					case "cpp":
-						testCommands = ["cmake -DEXERCISM_RUN_ALL_TESTS=1 .", "make"]
-						break
-					case "javascript":
-						testCommands = ["npm install", "npm test -- --testNamePattern=."]
-						break
-					case "python":
-						testCommands = ["python3 -m pytest -o markers=task *_test.py"]
-						break
-					case "go":
-						testCommands = ["GOWORK=off go test -v"]
-						break
-					case "java":
-						testCommands = ["./gradlew test"]
-						break
-					case "rust":
-						testCommands = ["cargo test -- --include-ignored"]
-						break
-					default:
-						testCommands = []
-				}
-
-				tasks.push({
-					id: `exercism-${language}-${exercise}`,
-					name: exercise,
-					description,
-					workspacePath: exerciseDir,
-					setupCommands: [],
-					verificationCommands: testCommands,
-					metadata: {
-						language,
-						type: "exercism",
-					},
-				})
-			}
-		}
-
-		return tasks
-	}
-
-	/**
-	 * Prepare a specific task for execution
-	 * @param taskId The ID of the task to prepare
-	 */
-	async prepareTask(taskId: string): Promise<Task> {
-		const tasks = await this.listTasks()
-		const task = tasks.find((t) => t.id === taskId)
-
-		if (!task) {
-			throw new Error(`Task ${taskId} not found`)
-		}
-
-		// Create temp directory outside workspace for hiding files
-		const tempDir = path.join(EVALS_DIR, "temp-files", task.id)
-		fs.mkdirSync(tempDir, { recursive: true })
-
-		// Read config.json to get solution and test files
-		const configPath = path.join(task.workspacePath, ".meta", "config.json")
-		let config: any = { files: { solution: [], test: [] } }
-
-		if (fs.existsSync(configPath)) {
-			config = JSON.parse(fs.readFileSync(configPath, "utf-8"))
-		}
-
-		// Build enhanced description with instructions
-		let description = ""
-		const instructionsPath = path.join(task.workspacePath, ".docs", "instructions.md")
-		const appendPath = path.join(task.workspacePath, ".docs", "instructions.append.md")
-
-		if (fs.existsSync(instructionsPath)) {
-			description = fs.readFileSync(instructionsPath, "utf-8")
-		}
-
-		if (fs.existsSync(appendPath)) {
-			description += "\n\n" + fs.readFileSync(appendPath, "utf-8")
-		}
-
-		// Add solution files constraint to description
-		const solutionFiles = config.files.solution || []
-		const fileList = solutionFiles.join(", ")
-		description += `\n\nUse the above instructions to modify the supplied files: ${fileList}. Don't change the names of existing functions or classes, as they may be referenced from other code like unit tests, etc. Only use standard libraries, don't suggest installing any packages.`
-		description +=
-			" You should ignore all test or test related files in this directory. The final test file has been removed and will be used to evaluate your work after your implementation is complete. Think deeply about the problem prior to working on the implementation. Consider all edge cases and test your solution prior to finalizing."
-
-		// Move test files to temp directory
-		if (config.files.test) {
-			config.files.test.forEach((testFile: string) => {
-				const src = path.join(task.workspacePath, testFile)
-				if (fs.existsSync(src)) {
-					const dest = path.join(tempDir, testFile)
-					fs.mkdirSync(path.dirname(dest), { recursive: true })
-					fs.renameSync(src, dest)
-				}
-			})
-		}
-
-		// Move all dot directories (except .git) to temp directory
-		const items = fs.readdirSync(task.workspacePath)
-		items.forEach((item) => {
-			if (item.startsWith(".") && item !== ".git") {
-				const src = path.join(task.workspacePath, item)
-				const stat = fs.statSync(src)
-				if (stat.isDirectory()) {
-					const dest = path.join(tempDir, item)
-					fs.renameSync(src, dest)
-				}
-			}
-		})
-
-		return {
-			...task,
-			description,
-			metadata: {
-				...task.metadata,
-				solutionFiles,
-				tempDir,
-				config,
-			},
-		}
-	}
-
-	/**
-	 * Cleanup after task execution (restores hidden files from temp directory)
-	 * @param task The task that was executed
-	 */
-	async cleanupTask(task: Task): Promise<void> {
-		const tempDir = path.join(EVALS_DIR, "temp-files", task.id)
-
-		if (fs.existsSync(tempDir)) {
-			const items = fs.readdirSync(tempDir)
-			items.forEach((item) => {
-				const src = path.join(tempDir, item)
-				const dest = path.join(task.workspacePath, item)
-				// Only move if destination doesn't exist (keeps newer test artifacts like .pytest_cache)
-				if (!fs.existsSync(dest)) {
-					fs.renameSync(src, dest)
-				}
-			})
-
-			// Clean up temp directory
-			fs.rmSync(tempDir, { recursive: true, force: true })
-		}
-	}
-
-	/**
-	 * Verify the result of a task execution by running tests
-	 * @param task The task that was executed
-	 */
-	async verifyResult(task: Task): Promise<VerificationResult> {
-		// Run verification commands
-		let success = true
-		let output = ""
-
-		for (const command of task.verificationCommands) {
-			try {
-				const { stdout, stderr } = await execa(command, {
-					cwd: task.workspacePath,
-					shell: true,
-				})
-				output += stdout + "\n"
-				if (stderr) {
-					output += stderr + "\n"
-				}
-			} catch (error: any) {
-				success = false
-				if (error.stdout) {
-					output += error.stdout + "\n"
-				}
-				if (error.stderr) {
-					output += error.stderr + "\n"
-				}
-			}
-		}
-
-		// Log the raw output
-		// console.log("\n=== TEST OUTPUT START ===")
-		// console.log(output)
-		// console.log("=== TEST OUTPUT END ===\n")
-
-		// Parse test results based on language
-		const language = task.metadata.language
-		let testsPassed = 0
-		let testsFailed = 0
-
-		switch (language) {
-			case "python":
-				const pyPassMatch = output.match(/(\d+) passed/)
-				const pyFailMatch = output.match(/(\d+) failed/)
-				testsPassed = pyPassMatch ? parseInt(pyPassMatch[1]) : 0
-				testsFailed = pyFailMatch ? parseInt(pyFailMatch[1]) : 0
-				break
-
-			case "javascript":
-				const jestMatch = output.match(/Tests:\s+(?:\d+ skipped,\s+)?(\d+) passed(?:,\s+(\d+) failed)?/)
-				if (jestMatch) {
-					testsPassed = parseInt(jestMatch[1])
-					testsFailed = jestMatch[2] ? parseInt(jestMatch[2]) : 0
-				} else {
-					// Fallback to counting test suites
-					testsPassed = (output.match(/PASS/g) || []).length
-					testsFailed = (output.match(/FAIL/g) || []).length
-				}
-				break
-
-			case "go":
-				// This incorrectly counts the parent, but minor and doesn't affect final boolean metric
-				testsPassed = (output.match(/--- PASS:/g) || []).length
-				testsFailed = (output.match(/--- FAIL:/g) || []).length
-				break
-
-			case "rust":
-				// Rust runs multiple test suites (unit, integration, doc tests)
-				// Sum results across all test result lines
-				const resultLines = output.match(/test result:.*?(\d+) passed; (\d+) failed/g)
-				if (resultLines) {
-					testsPassed = 0
-					testsFailed = 0
-					for (const line of resultLines) {
-						const match = line.match(/(\d+) passed; (\d+) failed/)
-						if (match) {
-							testsPassed += parseInt(match[1])
-							testsFailed += parseInt(match[2])
-						}
-					}
-				}
-				break
-
-			case "java":
-				testsPassed = (output.match(/PASSED/g) || []).length
-				testsFailed = (output.match(/FAILED/g) || []).length
-				break
-
-			case "cpp":
-				const cppAllPassedMatch = output.match(/All tests passed \(.*?(\d+) test cases?\)/)
-				const cppTestCasesMatch = output.match(/test cases?: (\d+) \| (\d+) passed/)
-				const cppFailedMatch = output.match(/(\d+) failed/)
-
-				if (cppAllPassedMatch) {
-					// All tests passed - extract total test cases
-					testsPassed = parseInt(cppAllPassedMatch[1])
-					testsFailed = 0
-				} else if (cppTestCasesMatch) {
-					// Mixed results - extract passed count and calculate failed
-					const totalTests = parseInt(cppTestCasesMatch[1])
-					testsPassed = parseInt(cppTestCasesMatch[2])
-					testsFailed = cppFailedMatch ? parseInt(cppFailedMatch[1]) : totalTests - testsPassed
-				}
-				break
-
-			default:
-				// Fallback to generic PASS/FAIL counting
-				testsPassed = (output.match(/PASS/g) || []).length
-				testsFailed = (output.match(/FAIL/g) || []).length
-		}
-
-		const testsTotal = testsPassed + testsFailed
-
-		return {
-			success,
-			rawOutput: output,
-			metrics: {
-				testsPassed,
-				testsFailed,
-				testsTotal,
-				functionalCorrectness: testsTotal > 0 ? testsPassed / testsTotal : 0,
-			},
-		}
-	}
-
-	/**
-	 * Hide test files by moving them to temp directory
-	 * @param task The task to hide test files for
-	 */
-	private hideTestFiles(task: Task): void {
-		const tempDir = task.metadata.tempDir
-		const config = task.metadata.config
-
-		if (config?.files?.test) {
-			config.files.test.forEach((testFile: string) => {
-				const src = path.join(task.workspacePath, testFile)
-				if (fs.existsSync(src)) {
-					const dest = path.join(tempDir, testFile)
-					fs.mkdirSync(path.dirname(dest), { recursive: true })
-					fs.renameSync(src, dest)
-				}
-			})
-		}
-
-		// Hide dot directories again (except .git)
-		const items = fs.readdirSync(task.workspacePath)
-		items.forEach((item) => {
-			if (item.startsWith(".") && item !== ".git") {
-				const src = path.join(task.workspacePath, item)
-				if (fs.existsSync(src)) {
-					const stat = fs.statSync(src)
-					if (stat.isDirectory()) {
-						const dest = path.join(tempDir, item)
-						if (!fs.existsSync(dest)) {
-							fs.renameSync(src, dest)
-						}
-					}
-				}
-			}
-		})
-	}
-
-	/**
-	 * Restore test files by moving them from temp directory
-	 * @param task The task to restore test files for
-	 */
-	private restoreTestFiles(task: Task): void {
-		const tempDir = task.metadata.tempDir
-		const config = task.metadata.config
-
-		if (config?.files?.test) {
-			config.files.test.forEach((testFile: string) => {
-				const src = path.join(tempDir, testFile)
-				if (fs.existsSync(src)) {
-					const dest = path.join(task.workspacePath, testFile)
-					fs.mkdirSync(path.dirname(dest), { recursive: true })
-					fs.renameSync(src, dest)
-				}
-			})
-		}
-
-		// Restore dot directories (except .git)
-		if (fs.existsSync(tempDir)) {
-			const items = fs.readdirSync(tempDir)
-			items.forEach((item) => {
-				if (item.startsWith(".") && item !== ".git") {
-					const src = path.join(tempDir, item)
-					const dest = path.join(task.workspacePath, item)
-					if (fs.existsSync(src) && !fs.existsSync(dest)) {
-						fs.renameSync(src, dest)
-					}
-				}
-			})
-		}
-	}
-
-	/**
-	 * Builds retry message with test errors and fix instructions
-	 * @param testOutput The raw test output showing errors
-	 * @param solutionFiles List of solution files to fix
-	 * @returns Formatted retry message
-	 */
-	private buildRetryMessage(testOutput: string, solutionFiles: string[]): string {
-		const fileList = solutionFiles.join(", ")
-		return `${testOutput}\n\nSee the testing errors above. The tests are correct, don't try and change them. Fix the code in ${fileList} to resolve the errors.`
-	}
-
-	/**
-	 * Unskip all JavaScript tests in the repository by replacing xtest with test
-	 * @param repoPath Path to the exercism repository
-	 */
-	private unskipAllJavaScriptTests(repoPath: string): void {
-		const jsDir = path.join(repoPath, "javascript", "exercises", "practice")
-
-		if (!fs.existsSync(jsDir)) {
-			console.log("JavaScript exercises directory not found, skipping test unskipping")
-			return
-		}
-
-		// Walk through all exercise directories
-		const exercises = fs.readdirSync(jsDir).filter((dir) => {
-			const fullPath = path.join(jsDir, dir)
-			return fs.statSync(fullPath).isDirectory()
-		})
-
-		let filesModified = 0
-		for (const exercise of exercises) {
-			const exerciseDir = path.join(jsDir, exercise)
-
-			// Find all .spec.js files
-			const files = fs.readdirSync(exerciseDir).filter((file) => file.endsWith(".spec.js"))
-
-			for (const file of files) {
-				const filePath = path.join(exerciseDir, file)
-				let content = fs.readFileSync(filePath, "utf-8")
-				const originalContent = content
-
-				// Replace xtest with test to unskip tests
-				content = content.replace(/xtest\(/g, "test(")
-
-				if (content !== originalContent) {
-					fs.writeFileSync(filePath, content)
-					filesModified++
-				}
-			}
-		}
-
-		console.log(`Unskipped tests in ${filesModified} JavaScript test files`)
-	}
-
-	/**
-	 * Unskip all Java tests in the repository by removing @Disabled annotations
-	 * @param repoPath Path to the exercism repository
-	 */
-	private unskipAllJavaTests(repoPath: string): void {
-		const javaDir = path.join(repoPath, "java", "exercises", "practice")
-
-		if (!fs.existsSync(javaDir)) {
-			console.log("Java exercises directory not found, skipping test unskipping")
-			return
-		}
-
-		// Walk through all exercise directories
-		const exercises = fs.readdirSync(javaDir).filter((dir) => {
-			const fullPath = path.join(javaDir, dir)
-			return fs.statSync(fullPath).isDirectory()
-		})
-
-		let filesModified = 0
-		for (const exercise of exercises) {
-			const testDir = path.join(javaDir, exercise, "src", "test", "java")
-
-			if (!fs.existsSync(testDir)) {
-				continue
-			}
-
-			// Find all .java test files
-			const files = fs.readdirSync(testDir).filter((file) => file.endsWith(".java"))
-
-			for (const file of files) {
-				const filePath = path.join(testDir, file)
-				let content = fs.readFileSync(filePath, "utf-8")
-				const originalContent = content
-
-				// Remove @Disabled("Remove to run test") annotations
-				content = content.replace(/@Disabled\("Remove to run test"\)\s*\n/g, "")
-
-				if (content !== originalContent) {
-					fs.writeFileSync(filePath, content)
-					filesModified++
-				}
-			}
-		}
-
-		console.log(`Unskipped tests in ${filesModified} Java test files`)
-	}
-
-	/**
-	 * Runs a Cline task with automatic retry on test failure
-	 * Creates a new Cline instance, runs the task, verifies with tests,
-	 * and retries once if tests fail
-	 * @param task The task to execute
-	 * @returns The final verification result, or null
-	 */
-	async runTask(task: Task): Promise<VerificationResult | null> {
-		const startTime = Date.now()
-		let instanceAddress: string | null = null
-		let attempts = 0
-		let finalVerification: VerificationResult | null = null
-
-		try {
-			// Step 1: Start a new Cline instance in the working directory
-			const instanceResult = await execa("cline", ["instance", "new"], {
-				cwd: task.workspacePath,
-				stdin: "ignore",
-			})
-
-			// Step 2: Parse the instance address from output
-			const addressMatch = instanceResult.stdout.match(/Address:\s*([\d.]+:\d+)/)
-			if (!addressMatch) {
-				throw new Error("Failed to parse instance address from output")
-			}
-			instanceAddress = addressMatch[1]
-
-			// Step 3: Create the initial task on this specific instance
-			await execa("cline", ["task", "new", "--yolo", "--address", instanceAddress, task.description], {
-				cwd: task.workspacePath,
-				stdin: "ignore",
-			})
-
-			// Step 4: Wait for initial implementation to complete
-			console.log(chalk.blue(`Waiting for first attempt to complete...`))
-			await execa("cline", ["task", "view", "--follow-complete", "--address", instanceAddress], {
-				cwd: task.workspacePath,
-				stdin: "ignore",
-			})
-
-			// Step 5: Run first test attempt
-			console.log(chalk.blue(`Running tests (attempt 1)...`))
-			this.restoreTestFiles(task)
-			attempts = 1
-			const firstVerification = await this.verifyResult(task)
-			finalVerification = firstVerification
-
-			// Step 6: Retry if tests failed
-			if (!firstVerification.success) {
-				console.log(chalk.blue(`Tests failed on first attempt. Retrying...`))
-
-				// Hide test files again for retry
-				this.hideTestFiles(task)
-
-				attempts = 2
-				const solutionFiles = task.metadata.solutionFiles || []
-				const retryMessage = this.buildRetryMessage(firstVerification.rawOutput || "", solutionFiles)
-
-				// Send retry task message
-				await execa("cline", ["task", "send", "--yolo", "--address", instanceAddress], {
-					cwd: task.workspacePath,
-					input: retryMessage,
-				})
-
-				// Follow retry until complete
-				await execa("cline", ["task", "view", "--follow-complete", "--address", instanceAddress], {
-					cwd: task.workspacePath,
-					stdin: "ignore",
-				})
-
-				// Run second test attempt (final)
-				console.log(chalk.blue(`Running tests (attempt 2)...`))
-				this.restoreTestFiles(task)
-				const secondVerification = await this.verifyResult(task)
-				finalVerification = secondVerification
-			}
-
-			const duration = Date.now() - startTime
-			console.log(
-				chalk.green(
-					`Task completed in ${(duration / 1000).toFixed(1)}s after ${attempts} attempt${attempts > 1 ? "s" : ""}`,
-				),
-			)
-
-			return finalVerification
-		} catch (error: any) {
-			const duration = Date.now() - startTime
-			console.error(chalk.red(`Task failed after ${(duration / 1000).toFixed(1)}s: ${error.message}`))
-
-			return finalVerification
-		} finally {
-			// Step 7: Always clean up the instance, even if task failed
-			if (instanceAddress) {
-				try {
-					await execa("cline", ["instance", "kill", instanceAddress], {
-						stdin: "ignore",
-					})
-				} catch (cleanupError: any) {
-					console.error(chalk.yellow(`Warning: Failed to kill instance ${instanceAddress}: ${cleanupError.message}`))
-				}
-			}
-		}
-	}
-}

evals/cli/src/adapters/index.ts

@@ -1,38 +0,0 @@
-import { ExercismAdapter } from "./exercism"
-import { BenchmarkAdapter } from "./types"
-
-// Registry of all available adapters
-const adapters: Record<string, BenchmarkAdapter> = {
-	exercism: new ExercismAdapter(),
-}
-
-/**
- * Get a specific adapter by name
- * @param name The name of the adapter to get
- * @returns The requested adapter
- * @throws Error if the adapter is not found
- */
-export function getAdapter(name: string): BenchmarkAdapter {
-	const adapter = adapters[name]
-	if (!adapter) {
-		throw new Error(`Adapter for benchmark '${name}' not found`)
-	}
-	return adapter
-}
-
-/**
- * Get all available adapters
- * @returns Array of all registered adapters
- */
-export function getAllAdapters(): BenchmarkAdapter[] {
-	return Object.values(adapters)
-}
-
-/**
- * Register a new adapter
- * @param name The name to register the adapter under
- * @param adapter The adapter to register
- */
-export function registerAdapter(name: string, adapter: BenchmarkAdapter): void {
-	adapters[name] = adapter
-}

evals/cli/src/adapters/types.ts

@@ -1,34 +0,0 @@
-/**
- * Represents a task to be executed
- */
-export interface Task {
-	id: string
-	name: string
-	description: string
-	workspacePath: string
-	setupCommands: string[]
-	verificationCommands: string[]
-	metadata: Record<string, any>
-}
-
-/**
- * Result of verifying a task execution
- */
-export interface VerificationResult {
-	success: boolean
-	metrics: Record<string, any>
-	rawOutput?: string
-}
-
-/**
- * Interface for benchmark adapters
- */
-export interface BenchmarkAdapter {
-	name: string
-	setup(): Promise<void>
-	listTasks(): Promise<Task[]>
-	prepareTask(taskId: string): Promise<Task>
-	cleanupTask(task: Task): Promise<void>
-	verifyResult(task: Task): Promise<VerificationResult>
-	runTask(task: Task): Promise<VerificationResult | null>
-}

evals/cli/src/commands/report.ts

@@ -1,223 +0,0 @@
-import chalk from "chalk"
-import * as fs from "fs"
-import ora from "ora"
-import * as path from "path"
-import { ResultsDatabase } from "../db"
-import { generateMarkdownReport } from "../utils/markdown"
-
-interface ReportOptions {
-	format?: "json" | "markdown"
-	output?: string
-}
-
-/**
- * Handler for the report command
- * @param options Command options
- */
-export async function reportHandler(options: ReportOptions): Promise<void> {
-	const format = options.format || "markdown"
-	const db = new ResultsDatabase()
-
-	try {
-		const spinner = ora("Generating report...").start()
-
-		// Get all runs
-		const runs = db.getRuns()
-
-		console.log(chalk.blue(`Found ${runs.length} evaluation runs`))
-
-		if (runs.length === 0) {
-			spinner.fail("No evaluation runs found")
-			return
-		}
-
-		// Generate summary report
-		const summary = {
-			runs: runs.length,
-			benchmarks: [...new Set(runs.map((run) => run.benchmark))],
-			tasks: 0,
-			successRate: 0,
-			averageTokens: 0,
-			averageCost: 0,
-			averageDuration: 0,
-			totalToolCalls: 0,
-			totalToolFailures: 0,
-			toolSuccessRate: 0,
-			toolUsage: {} as Record<string, { calls: number; failures: number }>,
-			totalTests: 0,
-			totalTestsPassed: 0,
-			totalTestsFailed: 0,
-			testSuccessRate: 0,
-		}
-
-		let totalTasks = 0
-		let successfulTasks = 0
-		let totalTokens = 0
-		let totalCost = 0
-		let totalDuration = 0
-		let totalToolCalls = 0
-		let totalToolFailures = 0
-		let totalTests = 0
-		let totalTestsPassed = 0
-		let totalTestsFailed = 0
-
-		for (const run of runs) {
-			const tasks = db.getRunTasks(run.id)
-			totalTasks += tasks.length
-
-			for (const task of tasks) {
-				if (task.success) {
-					successfulTasks++
-				}
-
-				const metrics = db.getTaskMetrics(task.id)
-
-				const tokensIn = metrics.find((m) => m.name === "tokensIn")?.value || 0
-				const tokensOut = metrics.find((m) => m.name === "tokensOut")?.value || 0
-				totalTokens += tokensIn + tokensOut
-
-				totalCost += metrics.find((m) => m.name === "cost")?.value || 0
-				totalDuration += metrics.find((m) => m.name === "duration")?.value || 0
-
-				// Collect test metrics
-				const testsPassed = metrics.find((m) => m.name === "testsPassed")?.value || 0
-				const testsFailed = metrics.find((m) => m.name === "testsFailed")?.value || 0
-				const testsTotal = metrics.find((m) => m.name === "testsTotal")?.value || 0
-				totalTestsPassed += testsPassed
-				totalTestsFailed += testsFailed
-				totalTests += testsTotal
-
-				// Collect tool call metrics
-				totalToolCalls += task.total_tool_calls || 0
-				totalToolFailures += task.total_tool_failures || 0
-
-				// Get detailed tool usage
-				const toolCalls = db.getTaskToolCalls(task.id)
-
-				for (const toolCall of toolCalls) {
-					if (!summary.toolUsage[toolCall.tool_name]) {
-						summary.toolUsage[toolCall.tool_name] = {
-							calls: 0,
-							failures: 0,
-						}
-					}
-
-					summary.toolUsage[toolCall.tool_name].calls += toolCall.call_count
-					summary.toolUsage[toolCall.tool_name].failures += toolCall.failure_count
-				}
-			}
-		}
-
-		// Calculate tool success rate
-		summary.totalToolCalls = totalToolCalls
-		summary.totalToolFailures = totalToolFailures
-		summary.toolSuccessRate = totalToolCalls > 0 ? 1 - totalToolFailures / totalToolCalls : 1.0
-
-		// Calculate test metrics
-		summary.totalTests = totalTests
-		summary.totalTestsPassed = totalTestsPassed
-		summary.totalTestsFailed = totalTestsFailed
-		summary.testSuccessRate = totalTests > 0 ? totalTestsPassed / totalTests : 0
-
-		summary.tasks = totalTasks
-		summary.successRate = totalTasks > 0 ? successfulTasks / totalTasks : 0
-		summary.averageTokens = totalTasks > 0 ? totalTokens / totalTasks : 0
-		summary.averageCost = totalTasks > 0 ? totalCost / totalTasks : 0
-		summary.averageDuration = totalTasks > 0 ? totalDuration / totalTasks : 0
-
-		// Generate benchmark-specific reports
-		const benchmarkReports: Record<string, any> = {}
-
-		for (const benchmark of summary.benchmarks) {
-			const benchmarkRuns = runs.filter((run) => run.benchmark === benchmark)
-			const benchmarkSummary = {
-				runs: benchmarkRuns.length,
-				tasks: 0,
-				successRate: 0,
-				averageTokens: 0,
-				averageCost: 0,
-				averageDuration: 0,
-				totalTests: 0,
-				totalTestsPassed: 0,
-				totalTestsFailed: 0,
-				testSuccessRate: 0,
-			}
-
-			let benchmarkTasks = 0
-			let benchmarkSuccessfulTasks = 0
-			let benchmarkTotalTokens = 0
-			let benchmarkTotalCost = 0
-			let benchmarkTotalDuration = 0
-			let benchmarkTotalTests = 0
-			let benchmarkTotalTestsPassed = 0
-			let benchmarkTotalTestsFailed = 0
-
-			for (const run of benchmarkRuns) {
-				const tasks = db.getRunTasks(run.id)
-				benchmarkTasks += tasks.length
-
-				for (const task of tasks) {
-					if (task.success) {
-						benchmarkSuccessfulTasks++
-					}
-
-					const metrics = db.getTaskMetrics(task.id)
-
-					const tokensIn = metrics.find((m) => m.name === "tokensIn")?.value || 0
-					const tokensOut = metrics.find((m) => m.name === "tokensOut")?.value || 0
-					benchmarkTotalTokens += tokensIn + tokensOut
-
-					benchmarkTotalCost += metrics.find((m) => m.name === "cost")?.value || 0
-					benchmarkTotalDuration += metrics.find((m) => m.name === "duration")?.value || 0
-
-					// Collect test metrics
-					const testsPassed = metrics.find((m) => m.name === "testsPassed")?.value || 0
-					const testsFailed = metrics.find((m) => m.name === "testsFailed")?.value || 0
-					const testsTotal = metrics.find((m) => m.name === "testsTotal")?.value || 0
-					benchmarkTotalTestsPassed += testsPassed
-					benchmarkTotalTestsFailed += testsFailed
-					benchmarkTotalTests += testsTotal
-				}
-			}
-
-			benchmarkSummary.tasks = benchmarkTasks
-			benchmarkSummary.successRate = benchmarkTasks > 0 ? benchmarkSuccessfulTasks / benchmarkTasks : 0
-			benchmarkSummary.averageTokens = benchmarkTasks > 0 ? benchmarkTotalTokens / benchmarkTasks : 0
-			benchmarkSummary.averageCost = benchmarkTasks > 0 ? benchmarkTotalCost / benchmarkTasks : 0
-			benchmarkSummary.averageDuration = benchmarkTasks > 0 ? benchmarkTotalDuration / benchmarkTasks : 0
-			benchmarkSummary.totalTests = benchmarkTotalTests
-			benchmarkSummary.totalTestsPassed = benchmarkTotalTestsPassed
-			benchmarkSummary.totalTestsFailed = benchmarkTotalTestsFailed
-			benchmarkSummary.testSuccessRate = benchmarkTotalTests > 0 ? benchmarkTotalTestsPassed / benchmarkTotalTests : 0
-
-			benchmarkReports[benchmark] = benchmarkSummary
-		}
-
-		// Save reports
-		const reportDir = path.join(path.resolve(__dirname, "../../../"), "results", "reports")
-		fs.mkdirSync(reportDir, { recursive: true })
-
-		const timestamp = new Date().toISOString().replace(/:/g, "-")
-
-		if (format === "json") {
-			// Save JSON reports
-			fs.writeFileSync(path.join(reportDir, `summary-${timestamp}.json`), JSON.stringify(summary, null, 2))
-
-			fs.writeFileSync(path.join(reportDir, `benchmarks-${timestamp}.json`), JSON.stringify(benchmarkReports, null, 2))
-
-			spinner.succeed(`JSON reports generated in ${reportDir}`)
-		} else {
-			// Generate markdown report
-			const outputPath = options.output || path.join(reportDir, `report-${timestamp}.md`)
-
-			generateMarkdownReport(summary, benchmarkReports, outputPath)
-
-			spinner.succeed(`Markdown report generated at ${outputPath}`)
-		}
-	} catch (error: any) {
-		console.error(chalk.red(`Error generating report: ${error.message}`))
-		console.error(error.stack)
-	} finally {
-		db.close()
-	}
-}

evals/cli/src/commands/run.ts

@@ -1,114 +0,0 @@
-import chalk from "chalk"
-import ora from "ora"
-import { v4 as uuidv4 } from "uuid"
-import { getAdapter } from "../adapters"
-import { ResultsDatabase } from "../db"
-import { storeTaskResult } from "../utils/results"
-
-interface RunOptions {
-	benchmark?: string
-	count?: number
-}
-
-/**
- * Handler for the run command
- * @param options Command options
- */
-export async function runHandler(options: RunOptions): Promise<void> {
-	// Determine which benchmarks to run
-	const benchmarks = options.benchmark ? [options.benchmark] : ["exercism"] // Default to exercism
-	const count = options.count || Infinity
-
-	console.log(chalk.blue(`Running evaluations for the following benchmarks: ${benchmarks.join(", ")}`))
-
-	// Create a run for each benchmark
-	for (const benchmark of benchmarks) {
-		const runId = uuidv4()
-		const db = new ResultsDatabase()
-
-		console.log(chalk.green(`\nStarting run for benchmark: ${benchmark}`))
-
-		// Create run in database
-		db.createRun(runId, benchmark)
-
-		// Get adapter for this benchmark
-		try {
-			const adapter = getAdapter(benchmark)
-
-			// List tasks
-			const spinner = ora("Listing tasks...").start()
-			const tasks = await adapter.listTasks()
-			spinner.succeed(`Found ${tasks.length} tasks for ${benchmark}`)
-
-			// Limit number of tasks if specified
-			const tasksToRun = tasks.slice(0, count)
-
-			console.log(chalk.blue(`Running ${tasksToRun.length} tasks...`))
-
-			// Run each task
-			for (let i = 0; i < tasksToRun.length; i++) {
-				const task = tasksToRun[i]
-
-				console.log(chalk.cyan(`\nTask ${i + 1}/${tasksToRun.length}: ${task.name}`))
-
-				// Prepare task
-				const prepareSpinner = ora("Preparing task...").start()
-				const preparedTask = await adapter.prepareTask(task.id)
-				prepareSpinner.succeed("Task prepared")
-
-				let cleanedUp = false
-
-				try {
-					// Run task using adapter's execution strategy
-					const finalVerification = await adapter.runTask(preparedTask)
-
-					// Cleanup task
-					const cleanupSpinner = ora("Cleaning up task...").start()
-					await adapter.cleanupTask(preparedTask)
-					cleanedUp = true
-					cleanupSpinner.succeed("Cleanup complete")
-
-					// Use final verification from runTask
-					const verification = finalVerification || (await adapter.verifyResult(preparedTask))
-
-					if (verification.success) {
-						console.log(
-							chalk.green(`Tests passed: ${verification.metrics.testsPassed}/${verification.metrics.testsTotal}`),
-						)
-					} else {
-						console.log(
-							chalk.red(`Tests failed: ${verification.metrics.testsPassed}/${verification.metrics.testsTotal}`),
-						)
-					}
-
-					// Store result
-					const storeSpinner = ora("Storing result...").start()
-					await storeTaskResult(runId, preparedTask, {}, verification)
-					storeSpinner.succeed("Result stored")
-				} catch (error: any) {
-					console.error(chalk.red(`Task failed: ${error.message}`))
-				} finally {
-					// Ensure cleanup always happens
-					if (!cleanedUp) {
-						try {
-							const finalCleanupSpinner = ora("Performing cleanup...").start()
-							await adapter.cleanupTask(preparedTask)
-							finalCleanupSpinner.succeed("Cleanup complete")
-						} catch (cleanupError: any) {
-							console.error(chalk.red(`Cleanup failed: ${cleanupError.message}`))
-						}
-					}
-				}
-			}
-
-			// Mark run as complete
-			db.completeRun(runId)
-
-			console.log(chalk.green(`\nRun complete for benchmark: ${benchmark}`))
-		} catch (error: any) {
-			console.error(chalk.red(`Error running benchmark ${benchmark}: ${error.message}`))
-		}
-	}
-
-	console.log(chalk.green("\nAll evaluations complete"))
-}

evals/cli/src/commands/runDiffEval.ts

@@ -1,107 +0,0 @@
-import execa from "execa"
-import chalk from "chalk"
-import path from "path"
-
-interface RunDiffEvalOptions {
-	modelIds: string
-	systemPromptName: string
-	validAttemptsPerCase: number
-	maxAttemptsPerCase?: number
-	parsingFunction: string
-	diffEditFunction: string
-	thinkingBudget: number
-	provider: string
-	parallel: boolean
-	verbose: boolean
-	testPath: string
-	outputPath: string
-	replay: boolean
-	replayRunId?: string
-	diffApplyFile?: string
-	saveLocally: boolean
-	maxCases?: number
-}
-
-export async function runDiffEvalHandler(options: RunDiffEvalOptions) {
-	console.log(chalk.blue("Starting diff editing evaluation..."))
-
-	// Resolve the path to the TestRunner.ts script relative to the current file
-	const scriptPath = path.resolve(__dirname, "../../../diff-edits/TestRunner.ts")
-
-	// Construct the arguments array for the execa call
-	const args = [
-		"--model-ids",
-		options.modelIds,
-		"--system-prompt-name",
-		options.systemPromptName,
-		"--valid-attempts-per-case",
-		String(options.validAttemptsPerCase),
-		"--parsing-function",
-		options.parsingFunction,
-		"--diff-edit-function",
-		options.diffEditFunction,
-		"--provider",
-		options.provider,
-	]
-
-	// Conditionally add the optional arguments
-	if (options.testPath) {
-		args.push("--test-path", options.testPath)
-	}
-	if (options.outputPath) {
-		args.push("--output-path", options.outputPath)
-	}
-	if (options.thinkingBudget > 0) {
-		args.push("--thinking-budget", String(options.thinkingBudget))
-	}
-
-	if (options.parallel) {
-		args.push("--parallel")
-	}
-
-	if (options.replay) {
-		args.push("--replay")
-	}
-
-	if (options.replayRunId) {
-		args.push("--replay-run-id", options.replayRunId)
-	}
-
-	if (options.diffApplyFile) {
-		args.push("--diff-apply-file", options.diffApplyFile)
-	}
-
-	if (options.verbose) {
-		args.push("--verbose")
-	}
-
-	if (options.maxAttemptsPerCase) {
-		args.push("--max-attempts-per-case", String(options.maxAttemptsPerCase))
-	}
-
-	if (options.maxCases) {
-		args.push("--max-cases", String(options.maxCases))
-	}
-
-	if (options.saveLocally) {
-		args.push("--save-locally")
-	}
-
-	try {
-		console.log(chalk.gray(`Executing: npx tsx ${scriptPath} ${args.join(" ")}`))
-
-		// Execute the script as a child process
-		// We use 'inherit' to stream the stdout/stderr directly to the user's terminal
-		const subprocess = execa("npx", ["tsx", "--tsconfig", path.resolve(__dirname, "../../../tsconfig.json"), scriptPath, ...args], {
-			stdio: "inherit",
-		})
-
-		await subprocess
-
-		console.log(chalk.green("Diff editing evaluation completed successfully."))
-	} catch (error) {
-		console.error(chalk.red("An error occurred during the diff editing evaluation."))
-		// The 'inherit' stdio will have already printed the error details from the script
-		process.exit(1)
-	}
-}

evals/cli/src/commands/setup.ts

@@ -1,72 +0,0 @@
-import * as path from "path"
-import * as fs from "fs"
-import execa from "execa"
-import chalk from "chalk"
-import ora from "ora"
-import { getAllAdapters } from "../adapters/index"
-import { BenchmarkAdapter } from "../adapters/types"
-
-interface SetupOptions {
-	benchmarks: string
-}
-
-/**
- * Handler for the setup command
- * @param options Command options
- */
-export async function setupHandler(options: SetupOptions): Promise<void> {
-	const benchmarks = options.benchmarks.split(",")
-
-	console.log(chalk.blue(`Setting up benchmarks: ${benchmarks.join(", ")}`))
-
-	// Create directories
-	const evalsDir = path.resolve(__dirname, "../../../")
-	const reposDir = path.join(evalsDir, "repositories")
-	const resultsDir = path.join(evalsDir, "results")
-
-	const spinner = ora("Creating directory structure").start()
-
-	try {
-		fs.mkdirSync(reposDir, { recursive: true })
-		fs.mkdirSync(resultsDir, { recursive: true })
-		fs.mkdirSync(path.join(resultsDir, "runs"), { recursive: true })
-		fs.mkdirSync(path.join(resultsDir, "reports"), { recursive: true })
-		spinner.succeed("Directory structure created")
-	} catch (error) {
-		spinner.fail(`Failed to create directory structure: ${(error as Error).message}`)
-		throw error
-	}
-
-	// Set up each benchmark
-	try {
-		const adapters = getAllAdapters().filter((adapter: BenchmarkAdapter) => benchmarks.includes(adapter.name))
-
-		if (adapters.length === 0) {
-			console.warn(chalk.yellow("No valid benchmarks specified. Available benchmarks:"))
-			console.warn(
-				chalk.yellow(
-					getAllAdapters()
-						.map((a: BenchmarkAdapter) => a.name)
-						.join(", "),
-				),
-			)
-			return
-		}
-
-		for (const adapter of adapters) {
-			const setupSpinner = ora(`Setting up ${adapter.name}...`).start()
-			try {
-				await adapter.setup()
-				setupSpinner.succeed(`${adapter.name} setup complete`)
-			} catch (error) {
-				setupSpinner.fail(`Failed to set up ${adapter.name}: ${(error as Error).message}`)
-				throw error
-			}
-		}
-
-		console.log(chalk.green("Setup complete"))
-	} catch (error) {
-		console.error(chalk.red(`Setup failed: ${(error as Error).message}`))
-		throw error
-	}
-}

evals/cli/src/db/index.ts

@@ -1,210 +0,0 @@
-import Database from "better-sqlite3"
-import * as fs from "fs"
-import * as path from "path"
-import { SCHEMA } from "./schema"
-
-const EVALS_DIR = path.resolve(__dirname, "../../../")
-
-/**
- * Database class for storing evaluation results
- */
-export class ResultsDatabase {
-	db: Database.Database
-
-	constructor() {
-		// Ensure results directory exists
-		const resultsDir = path.join(EVALS_DIR, "results")
-		fs.mkdirSync(resultsDir, { recursive: true })
-
-		// Create database file
-		const dbPath = path.join(resultsDir, "evals.db")
-		this.db = new Database(dbPath)
-
-		// Initialize schema
-		this.initSchema()
-	}
-
-	/**
-	 * Initialize the database schema
-	 */
-	private initSchema(): void {
-		this.db.exec(SCHEMA)
-	}
-
-	/**
-	 * Create a new evaluation run
-	 * @param id Run ID
-	 * @param benchmark Benchmark name
-	 */
-	createRun(id: string, benchmark: string): void {
-		const stmt = this.db.prepare(`
-      INSERT INTO runs (id, timestamp, benchmark)
-      VALUES (?, ?, ?)
-    `)
-
-		stmt.run(id, Date.now(), benchmark)
-	}
-
-	/**
-	 * Mark a run as completed
-	 * @param id Run ID
-	 */
-	completeRun(id: string): void {
-		const stmt = this.db.prepare(`
-      UPDATE runs SET completed = 1 WHERE id = ?
-    `)
-
-		stmt.run(id)
-	}
-
-	/**
-	 * Create a new task
-	 * @param id Task ID
-	 * @param runId Run ID
-	 * @param taskId Original task ID
-	 */
-	createTask(id: string, runId: string, taskId: string): void {
-		const stmt = this.db.prepare(`
-      INSERT INTO tasks (id, run_id, task_id, timestamp)
-      VALUES (?, ?, ?, ?)
-    `)
-
-		stmt.run(id, runId, taskId, Date.now())
-	}
-
-	/**
-	 * Mark a task as completed
-	 * @param id Task ID
-	 * @param success Whether the task was successful
-	 * @param toolCalls Total tool calls
-	 * @param toolFailures Total tool failures
-	 */
-	completeTask(id: string, success: boolean, toolCalls: number = 0, toolFailures: number = 0): void {
-		const stmt = this.db.prepare(`
-      UPDATE tasks 
-      SET success = ?, total_tool_calls = ?, total_tool_failures = ? 
-      WHERE id = ?
-    `)
-
-		stmt.run(success ? 1 : 0, toolCalls, toolFailures, id)
-	}
-
-	/**
-	 * Add a metric to a task
-	 * @param taskId Task ID
-	 * @param name Metric name
-	 * @param value Metric value
-	 */
-	addMetric(taskId: string, name: string, value: number): void {
-		const stmt = this.db.prepare(`
-      INSERT INTO metrics (task_id, name, value)
-      VALUES (?, ?, ?)
-    `)
-
-		stmt.run(taskId, name, value)
-	}
-
-	/**
-	 * Add a tool call record
-	 * @param taskId Task ID
-	 * @param toolName Tool name
-	 * @param callCount Number of calls
-	 * @param failureCount Number of failures
-	 */
-	addToolCall(taskId: string, toolName: string, callCount: number, failureCount: number): void {
-		const stmt = this.db.prepare(`
-      INSERT INTO tool_calls (task_id, tool_name, call_count, failure_count)
-      VALUES (?, ?, ?, ?)
-    `)
-
-		stmt.run(taskId, toolName, callCount, failureCount)
-	}
-
-	/**
-	 * Add a file record
-	 * @param taskId Task ID
-	 * @param filePath File path
-	 * @param status File status (created, modified, deleted)
-	 */
-	addFile(taskId: string, filePath: string, status: "created" | "modified" | "deleted"): void {
-		const stmt = this.db.prepare(`
-      INSERT INTO files (task_id, path, status)
-      VALUES (?, ?, ?)
-    `)
-
-		stmt.run(taskId, filePath, status)
-	}
-
-	/**
-	 * Get all runs
-	 * @returns Array of runs
-	 */
-	getRuns(): any[] {
-		const stmt = this.db.prepare(`
-      SELECT * FROM runs ORDER BY timestamp DESC
-    `)
-
-		return stmt.all()
-	}
-
-	/**
-	 * Get all tasks for a run
-	 * @param runId Run ID
-	 * @returns Array of tasks
-	 */
-	getRunTasks(runId: string): any[] {
-		const stmt = this.db.prepare(`
-      SELECT * FROM tasks WHERE run_id = ? ORDER BY timestamp ASC
-    `)
-
-		return stmt.all(runId)
-	}
-
-	/**
-	 * Get all metrics for a task
-	 * @param taskId Task ID
-	 * @returns Array of metrics
-	 */
-	getTaskMetrics(taskId: string): any[] {
-		const stmt = this.db.prepare(`
-      SELECT name, value FROM metrics WHERE task_id = ?
-    `)
-
-		return stmt.all(taskId)
-	}
-
-	/**
-	 * Get all tool calls for a task
-	 * @param taskId Task ID
-	 * @returns Array of tool calls
-	 */
-	getTaskToolCalls(taskId: string): any[] {
-		const stmt = this.db.prepare(`
-      SELECT tool_name, call_count, failure_count 
-      FROM tool_calls 
-      WHERE task_id = ?
-    `)
-
-		return stmt.all(taskId)
-	}
-
-	/**
-	 * Get all files for a task
-	 * @param taskId Task ID
-	 * @returns Array of files
-	 */
-	getTaskFiles(taskId: string): any[] {
-		const stmt = this.db.prepare(`
-      SELECT path, status FROM files WHERE task_id = ?
-    `)
-
-		return stmt.all(taskId)
-	}
-
-	/**
-	 * Close the database connection
-	 */
-	close(): void {
-		this.db.close()
-	}
-}

evals/cli/src/db/schema.ts

@@ -1,47 +0,0 @@
-/**
- * SQL schema for the evaluation database
- */
-export const SCHEMA = `
-CREATE TABLE IF NOT EXISTS runs (
-  id TEXT PRIMARY KEY,
-  timestamp INTEGER NOT NULL,
-  benchmark TEXT NOT NULL,
-  completed INTEGER NOT NULL DEFAULT 0
-);
-
-CREATE TABLE IF NOT EXISTS tasks (
-  id TEXT PRIMARY KEY,
-  run_id TEXT NOT NULL,
-  task_id TEXT NOT NULL,
-  timestamp INTEGER NOT NULL,
-  success INTEGER NOT NULL DEFAULT 0,
-  total_tool_calls INTEGER DEFAULT 0,
-  total_tool_failures INTEGER DEFAULT 0,
-  FOREIGN KEY (run_id) REFERENCES runs(id)
-);
-
-CREATE TABLE IF NOT EXISTS metrics (
-  id INTEGER PRIMARY KEY AUTOINCREMENT,
-  task_id TEXT NOT NULL,
-  name TEXT NOT NULL,
-  value REAL NOT NULL,
-  FOREIGN KEY (task_id) REFERENCES tasks(id)
-);
-
-CREATE TABLE IF NOT EXISTS tool_calls (
-  id INTEGER PRIMARY KEY AUTOINCREMENT,
-  task_id TEXT NOT NULL,
-  tool_name TEXT NOT NULL,
-  call_count INTEGER NOT NULL,
-  failure_count INTEGER NOT NULL,
-  FOREIGN KEY (task_id) REFERENCES tasks(id)
-);
-
-CREATE TABLE IF NOT EXISTS files (
-  id INTEGER PRIMARY KEY AUTOINCREMENT,
-  task_id TEXT NOT NULL,
-  path TEXT NOT NULL,
-  status TEXT NOT NULL,
-  FOREIGN KEY (task_id) REFERENCES tasks(id)
-);
-`

evals/cli/src/index.ts

@@ -1,106 +0,0 @@
-#!/usr/bin/env node
-import chalk from "chalk"
-import { Command } from "commander"
-import { reportHandler } from "./commands/report"
-import { runHandler } from "./commands/run"
-import { runDiffEvalHandler } from "./commands/runDiffEval"
-import { setupHandler } from "./commands/setup"
-
-// Create the CLI program
-const program = new Command()
-
-// Set up CLI metadata
-program.name("cline-eval").description("CLI tool for orchestrating Cline evaluations across multiple benchmarks").version("0.1.0")
-
-// Setup command
-program
-	.command("setup")
-	.description("Clone and set up benchmark repositories")
-	.option("-b, --benchmarks <benchmarks>", "Comma-separated list of benchmarks to set up", "exercism")
-	.action(async (options) => {
-		try {
-			await setupHandler(options)
-		} catch (error) {
-			console.error(chalk.red(`Error during setup: ${error instanceof Error ? error.message : String(error)}`))
-			process.exit(1)
-		}
-	})
-
-// Run command
-program
-	.command("run")
-	.description("Run evaluations")
-	.option("-b, --benchmark <benchmark>", "Specific benchmark to run")
-	.option("-c, --count <count>", "Number of tasks to run", parseInt)
-	.action(async (options) => {
-		try {
-			await runHandler(options)
-		} catch (error) {
-			console.error(chalk.red(`Error during run: ${error instanceof Error ? error.message : String(error)}`))
-			process.exit(1)
-		}
-	})
-
-// Report command
-program
-	.command("report")
-	.description("Generate reports")
-	.option("-f, --format <format>", "Report format (json, markdown)", "markdown")
-	.option("-o, --output <path>", "Output path for the report")
-	.action(async (options) => {
-		try {
-			await reportHandler(options)
-		} catch (error) {
-			console.error(chalk.red(`Error generating report: ${error instanceof Error ? error.message : String(error)}`))
-			process.exit(1)
-		}
-	})
-
-// Run-diff-eval command
-program
-	.command("run-diff-eval")
-	.description("Run the diff editing evaluation suite")
-	.option("--test-path <path>", "Path to the directory containing test case JSON files")
-	.option("--output-path <path>", "Path to the directory to save the test output JSON files")
-	.option("--model-ids <model_ids>", "Comma-separated list of model IDs to test")
-	.option("--system-prompt-name <name>", "The name of the system prompt to use", "basicSystemPrompt")
-	.option(
-		"-n, --valid-attempts-per-case <number>",
-		"Number of valid attempts per test case per model (will retry until this many valid attempts are collected)",
-		"1",
-	)
-	.option("--max-attempts-per-case <number>", "Maximum total attempts per test case (default: 10x valid attempts)")
-	.option("--max-cases <number>", "Maximum number of test cases to run (limits total cases loaded)")
-	.option("--parsing-function <name>", "The parsing function to use", "parseAssistantMessageV2")
-	.option("--diff-edit-function <name>", "The diff editing function to use", "diff-06-26-25")
-	.option("--thinking-budget <tokens>", "Set the thinking tokens budget", "0")
-	.option("--provider <provider>", "API provider to use (openrouter, openai)", "openrouter")
-	.option("--parallel", "Run tests in parallel", false)
-	.option("--replay", "Run evaluation from a pre-recorded LLM output, skipping the API call", false)
-	.option("--replay-run-id <run_id>", "The ID of the run to replay from the database")
-	.option("--diff-apply-file <filename>", "The name of the diff apply file to use for the replay")
-	.option("--save-locally", "Save results to local JSON files in addition to database", false)
-	.option("-v, --verbose", "Enable verbose logging", false)
-	.action(async (options) => {
-		try {
-			const fullOptions = {
-				...options,
-				validAttemptsPerCase: parseInt(options.validAttemptsPerCase, 10),
-				maxAttemptsPerCase: options.maxAttemptsPerCase ? parseInt(options.maxAttemptsPerCase, 10) : undefined,
-				thinkingBudget: parseInt(options.thinkingBudget, 10),
-				maxCases: options.maxCases ? parseInt(options.maxCases, 10) : undefined,
-			}
-			await runDiffEvalHandler(fullOptions)
-		} catch (error) {
-			console.error(chalk.red(`Error during diff eval run: ${error instanceof Error ? error.message : String(error)}`))
-			process.exit(1)
-		}
-	})
-
-// Parse command line arguments
-program.parse(process.argv)
-
-// If no arguments provided, show help
-if (process.argv.length === 2) {
-	program.help()
-}

evals/cli/src/utils/markdown.ts

@@ -1,80 +0,0 @@
-import * as fs from "fs"
-
-/**
- * Generate a markdown report from evaluation results
- * @param summary Overall summary
- * @param benchmarkReports Benchmark-specific reports
- * @param outputPath Output file path
- */
-export function generateMarkdownReport(summary: any, benchmarkReports: Record<string, any>, outputPath: string): void {
-	let markdown = `# Cline Evaluation Report\n\n`
-
-	// Generate summary section
-	markdown += `## Summary\n\n`
-	markdown += `- **Total Runs:** ${summary.runs}\n`
-	markdown += `- **Benchmarks:** ${summary.benchmarks.join(", ")}\n`
-	markdown += `- **Total Tasks:** ${summary.tasks}\n`
-	markdown += `- **Task Success Rate:** ${(summary.successRate * 100).toFixed(2)}%\n`
-	markdown += `- **Total Tests:** ${summary.totalTests}\n`
-	markdown += `- **Tests Passed:** ${summary.totalTestsPassed}\n`
-	markdown += `- **Tests Failed:** ${summary.totalTestsFailed}\n`
-	markdown += `- **Test Success Rate:** ${(summary.testSuccessRate * 100).toFixed(2)}%\n`
-	markdown += `- **Average Tokens:** ${Math.round(summary.averageTokens)}\n`
-	markdown += `- **Average Cost:** $${summary.averageCost.toFixed(4)}\n`
-	markdown += `- **Average Duration:** ${(summary.averageDuration / 1000).toFixed(2)}s\n`
-	markdown += `- **Total Tool Calls:** ${summary.totalToolCalls}\n`
-	markdown += `- **Tool Success Rate:** ${(summary.toolSuccessRate * 100).toFixed(2)}%\n\n`
-
-	// Generate tool usage section
-	markdown += `## Tool Usage\n\n`
-	markdown += `| Tool | Calls | Failures | Success Rate |\n`
-	markdown += `| ---- | ----- | -------- | ------------ |\n`
-
-	for (const [toolName, metrics] of Object.entries(summary.toolUsage)) {
-		const calls = (metrics as any).calls
-		const failures = (metrics as any).failures
-		const successRate = calls > 0 ? (1 - failures / calls) * 100 : 100
-
-		markdown += `| ${toolName} | ${calls} | ${failures} | ${successRate.toFixed(2)}% |\n`
-	}
-
-	// Generate benchmark results section
-	markdown += `\n## Benchmark Results\n\n`
-
-	for (const [benchmark, report] of Object.entries(benchmarkReports)) {
-		markdown += `### ${benchmark}\n\n`
-		markdown += `- **Runs:** ${report.runs}\n`
-		markdown += `- **Tasks:** ${report.tasks}\n`
-		markdown += `- **Task Success Rate:** ${(report.successRate * 100).toFixed(2)}%\n`
-		markdown += `- **Total Tests:** ${report.totalTests}\n`
-		markdown += `- **Tests Passed:** ${report.totalTestsPassed}\n`
-		markdown += `- **Tests Failed:** ${report.totalTestsFailed}\n`
-		markdown += `- **Test Success Rate:** ${(report.testSuccessRate * 100).toFixed(2)}%\n`
-		markdown += `- **Average Tokens:** ${Math.round(report.averageTokens)}\n`
-		markdown += `- **Average Cost:** $${report.averageCost.toFixed(4)}\n`
-		markdown += `- **Average Duration:** ${(report.averageDuration / 1000).toFixed(2)}s\n\n`
-	}
-
-	// Add charts using Mermaid
-	markdown += `## Charts\n\n`
-
-	// Success rate by benchmark chart
-	markdown += `### Success Rate by Benchmark\n\n`
-	markdown += "```mermaid\n"
-	markdown += "graph TD\n"
-	markdown += "  title[Success Rate by Benchmark]\n"
-	markdown += "  style title fill:none,stroke:none\n\n"
-
-	for (const [benchmark, report] of Object.entries(benchmarkReports)) {
-		const successRate = (report.successRate * 100).toFixed(2)
-		markdown += `  ${benchmark}[${benchmark}: ${successRate}%]\n`
-	}
-
-	markdown += "```\n\n"
-
-	// Add timestamp
-	markdown += `\n\n---\n\nReport generated on ${new Date().toISOString()}\n`
-
-	// Write markdown to file
-	fs.writeFileSync(outputPath, markdown)
-}

evals/cli/src/utils/results.ts

@@ -1,79 +0,0 @@
-import { v4 as uuidv4 } from "uuid"
-import { ResultsDatabase } from "../db"
-import { Task } from "../adapters/types"
-
-/**
- * Store task result in the database
- * @param runId The run ID
- * @param task The task that was executed
- * @param result The result from the test server
- * @param verification The verification result
- */
-export async function storeTaskResult(runId: string, task: Task, result: any, verification: any): Promise<void> {
-	const db = new ResultsDatabase()
-	const taskId = uuidv4()
-
-	try {
-		// Extract metrics from the result
-		const { metrics } = result
-		const totalToolCalls = metrics?.totalToolCalls || 0
-		const totalToolFailures = metrics?.totalToolFailures || 0
-
-		// Create task with tool metrics
-		db.createTask(taskId, runId, task.id)
-		db.completeTask(taskId, verification.success, totalToolCalls, totalToolFailures)
-
-		// Store metrics
-		if (metrics) {
-			// Store token metrics
-			if (metrics.tokensIn) db.addMetric(taskId, "tokensIn", metrics.tokensIn)
-			if (metrics.tokensOut) db.addMetric(taskId, "tokensOut", metrics.tokensOut)
-			if (metrics.cost) db.addMetric(taskId, "cost", metrics.cost)
-			if (metrics.duration) db.addMetric(taskId, "duration", metrics.duration)
-
-			// Store tool call metrics
-			if (metrics.toolCalls) {
-				for (const [toolName, callCount] of Object.entries(metrics.toolCalls)) {
-					const failureCount = metrics.toolFailures?.[toolName] || 0
-					db.addToolCall(taskId, toolName, callCount as number, failureCount)
-				}
-			}
-		}
-
-		// Store verification metrics
-		if (verification.metrics) {
-			for (const [key, value] of Object.entries(verification.metrics)) {
-				if (typeof value === "number") {
-					db.addMetric(taskId, key, value)
-				}
-			}
-		}
-
-		// Store file changes
-		if (result.files) {
-			// Store created files
-			if (result.files.created) {
-				for (const file of result.files.created) {
-					db.addFile(taskId, file, "created")
-				}
-			}
-
-			// Store modified files
-			if (result.files.modified) {
-				for (const file of result.files.modified) {
-					db.addFile(taskId, file, "modified")
-				}
-			}
-
-			// Store deleted files
-			if (result.files.deleted) {
-				for (const file of result.files.deleted) {
-					db.addFile(taskId, file, "deleted")
-				}
-			}
-		}
-	} finally {
-		// Close the database connection
-		db.close()
-	}
-}

evals/cli/tsconfig.json

@@ -1,17 +0,0 @@
-{
-	"compilerOptions": {
-		"target": "ES2020",
-		"module": "commonjs",
-		"lib": ["ES2020"],
-		"declaration": true,
-		"outDir": "dist",
-		"rootDir": "src",
-		"strict": true,
-		"esModuleInterop": true,
-		"skipLibCheck": true,
-		"forceConsistentCasingInFileNames": true,
-		"resolveJsonModule": true
-	},
-	"include": ["src/**/*"],
-	"exclude": ["node_modules", "dist"]
-}

evals/e2e/README.md

@@ -0,0 +1,128 @@
+# E2E Agent Tests
+
+Full end-to-end tests using real-world tasks from cline-bench.
+
+## Overview
+
+These tests run Cline against production-grade coding problems derived from actual user sessions. Each task:
+- Starts with a broken codebase in Docker
+- Gives Cline the task description
+- Verifies the fix with pytest
+
+## Prerequisites
+
+1. **Python 3.13 with uv**
+   ```bash
+   # macOS
+   brew install python@3.13
+   pip install uv
+   ```
+
+2. **Harbor** (benchmark execution framework)
+   ```bash
+   uv tool install harbor
+   ```
+
+3. **Docker** (for local execution)
+   ```bash
+   # Verify Docker is running
+   docker info
+   ```
+
+4. **API Keys**
+   ```bash
+   export ANTHROPIC_API_KEY=sk-ant-...
+   # or
+   export API_KEY=sk-ant-...  # Generic fallback
+   ```
+
+## Running Locally
+
+```bash
+# Run all tasks with default settings (Anthropic, Docker)
+npx tsx evals/e2e/run-cline-bench.ts
+
+# Run specific task
+npx tsx evals/e2e/run-cline-bench.ts --tasks discord
+
+# Use different provider/model
+npx tsx evals/e2e/run-cline-bench.ts --provider openai --model gpt-4o
+
+# Run on Daytona cloud (faster, parallel)
+export DAYTONA_API_KEY=dtn_...
+npx tsx evals/e2e/run-cline-bench.ts --env daytona
+
+# Output to JSON
+npx tsx evals/e2e/run-cline-bench.ts --output results.json
+```
+
+## CLI Options
+
+| Option | Default | Description |
+|--------|---------|-------------|
+| `--env` | `docker` | Execution environment: `docker` or `daytona` |
+| `--provider` | `anthropic` | Provider: `anthropic`, `openai`, `openrouter`, `gemini` |
+| `--model` | `claude-sonnet-4-20250514` | Model ID |
+| `--tasks` | `all` | Task filter pattern |
+| `--trials` | `1` | Number of trials per task |
+| `--output` | - | Write JSON results to file |
+
+## Tasks
+
+Current tasks from cline-bench (12 total):
+
+1. **every-plugin-api-migration** - Migrate API calls in plugin
+2. **police-sync-segfault** - Fix segmentation fault
+3. **intercept-axios-error-handling** - Fix Axios error handling
+4. **telegram-plugin-refactor** - Refactor Telegram plugin
+5. **discord-trivia-approval-keyerror** - Fix KeyError in Discord bot
+6. **terraform-azurerm-deployment-stacks** - Terraform provider fix
+7. **orpc-client-migration** - Client migration task
+8. **v-edit-workspace-tests** - Fix workspace tests
+9. **healthchain-prefetch-removal** - Remove prefetch logic
+10. **aenet-pytorch-pbc-neighborlist** - PyTorch PBC fix
+11. **suave-http-data-bleeding** - Fix HTTP data bleeding
+12. **filmarchiver** - Film archiver fixes
+
+## CI Integration
+
+These tests run nightly (not on every PR) due to:
+- Long execution time (20-30 min per task)
+- API costs (~$1-5 per run depending on model)
+- Docker/Daytona infrastructure requirements
+
+See `.github/workflows/nightly-evals.yml` for CI configuration.
+
+## Results
+
+Results are written to `evals/cline-bench/jobs/` directory by Harbor:
+
+```
+jobs/
+└── 2025-01-25__10-00-00/
+    ├── result.json              # Aggregate results
+    └── <task-id>__<hash>/
+        ├── result.json          # Trial result
+        ├── agent/cline.txt      # Conversation log
+        └── verifier/reward.txt  # 1 (pass) or 0 (fail)
+```
+
+## Troubleshooting
+
+### "Harbor not found"
+
+```bash
+source .venv/bin/activate  # If using venv
+uv tool install harbor
+```
+
+### "Docker not available"
+
+```bash
+# Start Docker daemon
+docker info  # Should show Docker info
+```
+
+### Task timeouts
+
+Some tasks (Qt WASM, Android) can take 20-30 minutes. If running locally, ensure Docker has sufficient resources (8GB+ RAM).

evals/e2e/run-cline-bench.ts

@@ -0,0 +1,334 @@
+#!/usr/bin/env npx tsx
+/**
+ * cline-bench Runner
+ *
+ * Runs real-world tasks from cline-bench using Harbor framework.
+ * Designed for nightly CI execution.
+ *
+ * Prerequisites:
+ *   - Python 3.13 with uv
+ *   - Harbor installed (`uv tool install harbor`)
+ *   - Docker (for local) or DAYTONA_API_KEY (for cloud)
+ *
+ * Usage:
+ *   npx tsx evals/e2e/run-cline-bench.ts [options]
+ *
+ * Options:
+ *   --env <docker|daytona>  Execution environment (default: docker)
+ *   --provider <name>       Provider to use (default: anthropic)
+ *   --model <id>            Model ID (default: claude-sonnet-4-20250514)
+ *   --tasks <pattern>       Task filter pattern (default: all)
+ *   --trials <n>            Number of trials per task (default: 1)
+ *   --output <file>         Write results to JSON file
+ */
+
+import { execSync, spawnSync } from "child_process"
+import * as fs from "fs"
+import * as path from "path"
+
+interface RunOptions {
+	env: "docker" | "daytona"
+	provider: string
+	model: string
+	tasks: string
+	trials: number
+	outputFile?: string
+}
+
+// Provider configurations for Harbor model format
+const PROVIDER_MODEL_PREFIX: Record<string, string> = {
+	anthropic: "anthropic",
+	openrouter: "openrouter",
+	openai: "openai-native",
+	gemini: "gemini", // Needs different handling
+}
+
+const PROVIDER_API_KEY_ENV: Record<string, string> = {
+	anthropic: "ANTHROPIC_API_KEY",
+	openrouter: "OPENROUTER_API_KEY",
+	openai: "OPENAI_API_KEY",
+	gemini: "GEMINI_API_KEY",
+}
+
+function checkPrerequisites(): { ok: boolean; error?: string } {
+	// Check Python
+	try {
+		const pythonVersion = execSync("python3 --version", { encoding: "utf-8" })
+		if (!pythonVersion.includes("3.13")) {
+			console.warn(`Warning: Python 3.13 recommended, found: ${pythonVersion.trim()}`)
+		}
+	} catch {
+		return { ok: false, error: "Python 3 not found" }
+	}
+
+	// Check Harbor
+	try {
+		execSync("which harbor", { encoding: "utf-8" })
+	} catch {
+		return { ok: false, error: "Harbor not found. Install with: uv tool install harbor" }
+	}
+
+	// Check Docker (for local env)
+	try {
+		execSync("docker info > /dev/null 2>&1")
+	} catch {
+		console.warn("Warning: Docker not available. Use --env daytona for cloud execution.")
+	}
+
+	return { ok: true }
+}
+
+function getTaskList(clineBenchDir: string, filter?: string): string[] {
+	const tasksDir = path.join(clineBenchDir, "tasks")
+	if (!fs.existsSync(tasksDir)) {
+		throw new Error(`Tasks directory not found: ${tasksDir}`)
+	}
+
+	let tasks = fs
+		.readdirSync(tasksDir, { withFileTypes: true })
+		.filter((d) => d.isDirectory())
+		.map((d) => d.name)
+
+	if (filter && filter !== "all") {
+		tasks = tasks.filter((t) => t.includes(filter))
+	}
+
+	return tasks
+}
+
+interface TaskResult {
+	taskId: string
+	passed: boolean
+	duration_sec: number
+	error?: string
+}
+
+function runHarborTask(clineBenchDir: string, taskId: string, options: RunOptions): TaskResult {
+	const startTime = Date.now()
+
+	// Build Harbor model string
+	const modelPrefix = PROVIDER_MODEL_PREFIX[options.provider] || options.provider
+	const harborModel = `${modelPrefix}:${options.model}`
+
+	// Set up environment
+	const apiKeyEnv = PROVIDER_API_KEY_ENV[options.provider]
+	const apiKey = process.env[apiKeyEnv] || process.env.API_KEY
+
+	if (!apiKey) {
+		return {
+			taskId,
+			passed: false,
+			duration_sec: 0,
+			error: `Missing API key: ${apiKeyEnv} or API_KEY`,
+		}
+	}
+
+	const harborEnv = {
+		...process.env,
+		API_KEY: apiKey,
+	}
+
+	// Build Harbor command
+	const harborArgs = ["run", "-p", `tasks/${taskId}`, "-a", "cline-cli", "-m", harborModel, "--env", options.env]
+
+	console.log(`  Running: harbor ${harborArgs.join(" ")}`)
+
+	try {
+		const result = spawnSync("harbor", harborArgs, {
+			cwd: clineBenchDir,
+			env: harborEnv,
+			stdio: ["inherit", "pipe", "pipe"],
+			timeout: 30 * 60 * 1000, // 30 minutes
+		})
+
+		const duration_sec = (Date.now() - startTime) / 1000
+
+		if (result.status !== 0) {
+			return {
+				taskId,
+				passed: false,
+				duration_sec,
+				error: result.stderr?.toString() || `Exit code: ${result.status}`,
+			}
+		}
+
+		// Check if task passed by looking at the latest job results
+		// Harbor writes results to jobs/ directory
+		const jobsDir = path.join(clineBenchDir, "jobs")
+		if (fs.existsSync(jobsDir)) {
+			const latestJob = fs
+				.readdirSync(jobsDir)
+				.filter((d) => d.startsWith("2"))
+				.sort()
+				.pop()
+
+			if (latestJob) {
+				const jobDir = path.join(jobsDir, latestJob)
+				const trialDirs = fs.readdirSync(jobDir).filter((d) => d.includes(taskId.substring(0, 10)))
+
+				for (const trialDir of trialDirs) {
+					const rewardFile = path.join(jobDir, trialDir, "verifier", "reward.txt")
+					if (fs.existsSync(rewardFile)) {
+						const reward = fs.readFileSync(rewardFile, "utf-8").trim()
+						return {
+							taskId,
+							passed: reward === "1",
+							duration_sec,
+						}
+					}
+				}
+			}
+		}
+
+		// Couldn't determine result from files
+		return {
+			taskId,
+			passed: false,
+			duration_sec,
+			error: "Could not determine task result",
+		}
+	} catch (error: any) {
+		return {
+			taskId,
+			passed: false,
+			duration_sec: (Date.now() - startTime) / 1000,
+			error: error.message || String(error),
+		}
+	}
+}
+
+interface BenchmarkReport {
+	timestamp: string
+	provider: string
+	model: string
+	environment: string
+	trialsPerTask: number
+	results: TaskResult[]
+	summary: {
+		total: number
+		passed: number
+		failed: number
+		passRate: number
+	}
+}
+
+async function main() {
+	const args = process.argv.slice(2)
+
+	// Parse arguments
+	const options: RunOptions = {
+		env: "docker",
+		provider: "anthropic",
+		model: "claude-sonnet-4-20250514",
+		tasks: "all",
+		trials: 1,
+	}
+
+	for (let i = 0; i < args.length; i++) {
+		if (args[i] === "--env" && args[i + 1]) {
+			options.env = args[++i] as "docker" | "daytona"
+		} else if (args[i] === "--provider" && args[i + 1]) {
+			options.provider = args[++i]
+		} else if (args[i] === "--model" && args[i + 1]) {
+			options.model = args[++i]
+		} else if (args[i] === "--tasks" && args[i + 1]) {
+			options.tasks = args[++i]
+		} else if (args[i] === "--trials" && args[i + 1]) {
+			options.trials = parseInt(args[++i], 10)
+		} else if (args[i] === "--output" && args[i + 1]) {
+			options.outputFile = args[++i]
+		}
+	}
+
+	// Check prerequisites
+	const prereq = checkPrerequisites()
+	if (!prereq.ok) {
+		console.error(`Prerequisite check failed: ${prereq.error}`)
+		process.exit(1)
+	}
+
+	// Find cline-bench directory
+	const clineBenchDir = path.join(__dirname, "..", "cline-bench")
+	if (!fs.existsSync(clineBenchDir)) {
+		console.error(`cline-bench not found at: ${clineBenchDir}`)
+		console.error("Ensure the submodule is initialized: git submodule update --init")
+		process.exit(1)
+	}
+
+	// Get task list
+	const tasks = getTaskList(clineBenchDir, options.tasks)
+	if (tasks.length === 0) {
+		console.error("No tasks found matching filter:", options.tasks)
+		process.exit(1)
+	}
+
+	console.log(`cline-bench E2E Runner`)
+	console.log(`======================`)
+	console.log(`Provider: ${options.provider}`)
+	console.log(`Model: ${options.model}`)
+	console.log(`Environment: ${options.env}`)
+	console.log(`Tasks: ${tasks.length}`)
+	console.log(`Trials per task: ${options.trials}`)
+	console.log("")
+
+	const results: TaskResult[] = []
+
+	// Run tasks
+	for (const taskId of tasks) {
+		console.log(`\n[${taskId}]`)
+
+		for (let trial = 0; trial < options.trials; trial++) {
+			if (options.trials > 1) {
+				console.log(`  Trial ${trial + 1}/${options.trials}`)
+			}
+
+			const result = runHarborTask(clineBenchDir, taskId, options)
+			results.push(result)
+
+			console.log(`  Result: ${result.passed ? "✓ PASS" : `✗ FAIL: ${result.error || "unknown"}`}`)
+			console.log(`  Duration: ${result.duration_sec.toFixed(1)}s`)
+		}
+	}
+
+	// Generate report
+	const passed = results.filter((r) => r.passed).length
+	const report: BenchmarkReport = {
+		timestamp: new Date().toISOString(),
+		provider: options.provider,
+		model: options.model,
+		environment: options.env,
+		trialsPerTask: options.trials,
+		results,
+		summary: {
+			total: results.length,
+			passed,
+			failed: results.length - passed,
+			passRate: results.length > 0 ? passed / results.length : 0,
+		},
+	}
+
+	// Output
+	if (options.outputFile) {
+		fs.writeFileSync(options.outputFile, JSON.stringify(report, null, 2))
+		console.log(`\nResults written to: ${options.outputFile}`)
+	}
+
+	// Summary
+	console.log("\n" + "=".repeat(60))
+	console.log("SUMMARY")
+	console.log("=".repeat(60))
+	console.log(`Total: ${report.summary.total}`)
+	console.log(`Passed: ${report.summary.passed}`)
+	console.log(`Failed: ${report.summary.failed}`)
+	console.log(`Pass Rate: ${(report.summary.passRate * 100).toFixed(1)}%`)
+
+	// Exit with error if any failures
+	if (report.summary.failed > 0) {
+		process.exit(1)
+	}
+}
+
+main().catch((err) => {
+	console.error(err)
+	process.exit(1)
+})

evals/package.json

@@ -1,48 +1,24 @@
 {
 	"name": "cline-evals",
-	"version": "0.1.0",
-	"description": "Evaluation scripts and tools for Cline",
-	"main": "cli/dist/index.js",
+	"version": "2.0.0",
+	"description": "Evaluation framework for Cline: smoke tests, analysis, and benchmarks",
 	"scripts": {
-		"build:cli": "cd cli && tsc",
-		"start:cli": "cd cli && node dist/index.js",
-		"dev:cli": "cd cli && ts-node src/index.ts",
-		"diff-eval": "./diff-edits/run_and_open_dashboard.sh",
-		"test": "echo \"Error: no test specified\" && exit 1"
+		"analysis": "cd analysis && npm start --",
+		"test:tool-precision": "cd benchmarks/tool-precision/replace-in-file && npm test"
 	},
-	"keywords": [
-		"cline",
-		"evaluation",
-		"benchmark",
-		"diff-edits"
-	],
-	"author": "",
 	"license": "MIT",
 	"dependencies": {
 		"axios": "^1.12.0",
 		"better-sqlite3": "^12.4.1",
 		"chalk": "5.6.2",
-		"dotenv": "^16.5.0",
 		"commander": "^9.4.1",
-		"execa": "^5.1.1",
-		"node-fetch": "^2.7.0",
-		"ora": "^5.4.1",
-		"sqlite": "^4.1.2",
-		"tiktoken": "^1.0.21",
-		"uuid": "^9.0.0",
-		"yargs": "^17.6.2"
+		"dotenv": "^16.5.0",
+		"tiktoken": "^1.0.21"
 	},
 	"devDependencies": {
 		"@types/better-sqlite3": "^7.6.3",
 		"@types/node": "^18.11.18",
-		"@types/node-fetch": "^2.6.12",
-		"@types/uuid": "^9.0.0",
-		"@types/yargs": "^17.0.19",
 		"ts-node": "^10.9.1",
 		"typescript": "^4.9.4"
-	},
-	"overrides": {
-		"tar-fs": "^3.1.1",
-		"js-yaml": "^4.1.1"
 	}
 }

evals/smoke-tests/README.md

@@ -0,0 +1,150 @@
+# Smoke Tests
+
+Curated smoke tests that verify Cline works correctly with LLM providers.
+
+## Purpose
+
+These tests catch regressions in:
+- Tool execution (read, write, edit files)
+- Provider response parsing
+- Tool chaining (multiple operations)
+- Basic code generation
+
+## Quick Start
+
+```bash
+# One-time auth setup
+cline auth
+
+# Build CLI from source (after code changes)
+npm run eval:smoke:build
+
+# Run tests (3 trials by default)
+npm run eval:smoke:run
+
+# Or build + run in one command
+npm run eval:smoke
+```
+
+## Commands
+
+| Command | What it does |
+|---------|--------------|
+| `npm run eval:smoke:build` | Build/install CLI from source |
+| `npm run eval:smoke:run` | Run tests (uses installed CLI) |
+| `npm run eval:smoke` | Build + run (3 trials) |
+| `npm run eval:smoke:ci` | Build + run (1 trial, for CI) |
+
+## Options
+
+```bash
+# Run specific scenario
+npm run eval:smoke:run -- --scenario 01-create-file
+
+# Run with fewer trials (faster)
+npm run eval:smoke:run -- --trials 1
+
+# Run with specific model (overrides any per-scenario models)
+npm run eval:smoke:run -- --model claude-sonnet-4-5-20250929
+```
+
+## Authentication
+
+### Interactive (recommended for local dev)
+
+```bash
+cline auth
+```
+
+### With API key (for automation)
+
+```bash
+cline auth -p cline -k "$CLINE_API_KEY" -m anthropic/claude-sonnet-4.5
+```
+
+## Scenarios
+
+| ID | Name | What it tests |
+|----|------|---------------|
+| 01-create-file | Create a simple file | `write_to_file` |
+| 02-edit-file | Edit existing file | `replace_in_file` |
+| 03-read-summarize | Read and summarize | `read_file` |
+| 04-multi-file | Create multiple files | Multiple tool calls |
+| 05-typescript-function | Generate TypeScript | Code generation |
+| 06-apply-patch | Edit file (GPT-5) | `apply_patch` tool, native tool calling |
+| 07-edit-gemini | Edit file (Gemini) | Gemini model variant |
+
+### Per-Scenario Models
+
+Scenarios can specify their own model(s) via the `models` field in `config.json`. This is useful for testing model-specific code paths like `apply_patch` (GPT-5 only).
+
+If you pass `--model`, it overrides any per-scenario `models` list.
+
+Examples:
+```bash
+# Run apply_patch scenario with its default model (GPT-5)
+npm run eval:smoke:run -- --scenario 06-apply-patch
+
+# Force that scenario to use a specific model
+npm run eval:smoke:run -- --scenario 06-apply-patch --model openai/gpt-4o
+```
+
+## Metrics
+
+- **pass@k**: Probability at least 1 of k trials succeeds
+- **pass^k**: Probability ALL k trials succeed (reliability)
+
+Shows `pass@1` when trials < 3, `pass@3` otherwise.
+
+## Adding New Scenarios
+
+1. Create directory: `scenarios/<name>/`
+2. Add `config.json`:
+   ```json
+   {
+     "name": "Human-readable name",
+     "description": "What this tests",
+     "prompt": "The task prompt for Cline",
+     "expectedFiles": ["file1.txt"],
+     "expectedContent": [
+       { "file": "file1.txt", "contains": "expected text" }
+     ],
+     "timeout": 60
+   }
+   ```
+3. (Optional) Add `template/` directory with starting files
+
+## CI Integration
+
+Smoke tests run automatically via `.github/workflows/cline-evals-regression.yml`.
+
+### Triggers
+
+- Push to `main` (when core code changes)
+- Pull requests
+- Manual dispatch
+
+### Architecture
+
+```
+Build Job (1x)          Test Jobs (5x parallel)       Summarize
+─────────────────       ─────────────────────────     ──────────
+compile-cli             Download artifact             Merge results
+compile-standalone  →   Install CLI               →   Post summary
+Upload artifact         Configure auth
+                        Run single scenario
+```
+
+### Required Secrets
+
+- `CLINE_API_KEY` - Cline API key
+
+### Viewing Results
+
+- Actions tab → "Smoke Tests" workflow
+- View "Summary" for quick results
+- Download "smoke-test-results" artifact for details
+
+## TODO
+
+- [ ] **Native tool calling tests**: Add CLI support for `native_tool_call_enabled` setting, then create a scenario that tests Claude 4 with native tool calling enabled (currently only GPT-5 models automatically use native tools via the Responses API)

evals/smoke-tests/run-smoke-tests.ts

@@ -0,0 +1,679 @@
+#!/usr/bin/env npx tsx
+/**
+ * Smoke Test Runner for Cline
+ *
+ * Runs curated smoke tests against configured providers to verify:
+ * - Basic tool execution works
+ * - Provider responses are correctly parsed
+ * - Thinking traces are preserved
+ *
+ * Usage:
+ *   npx tsx evals/smoke-tests/run-smoke-tests.ts [options]
+ *
+ * Options:
+ *   --provider <name>  Run tests for a specific provider (default: all configured)
+ *   --trials <n>       Number of trials per test (default: 3)
+ *   --scenario <name>  Run a specific scenario (default: all)
+ *   --output <file>    Write JSON results to file
+ */
+
+import { execSync, spawn } from "child_process"
+import * as fs from "fs"
+import * as path from "path"
+import { MetricsCalculator } from "../analysis/src/metrics"
+
+// Default provider and model for smoke tests
+// These ensure deterministic behavior regardless of local config
+const DEFAULT_PROVIDER = "cline"
+const DEFAULT_MODEL = "anthropic/claude-sonnet-4.5"
+
+// Models to test - can be overridden with --model flag
+const MODELS: string[] = [DEFAULT_MODEL]
+
+// Check if cline CLI is available
+function checkClineCli(): boolean {
+	try {
+		execSync("which cline", { encoding: "utf-8", timeout: 5000 })
+		return true
+	} catch {
+		return false
+	}
+}
+
+// Use user's existing Cline config (already has auth configured)
+// For CI, this would be set up by the auth step before tests run
+const CLINE_CONFIG_DIR = path.join(process.env.HOME || "", ".cline")
+
+// Configure authentication using CLINE_API_KEY environment variable
+// Returns success if auth is configured, error message otherwise
+function configureAuth(): { ok: boolean; error?: string } {
+	const apiKey = process.env.CLINE_API_KEY
+	if (!apiKey) {
+		return {
+			ok: false,
+			error: "CLINE_API_KEY environment variable not set",
+		}
+	}
+
+	// Ensure config directory exists
+	fs.mkdirSync(CLINE_CONFIG_DIR, { recursive: true })
+
+	try {
+		// Run quick auth setup (non-interactive when all flags provided)
+		execSync(`cline auth --config "${CLINE_CONFIG_DIR}" -p ${DEFAULT_PROVIDER} -k "${apiKey}" -m "${DEFAULT_MODEL}"`, {
+			encoding: "utf-8",
+			timeout: 10000,
+			stdio: "pipe",
+		})
+		return { ok: true }
+	} catch (err: any) {
+		return {
+			ok: false,
+			error: err.message || "Auth command failed",
+		}
+	}
+}
+
+// Smoke test scenario definition
+interface SmokeScenario {
+	id: string
+	name: string
+	description: string
+	prompt: string
+	workdir: string // Relative to scenario directory
+	expectedFiles?: string[] // Files that should exist after
+	expectedContent?: { file: string; contains: string }[] // Content checks
+	timeout: number // Seconds
+	models?: string[] // Optional: override default models for this scenario
+}
+
+// Load scenarios from disk
+function loadScenarios(scenariosDir: string): SmokeScenario[] {
+	const scenarios: SmokeScenario[] = []
+
+	for (const entry of fs.readdirSync(scenariosDir, { withFileTypes: true })) {
+		if (entry.isDirectory()) {
+			const configPath = path.join(scenariosDir, entry.name, "config.json")
+			if (fs.existsSync(configPath)) {
+				const config = JSON.parse(fs.readFileSync(configPath, "utf-8"))
+				scenarios.push({
+					...config,
+					id: entry.name,
+					workdir: path.join(scenariosDir, entry.name, "workspace"),
+				})
+			}
+		}
+	}
+
+	return scenarios
+}
+
+// Run a single trial
+interface TrialResult {
+	passed: boolean
+	error?: string
+	durationMs: number
+	stdout: string
+	stderr: string
+}
+
+async function runTrial(scenario: SmokeScenario, modelId: string, trialWorkdir: string): Promise<TrialResult> {
+	const startTime = Date.now()
+
+	// Ensure workspace exists and is clean
+	if (fs.existsSync(trialWorkdir)) {
+		fs.rmSync(trialWorkdir, { recursive: true })
+	}
+	fs.mkdirSync(trialWorkdir, { recursive: true })
+
+	// Copy any template files from scenario
+	const templateDir = path.join(path.dirname(scenario.workdir), "template")
+	if (fs.existsSync(templateDir)) {
+		fs.cpSync(templateDir, trialWorkdir, { recursive: true })
+	}
+
+	// Build CLI command with explicit model setting for determinism
+	// Provider is configured via `cline auth` before running tests
+	const args = [
+		"--config",
+		CLINE_CONFIG_DIR, // Use shared config directory for auth
+		"-y", // YOLO mode - auto-approve all actions, exits after completion
+		"-t",
+		String(scenario.timeout), // CLI timeout (matches our timeout)
+		"-m",
+		modelId, // Model to use (overrides configured default)
+		scenario.prompt,
+	]
+
+	try {
+		// Run cline CLI
+		const result = await runClineWithTimeout(args, trialWorkdir, scenario.timeout * 1000)
+
+		if (!result.success) {
+			return {
+				passed: false,
+				error: result.error || "CLI execution failed",
+				durationMs: Date.now() - startTime,
+				stdout: result.stdout,
+				stderr: result.stderr,
+			}
+		}
+
+		// Verify expected files
+		if (scenario.expectedFiles) {
+			for (const file of scenario.expectedFiles) {
+				const filePath = path.join(trialWorkdir, file)
+				if (!fs.existsSync(filePath)) {
+					return {
+						passed: false,
+						error: `Expected file not found: ${file}`,
+						durationMs: Date.now() - startTime,
+						stdout: result.stdout,
+						stderr: result.stderr,
+					}
+				}
+			}
+		}
+
+		// Verify expected content
+		if (scenario.expectedContent) {
+			for (const check of scenario.expectedContent) {
+				const filePath = path.join(trialWorkdir, check.file)
+				if (!fs.existsSync(filePath)) {
+					return {
+						passed: false,
+						error: `File not found for content check: ${check.file}`,
+						durationMs: Date.now() - startTime,
+						stdout: result.stdout,
+						stderr: result.stderr,
+					}
+				}
+				const content = fs.readFileSync(filePath, "utf-8")
+				if (!content.includes(check.contains)) {
+					return {
+						passed: false,
+						error: `Expected content not found in ${check.file}: "${check.contains}"`,
+						durationMs: Date.now() - startTime,
+						stdout: result.stdout,
+						stderr: result.stderr,
+					}
+				}
+			}
+		}
+
+		return {
+			passed: true,
+			durationMs: Date.now() - startTime,
+			stdout: result.stdout,
+			stderr: result.stderr,
+		}
+	} catch (error: any) {
+		return {
+			passed: false,
+			error: error.message || String(error),
+			durationMs: Date.now() - startTime,
+			stdout: "",
+			stderr: "",
+		}
+	}
+}
+
+// Run cline CLI with timeout
+interface ClineResult {
+	success: boolean
+	error?: string
+	stdout: string
+	stderr: string
+}
+
+function runClineWithTimeout(args: string[], cwd: string, timeoutMs: number): Promise<ClineResult> {
+	return new Promise((resolve) => {
+		let stdout = ""
+		let stderr = ""
+
+		const proc = spawn("cline", args, {
+			cwd,
+			env: { ...process.env },
+			stdio: ["ignore", "pipe", "pipe"], // stdin: ignore, stdout/stderr: pipe
+		})
+
+		const timeout = setTimeout(() => {
+			proc.kill("SIGKILL")
+			resolve({
+				success: false,
+				error: "Timeout exceeded",
+				stdout,
+				stderr,
+			})
+		}, timeoutMs)
+
+		proc.stdout?.on("data", (data) => {
+			stdout += data.toString()
+		})
+
+		proc.stderr?.on("data", (data) => {
+			stderr += data.toString()
+		})
+
+		proc.on("error", (err) => {
+			clearTimeout(timeout)
+			resolve({
+				success: false,
+				error: err.message,
+				stdout,
+				stderr,
+			})
+		})
+
+		proc.on("close", (code) => {
+			clearTimeout(timeout)
+			let error: string | undefined
+			if (code !== 0) {
+				// Include last line of stderr for context
+				const lastStderr = stderr.trim().split("\n").slice(-3).join(" | ")
+				error = `Exit code: ${code}${lastStderr ? ` - ${lastStderr}` : ""}`
+			}
+			resolve({
+				success: code === 0,
+				error,
+				stdout,
+				stderr,
+			})
+		})
+	})
+}
+
+// Result types
+interface ScenarioResult {
+	scenarioId: string
+	scenarioName: string
+	model: string
+	modelId: string
+	trials: TrialResult[]
+	metrics: {
+		passAt1: number
+		passAt3: number
+		passCaret3: number
+		flakinessScore: number
+	}
+	status: "pass" | "fail" | "flaky"
+}
+
+interface SmokeTestReport {
+	timestamp: string
+	provider: string
+	models: string[]
+	scenarios: string[]
+	trialsPerTest: number
+	results: ScenarioResult[]
+	summary: {
+		total: number
+		passed: number
+		failed: number
+		flaky: number
+		passAt1Overall: number
+		passAt3Overall: number
+	}
+}
+
+// Main execution
+async function main() {
+	const args = process.argv.slice(2)
+
+	// Parse arguments
+	let selectedModel: string | undefined
+	let trials = 3
+	let selectedScenario: string | undefined
+	let outputFile: string | undefined
+	let parallel = false
+	let parallelLimit = 4
+
+	for (let i = 0; i < args.length; i++) {
+		if (args[i] === "--model" && args[i + 1]) {
+			selectedModel = args[++i]
+		} else if (args[i] === "--trials" && args[i + 1]) {
+			trials = parseInt(args[++i], 10)
+		} else if (args[i] === "--scenario" && args[i + 1]) {
+			selectedScenario = args[++i]
+		} else if (args[i] === "--output" && args[i + 1]) {
+			outputFile = args[++i]
+		} else if (args[i] === "--parallel") {
+			parallel = true
+			if (args[i + 1] && !args[i + 1].startsWith("--")) {
+				parallelLimit = parseInt(args[++i], 10)
+			}
+		}
+	}
+
+	// Check cline CLI is available
+	if (!checkClineCli()) {
+		console.error("ERROR: cline CLI not found in PATH")
+		console.error("")
+		console.error("For local development:")
+		console.error("  cd cli && npm install && npm run build && npm link")
+		console.error("")
+		console.error("For CI:")
+		console.error("  Ensure CLI build and 'npm link' steps completed")
+		process.exit(1)
+	}
+
+	// Configure authentication if CLINE_API_KEY is set
+	// Otherwise use existing auth from ~/.cline
+	if (process.env.CLINE_API_KEY) {
+		console.log("Configuring authentication from CLINE_API_KEY...")
+		const authResult = configureAuth()
+		if (!authResult.ok) {
+			console.error("")
+			console.error("ERROR: Authentication failed")
+			console.error(`  ${authResult.error}`)
+			console.error("")
+			process.exit(1)
+		}
+		console.log("Authentication configured")
+	} else {
+		console.log("Using existing authentication from ~/.cline")
+	}
+	console.log("")
+
+	// Load scenarios
+	const scenariosDir = path.join(__dirname, "scenarios")
+	let scenarios = loadScenarios(scenariosDir)
+
+	if (scenarios.length === 0) {
+		console.error("No scenarios found in", scenariosDir)
+		process.exit(1)
+	}
+
+	if (selectedScenario) {
+		scenarios = scenarios.filter((s) => s.id === selectedScenario)
+		if (scenarios.length === 0) {
+			console.error(`Scenario not found: ${selectedScenario}`)
+			process.exit(1)
+		}
+	}
+
+	// Filter models
+	let models = MODELS
+	if (selectedModel) {
+		models = [selectedModel]
+	}
+
+	// Create results directory with timestamp
+	const timestamp = new Date().toISOString().replace(/[:.]/g, "-")
+	const resultsBaseDir = path.join(__dirname, "results")
+	const resultsDir = path.join(resultsBaseDir, timestamp)
+	fs.mkdirSync(resultsDir, { recursive: true })
+
+	// Models are now always explicit
+	const resolvedModels = models
+
+	console.log(`Running ${scenarios.length} scenarios × ${models.length} models × ${trials} trials`)
+	console.log(`Provider: ${DEFAULT_PROVIDER}`)
+	console.log(`Models: ${resolvedModels.join(", ")}`)
+	console.log(`Scenarios: ${scenarios.map((s) => s.id).join(", ")}`)
+	console.log(`Results: ${resultsDir}`)
+	console.log(`Parallel: ${parallel ? `yes (limit: ${parallelLimit})` : "no"}`)
+	console.log("")
+
+	const metricsCalc = new MetricsCalculator()
+	const results: ScenarioResult[] = []
+
+	// Build list of all scenario+model combinations
+	interface TestJob {
+		scenario: Scenario
+		modelId: string
+	}
+	const jobs: TestJob[] = []
+	for (const scenario of scenarios) {
+		const scenarioModels = selectedModel ? [selectedModel] : scenario.models || models
+		for (const modelId of scenarioModels) {
+			jobs.push({ scenario, modelId })
+		}
+	}
+
+	// Run a single job
+	async function runJob(job: TestJob): Promise<ScenarioResult> {
+		const { scenario, modelId } = job
+		const logDir = path.join(resultsDir, scenario.id, modelId)
+		fs.mkdirSync(logDir, { recursive: true })
+
+		const trialResults: TrialResult[] = []
+		const trialWorkdirs: string[] = []
+
+		for (let t = 0; t < trials; t++) {
+			const trialWorkdir = path.join(logDir, `workspace-trial-${t + 1}`)
+			trialWorkdirs.push(trialWorkdir)
+			const result = await runTrial(scenario, modelId, trialWorkdir)
+			trialResults.push(result)
+		}
+
+		trialResults.forEach((result, t) => {
+			const trialNum = t + 1
+			const logContent =
+				`# Trial ${trialNum}\n` +
+				`Status: ${result.passed ? "PASS" : "FAIL"}\n` +
+				`Duration: ${result.durationMs}ms\n` +
+				(result.error ? `Error: ${result.error}\n` : "") +
+				`\n## STDOUT\n${result.stdout || "(empty)"}\n` +
+				`\n## STDERR\n${result.stderr || "(empty)"}\n`
+			fs.writeFileSync(path.join(logDir, `trial-${trialNum}.log`), logContent)
+		})
+
+		const trialBools = trialResults.map((t) => t.passed)
+		const metrics = metricsCalc.calculateTaskMetrics(trialBools)
+		const status = metricsCalc.getTaskStatus(trialBools)
+
+		return {
+			scenarioId: scenario.id,
+			scenarioName: scenario.name,
+			model: modelId,
+			modelId: modelId,
+			trials: trialResults,
+			metrics,
+			status,
+		}
+	}
+
+	if (parallel) {
+		// Run jobs in parallel with concurrency limit
+		console.log(`Running ${jobs.length} jobs in parallel...`)
+		const executing: Promise<void>[] = []
+
+		for (const job of jobs) {
+			const p = runJob(job).then((result) => {
+				results.push(result)
+				const passMetric = trials >= 3 ? result.metrics.passAt3 : result.metrics.passAt1
+				const icon = result.status === "pass" ? "✓" : result.status === "flaky" ? "~" : "✗"
+				console.log(
+					`  ${icon} [${result.scenarioId}] ${result.model}: ${result.status.toUpperCase()} (${(passMetric * 100).toFixed(0)}%)`,
+				)
+			})
+			executing.push(p as unknown as Promise<void>)
+
+			if (executing.length >= parallelLimit) {
+				await Promise.race(executing)
+				// Remove settled promises
+				for (let i = executing.length - 1; i >= 0; i--) {
+					const settled = await Promise.race([executing[i].then(() => true).catch(() => true), Promise.resolve(false)])
+					if (settled) executing.splice(i, 1)
+				}
+			}
+		}
+		await Promise.all(executing)
+	} else {
+		// Sequential execution
+		for (const job of jobs) {
+			console.log(`\n[${job.scenario.id}] ${job.scenario.name} (${job.modelId})`)
+			const logDir = path.join(resultsDir, job.scenario.id, job.modelId)
+			fs.mkdirSync(logDir, { recursive: true })
+
+			const trialResults: TrialResult[] = []
+			const trialWorkdirs: string[] = []
+
+			for (let t = 0; t < trials; t++) {
+				const trialWorkdir = path.join(logDir, `workspace-trial-${t + 1}`)
+				trialWorkdirs.push(trialWorkdir)
+				process.stdout.write(`  Trial ${t + 1}/${trials}... `)
+				const result = await runTrial(job.scenario, job.modelId, trialWorkdir)
+				trialResults.push(result)
+				console.log(result.passed ? "✓ PASS" : `✗ FAIL: ${result.error}`)
+			}
+
+			trialResults.forEach((result, t) => {
+				const trialNum = t + 1
+				const logContent =
+					`# Trial ${trialNum}\n` +
+					`Status: ${result.passed ? "PASS" : "FAIL"}\n` +
+					`Duration: ${result.durationMs}ms\n` +
+					(result.error ? `Error: ${result.error}\n` : "") +
+					`\n## STDOUT\n${result.stdout || "(empty)"}\n` +
+					`\n## STDERR\n${result.stderr || "(empty)"}\n`
+				fs.writeFileSync(path.join(logDir, `trial-${trialNum}.log`), logContent)
+			})
+
+			const trialBools = trialResults.map((t) => t.passed)
+			const metrics = metricsCalc.calculateTaskMetrics(trialBools)
+			const status = metricsCalc.getTaskStatus(trialBools)
+
+			results.push({
+				scenarioId: job.scenario.id,
+				scenarioName: job.scenario.name,
+				model: job.modelId,
+				modelId: job.modelId,
+				trials: trialResults,
+				metrics,
+				status,
+			})
+
+			// Display pass@k where k = actual trials (pass@3 is meaningless with fewer trials)
+			const passMetric = trials >= 3 ? metrics.passAt3 : metrics.passAt1
+			const passLabel = trials >= 3 ? "pass@3" : "pass@1"
+			console.log(`  Result: ${status.toUpperCase()} | ${passLabel}: ${(passMetric * 100).toFixed(0)}%`)
+		}
+	}
+
+	// Generate report
+	const report: SmokeTestReport = {
+		timestamp: new Date().toISOString(),
+		provider: DEFAULT_PROVIDER,
+		models: resolvedModels,
+		scenarios: scenarios.map((s) => s.id),
+		trialsPerTest: trials,
+		results,
+		summary: {
+			total: results.length,
+			passed: results.filter((r) => r.status === "pass").length,
+			failed: results.filter((r) => r.status === "fail").length,
+			flaky: results.filter((r) => r.status === "flaky").length,
+			passAt1Overall: results.length > 0 ? results.reduce((sum, r) => sum + r.metrics.passAt1, 0) / results.length : 0,
+			passAt3Overall: results.length > 0 ? results.reduce((sum, r) => sum + r.metrics.passAt3, 0) / results.length : 0,
+		},
+	}
+
+	// Save report.json
+	fs.writeFileSync(path.join(resultsDir, "report.json"), JSON.stringify(report, null, 2))
+
+	// Generate summary.md for CI job summary
+	const summaryMd = generateSummaryMarkdown(report)
+	fs.writeFileSync(path.join(resultsDir, "summary.md"), summaryMd)
+
+	// Create/update 'latest' symlink
+	const latestLink = path.join(resultsBaseDir, "latest")
+	try {
+		if (fs.existsSync(latestLink)) {
+			fs.unlinkSync(latestLink)
+		}
+		fs.symlinkSync(timestamp, latestLink)
+	} catch {
+		// Symlinks may fail on some systems, ignore
+	}
+
+	// Also write to custom output file if specified
+	if (outputFile) {
+		fs.writeFileSync(outputFile, JSON.stringify(report, null, 2))
+		console.log(`\nResults also written to: ${outputFile}`)
+	}
+
+	// Summary
+	console.log("\n" + "=".repeat(60))
+	console.log("SUMMARY")
+	console.log("=".repeat(60))
+	console.log(`Total: ${report.summary.total}`)
+	console.log(`Passed: ${report.summary.passed}`)
+	console.log(`Failed: ${report.summary.failed}`)
+	console.log(`Flaky: ${report.summary.flaky}`)
+	const passLabel = report.trialsPerTest >= 3 ? "pass@3" : "pass@1"
+	const passOverall = report.trialsPerTest >= 3 ? report.summary.passAt3Overall : report.summary.passAt1Overall
+	console.log(`Overall ${passLabel}: ${(passOverall * 100).toFixed(1)}%`)
+	console.log(`\nFull results: ${resultsDir}`)
+	console.log(`Latest link: ${latestLink}`)
+
+	// Exit with error if any failures
+	if (report.summary.failed > 0) {
+		process.exit(1)
+	}
+}
+
+// Generate markdown summary for CI
+function generateSummaryMarkdown(report: SmokeTestReport): string {
+	const lines: string[] = []
+	lines.push("## Smoke Test Results")
+	lines.push("")
+	lines.push(`**Date:** ${report.timestamp}`)
+
+	// Show unique model IDs from results
+	const modelIds = [...new Set(report.results.map((r) => r.modelId))]
+	lines.push(`**Models:** ${modelIds.join(", ")}`)
+	lines.push(`**Trials per test:** ${report.trialsPerTest}`)
+	lines.push("")
+	lines.push("### Summary")
+	lines.push("")
+	lines.push(`| Metric | Value |`)
+	lines.push(`|--------|-------|`)
+	lines.push(`| Total | ${report.summary.total} |`)
+	lines.push(`| Passed | ${report.summary.passed} |`)
+	lines.push(`| Failed | ${report.summary.failed} |`)
+	lines.push(`| Flaky | ${report.summary.flaky} |`)
+	const mdPassLabel = report.trialsPerTest >= 3 ? "pass@3" : "pass@1"
+	const mdPassOverall = report.trialsPerTest >= 3 ? report.summary.passAt3Overall : report.summary.passAt1Overall
+	lines.push(`| Overall ${mdPassLabel} | ${(mdPassOverall * 100).toFixed(1)}% |`)
+	lines.push("")
+
+	// Results table
+	lines.push("### Results by Scenario")
+	lines.push("")
+	lines.push(`| Scenario | Model | Status | ${mdPassLabel} |`)
+	lines.push("|----------|-------|--------|--------|")
+	for (const r of report.results) {
+		const statusEmoji = r.status === "pass" ? "✅" : r.status === "flaky" ? "⚠️" : "❌"
+		const rPassMetric = report.trialsPerTest >= 3 ? r.metrics.passAt3 : r.metrics.passAt1
+		lines.push(
+			`| ${r.scenarioId} | ${r.modelId} | ${statusEmoji} ${r.status.toUpperCase()} | ${(rPassMetric * 100).toFixed(0)}% |`,
+		)
+	}
+	lines.push("")
+
+	// Failed/flaky details
+	const problemResults = report.results.filter((r) => r.status !== "pass")
+	if (problemResults.length > 0) {
+		lines.push("### Failed/Flaky Details")
+		lines.push("")
+		for (const r of problemResults) {
+			lines.push(`#### ${r.scenarioId} (${r.modelId})`)
+			lines.push("")
+			r.trials.forEach((t, i) => {
+				if (!t.passed) {
+					lines.push(`- Trial ${i + 1}: ${t.error}`)
+				}
+			})
+			lines.push("")
+		}
+	}
+
+	return lines.join("\n")
+}
+
+main().catch((err) => {
+	console.error(err)
+	process.exit(1)
+})

evals/smoke-tests/scenarios/01-create-file/config.json

@@ -0,0 +1,15 @@
+{
+	"name": "Create a simple file",
+	"description": "Tests basic file creation with write_to_file tool",
+	"prompt": "Create a file called hello.txt that contains the text 'Hello, World!'",
+	"expectedFiles": [
+		"hello.txt"
+	],
+	"expectedContent": [
+		{
+			"file": "hello.txt",
+			"contains": "Hello"
+		}
+	],
+	"timeout": 120
+}

evals/smoke-tests/scenarios/02-edit-file/config.json

@@ -0,0 +1,15 @@
+{
+	"name": "Edit an existing file",
+	"description": "Tests file editing with replace_in_file tool",
+	"prompt": "Edit the file greeting.txt and change 'Hello' to 'Goodbye'",
+	"expectedFiles": [
+		"greeting.txt"
+	],
+	"expectedContent": [
+		{
+			"file": "greeting.txt",
+			"contains": "Goodbye"
+		}
+	],
+	"timeout": 120
+}

evals/smoke-tests/scenarios/02-edit-file/template/greeting.txt

@@ -0,0 +1 @@
+Hello World, this is a greeting file.

evals/smoke-tests/scenarios/03-read-summarize/config.json

@@ -0,0 +1,15 @@
+{
+	"name": "Read and create summary",
+	"description": "Tests read_file tool and response generation",
+	"prompt": "Read the file data.txt and create a summary.txt file that contains a one-sentence summary of what the data file contains",
+	"expectedFiles": [
+		"summary.txt"
+	],
+	"expectedContent": [
+		{
+			"file": "summary.txt",
+			"contains": "user"
+		}
+	],
+	"timeout": 180
+}

evals/smoke-tests/scenarios/03-read-summarize/template/data.txt

@@ -0,0 +1,11 @@
+User Analytics Report
+=====================
+Total users: 1,523
+Active users (last 30 days): 892
+New signups this month: 147
+Churn rate: 3.2%
+
+Top features by usage:
+1. Dashboard (89%)
+2. Reports (67%)
+3. Settings (45%)

evals/smoke-tests/scenarios/04-multi-file/config.json

@@ -0,0 +1,20 @@
+{
+	"name": "Create multiple files",
+	"description": "Tests tool chaining with multiple file operations",
+	"prompt": "Create a simple project with two files: index.html with a basic HTML structure, and style.css with a simple body style that sets the font family to Arial",
+	"expectedFiles": [
+		"index.html",
+		"style.css"
+	],
+	"expectedContent": [
+		{
+			"file": "index.html",
+			"contains": "html"
+		},
+		{
+			"file": "style.css",
+			"contains": "Arial"
+		}
+	],
+	"timeout": 180
+}

evals/smoke-tests/scenarios/05-typescript-function/config.json

@@ -0,0 +1,19 @@
+{
+	"name": "Write TypeScript function",
+	"description": "Tests code generation with correct syntax",
+	"prompt": "Create a file called utils.ts that exports a function called 'capitalize' that takes a string and returns it with the first letter capitalized",
+	"expectedFiles": [
+		"utils.ts"
+	],
+	"expectedContent": [
+		{
+			"file": "utils.ts",
+			"contains": "export"
+		},
+		{
+			"file": "utils.ts",
+			"contains": "capitalize"
+		}
+	],
+	"timeout": 180
+}