fix types and linter errors:

* move apiConversationHistory to MessageStateHandler

* move clineMessages state to MessageStateManager class

* remove unused imports

* reorganize task class state variables and refactor out utility functions in recursivelyMakeClineRequests

* move task ephemeral state to state class

* extract tool logic into tool executor class

* integrate ToolExecutor into task class

.changeset/angry-terms-mate.md

@@ -1,5 +0,0 @@
----
-"claude-dev": patch
----
-
-updating o3 model pricing

.changeset/calm-snails-hammer.md

@@ -1,5 +0,0 @@
----
-"claude-dev": patch
----
-
-Sorting mcp marketplace by newest listings by default

.changeset/neat-guests-itch.md

@@ -1,5 +0,0 @@
----
-"claude-dev": patch
----
-
-fix(bedrock): remove Anthropic-Bedrock SDK

.changeset/purple-islands-move.md

@@ -1,5 +0,0 @@
----
-"claude-dev": patch
----
-
-Remove ‘-beta’ from grok model id

.changeset/quick-moons-do.md

@@ -0,0 +1,5 @@
+---
+"claude-dev": minor
+---
+
+Migrate authStateChanged to protobus

.changeset/seven-insects-shop.md

@@ -1,5 +0,0 @@
----
-"claude-dev": patch
----
-
-Add prompt caching indicator to grok 3

.eslintrc.json

@@ -21,7 +21,14 @@
 		"semi": "off",
 		"react-hooks/exhaustive-deps": "off",
 		"eslint-rules/no-protobuf-object-literals": "error",
-		"eslint-rules/no-grpc-client-object-literals": "error"
+		"eslint-rules/no-grpc-client-object-literals": "error",
+		"no-restricted-syntax": [
+			"error",
+			{
+				"selector": "VariableDeclarator[id.type=\"ObjectPattern\"][init.object.name=\"process\"][init.property.name=\"env\"]",
+				"message": "Use process.env.VARIABLE_NAME directly instead of destructuring"
+			}
+		]
 	},
 	"ignorePatterns": ["out", "dist", "**/*.d.ts"]
 }

.github/ISSUE_TEMPLATE/config.yml

@@ -1,4 +1,4 @@
-blank_issues_enabled: true
+blank_issues_enabled: false
 contact_links:
     - name: ✨ Feature Request
       url: https://github.com/cline/cline/discussions/categories/feature-requests?discussions_q=is%3Aopen+category%3A%22Feature+Requests%22+sort%3Atop
@@ -6,6 +6,3 @@ contact_links:
     - name: 👋 Cline Discord
       url: https://discord.gg/cline
       about: Join our Discord community for discussions and support
-    - name: ❓ Other Questions?
-      url: https://x.com/sdrzn
-      about: Contact the developer on X @sdrzn for other inquiries

.github/ISSUE_TEMPLATE/feature_proposal.yml

@@ -0,0 +1,130 @@
+name: 📝 Detailed Feature Proposal
+description: Propose a new feature or improvement
+labels: ["proposal"]
+body:
+    - type: markdown
+      attributes:
+          value: |
+              **Feature Proposal for Cline**
+
+              Thank you for creating a feature proposal for Cline! This template is for clear, actionable proposals that define a specific problem and a high-confidence solution. Please provide enough detail to enable fast prioritization, discussion, and execution.
+
+              Detailed proposals will be prioritized, while vague proposals may be closed or require extensive back and forth communication.
+
+              Before submitting:
+              - Search existing [Issues](https://github.com/cline/cline/issues) and [Discussions](https://github.com/cline/cline/discussions) to avoid duplicates
+              - Don’t start implementation until the proposal is reviewed and approved
+
+    - type: textarea
+      id: problem-description
+      attributes:
+          label: What problem does this solve?
+          description: |
+              Describe the problem clearly from a user's point of view. Focus on why this matters, who it affects, and when it occurs.
+
+              ✅ Solid:
+              - "LLM provider returns 400 error when nearing the context window instead of truncating"
+              - "Submit button is invisible in dark mode"
+
+              ❌ Avoid:
+              - "Performance is bad"
+              - "UI needs work"
+
+              Your description should include:
+              - Who is affected?
+              - When does it happen?
+              - What's the current vs expected behavior?
+              - What is the impact?
+          placeholder: Be specific about the problem, who it affects, and the impact.
+      validations:
+          required: true
+
+    - type: textarea
+      id: proposed-solution
+      attributes:
+          label: What’s the proposed solution?
+          description: |
+              Describe how the problem should be solved. Be specific about UX, system behavior, and any flows that would change.
+
+              ✅ Solid:
+              - "Add error handling immediately after attempting to create the llm stream and retry after manually truncating"
+              - "Update button styling to ensure contrast in all themes"
+
+              ❌ Avoid:
+              - "Improve performance"
+              - "Fix the bug"
+
+              Your solution should include:
+              - What exactly will change?
+              - How will users interact with it?
+              - What’s the expected outcome?
+          placeholder: Describe the proposed changes and how they solve the problem.
+      validations:
+          required: true
+
+    - type: textarea
+      id: acceptance-criteria
+      attributes:
+          label: How will we know it works? (Acceptance Criteria)
+          description: Define clear, testable success criteria.
+          placeholder: Provide specific and testable conditions for success.
+      validations:
+          required: true
+
+    - type: textarea
+      id: estimated-effort
+      attributes:
+          label: Estimated effort and complexity
+          description: |
+              Help us understand scope and risks. Include:
+
+              - Size estimate (XS/S/M/L/XL or hours/days)
+              - Why this size? What’s technically involved?
+              - Any tricky parts, refactors, or risks?
+              - Performance or compatibility concerns?
+              - Any dependencies on systems, teams, or libraries?
+          placeholder: Size, reasoning, risks, and dependencies.
+      validations:
+          required: true
+
+    - type: textarea
+      id: technical-considerations
+      attributes:
+          label: Technical considerations, tradeoffs, and/or risks (optional)
+          description: |
+              Include any technical context that helps us evaluate or implement the proposal more effectively.
+
+              You may include:
+              - Architectural changes or required refactors
+              - Performance implications or system-level impacts
+              - Known limitations, risks, or tricky edge cases
+              - Compatibility concerns or migration steps
+              - Alternative approaches you considered and why they were not chosen
+              - Dependencies on other systems, teams, or libraries
+              - Were other approaches considered? Why is this one preferred?
+          placeholder: Technical considerations, tradeoffs, and/or risks.
+
+    - type: textarea
+      id: additional-context
+      attributes:
+          label: Additional context (optional)
+          description: Diagrams, mockups, logs, links, or anything else that helps explain or justify the proposal.
+          placeholder: Diagrams, mockups, logs, links, or anything else that helps explain or justify the proposal.
+
+    - type: checkboxes
+      id: checklist
+      attributes:
+          label: Proposal checklist
+          options:
+              - label: I’ve checked for existing issues or related proposals
+                required: true
+              - label: I understand this needs review before implementation can start
+                required: true
+
+    - type: checkboxes
+      id: willingness-to-contribute
+      attributes:
+          label: Interested in implementing this?
+          description: Optional
+          options:
+              - label: Yes, I’d like to help implement this feature

.github/pull_request_template.md

@@ -1,10 +1,47 @@
+<!--
+Thank you for contributing to Cline!
+
+⚠️ Important: Before submitting this PR, please ensure you have:
+- Opened an issue and discussed your proposed changes with the community / contributors
+- Received approval from a core Cline contributor prior to proceeding with the implementation
+- Link the associated issue in the "Related Issue" section
+
+Limited exceptions:
+Small bug fixes, typo corrections, minor wording improvements, or simple type fixes that don't change functionality may be submitted directly.
+
+Why this requirement?
+We deeply appreciate all community contributions - they are the core reason we're able to operate successfully and keep innovating! We welcome community input and want to make it as easy as possible for people to submit quality work. This process helps our core maintainers review new ideas faster and saves contributor time by ensuring you have the go-ahead before spending time on implementation.
+-->
+
+### Related Issue
+
+<!-- Replace XXXX with the issue number that this PR addresses -->
+**Issue:** #XXXX
+
 ### Description
 
-<!-- Describe your changes in detail. What problem does this PR solve? -->
+<!-- 
+Help reviewers understand your changes by making this PR readable and well-organized:
+
+- What problem does this PR solve?
+- Why were these changes introduced and what purpose do they serve?
+- For larger changes, provide context about your approach and reasoning
+
+Small PRs may need minimal description, but larger changes benefit from explaining where you're coming from. Much of this context can be in the linked issue above, so feel free to reference it rather than repeating everything here.
+-->
 
 ### Test Procedure
 
-<!-- How did you test this? Are you confident that it will not introduce bugs? If so, why? -->
+<!-- 
+Please walk us through your testing approach and thought process. This helps reviewers understand that you've thoroughly considered the impact of your changes:
+
+- How did you test this change?
+- What could potentially break and how did you verify it doesn't?
+- What existing functionality might be affected and how did you check it still works?
+- Why are you confident this is ready for merge?
+
+We're not looking for exhaustive documentation - just evidence that you've thought through the implications of your changes and tested accordingly.
+-->
 
 ### Type of Change
 
@@ -29,7 +66,15 @@
 
 ### Screenshots
 
-<!-- For UI changes, add screenshots here -->
+<!-- 
+Help reviewers quickly understand your changes:
+
+- **UI Changes**: Please include screenshots showing before/after states
+- **Complex Workflows**: Consider uploading a screen recording (video) if your changes involve multiple steps or state transitions
+- **Backend Changes**: Not required, but feel free to include terminal output or other evidence that demonstrates functionality
+
+This helps reviewers see what you've built without having to pull down and test your branch first.
+-->
 
 ### Additional Notes
 

.gitignore

@@ -21,7 +21,9 @@ coverage
 
 *evals.env
 
-# Generated proto files
+# Generated files
+src/generated/
+# Core
 src/core/controller/*/methods.ts
 src/core/controller/*/index.ts
 src/core/controller/grpc-service-config.ts
@@ -30,10 +32,9 @@ src/shared/proto/*.ts
 src/shared/proto/host/*.ts
 # Webview
 webview-ui/src/services/grpc-client.ts
-# Standalone
-src/standalone/server-setup.ts
 # Host bridge
 src/hosts/vscode/*/methods.ts
 src/hosts/vscode/*/index.ts
 src/hosts/vscode/client/host-grpc-client.ts
 src/hosts/vscode/host-grpc-service-config.ts
+src/standalone/server-setup.ts

.husky/pre-commit

@@ -9,9 +9,9 @@ npm run lint || {
 
 # Run Prettier
 echo "Running Prettier..."
-npm run format || {
-    echo "❌ Prettier check failed. Run 'npm run format:fix' to automatically fix formatting issues."
-    exit 1
-}
+npx lint-staged --verbose || {
+     echo "❌ Prettier failed. Please fix the errors and try committing again."
+     exit 1
+ }
 
 echo "✅ All checks passed!"

.prettierignore

@@ -5,3 +5,4 @@ webview-ui/build/
 package-lock.json
 src/core/prompts/system.ts
 src/core/prompts/model_prompts/claude4.ts
+evals/

.vscode/launch.json

@@ -42,7 +42,7 @@
 		{
 			"type": "node",
 			"request": "launch",
-			"name": "Run Standalone Extension",
+			"name": "Run Standalone Service",
 			"skipFiles": ["<node_internals>/**"],
 			"sourceMaps": true,
 			"resolveSourceMapLocations": ["${workspaceFolder}/**", "!**/node_modules/**"],
@@ -50,10 +50,12 @@
 			"outFiles": ["${workspaceFolder}/dist-standalone/**/*.js"],
 			"preLaunchTask": "compile-standalone",
 			"env": {
-				"GRPC_TRACE": "all",
-				"GRPC_VERBOSITY": "DEBUG",
+				// Turns on grpc debug log.
+				//"GRPC_TRACE": "all",
+				//"GRPC_VERBOSITY": "DEBUG",
 				"NODE_PATH": "${workspaceFolder}/dist-standalone/node_modules",
-				"CLINE_DIR": "${userHome}/.cline-standalone"
+
+				"HOST_BRIDGE_ADDRESS": "localhost:50052"
 			},
 			"program": "standalone.js"
 		}

.vscode/tasks.json

@@ -128,7 +128,25 @@
 			"type": "npm",
 			"script": "watch:esbuild",
 			"group": "build",
-			"problemMatcher": "$esbuild-watch",
+			"problemMatcher": {
+				"pattern": [
+					{
+						"regexp": "^✘ \\[ERROR\\] (.*)$",
+						"message": 1
+					},
+					{
+						"regexp": "^\\s+(.*):(\\d+):(\\d+):$",
+						"file": 1,
+						"line": 2,
+						"column": 3
+					}
+				],
+				"background": {
+					"activeOnStart": true,
+					"beginsPattern": "^\\[watch\\] build started$",
+					"endsPattern": "^\\[watch\\] build finished$"
+				}
+			},
 			"isBackground": true,
 			"label": "npm: watch:esbuild",
 			"dependsOn": ["npm: protos"],
@@ -146,7 +164,25 @@
 			"type": "npm",
 			"script": "watch:esbuild:test",
 			"group": "build",
-			"problemMatcher": "$esbuild-watch",
+			"problemMatcher": {
+				"pattern": [
+					{
+						"regexp": "^✘ \\[ERROR\\] (.*)$",
+						"message": 1
+					},
+					{
+						"regexp": "^\\s+(.*):(\\d+):(\\d+):$",
+						"file": 1,
+						"line": 2,
+						"column": 3
+					}
+				],
+				"background": {
+					"activeOnStart": true,
+					"beginsPattern": "^\\[watch\\] build started$",
+					"endsPattern": "^\\[watch\\] build finished$"
+				}
+			},
 			"isBackground": true,
 			"label": "npm: watch:esbuild:test",
 			"dependsOn": ["npm: protos"],

CHANGELOG.md

@@ -1,5 +1,45 @@
 # Changelog
 
+## [3.17.15]
+
+-   Fix LiteLLM provider to properly respect selected model IDs when switching between Plan and Act modes (Thanks @sammcj!)
+-   Fix chat input being cleared when switching between Plan/Act modes without sending a message (Thanks @BarreiroT!)
+-   Fix MCP server name display to avoid showing "undefined" for SSE servers, preventing tool/resource invocation failures (Thanks @ramybenaroya!)
+-   Fix AWS Bedrock provider by removing deprecated custom model encoding (Thanks @watany-dev!)
+-   Fix timeline tooltips for followup messages and improve color retrieval code (Thanks @char8x!)
+-   Improve accessibility by making task header buttons properly announced by screen readers (Thanks @yncat!)
+-   Improve accessibility by adding proper state reporting for Plan/Act mode switch for screen readers (Thanks @yncat!)
+-   Prevent reading development environment variables from user's environment (Thanks @BarreiroT!)
+
+## [3.17.14]
+
+-   Add Claude Code as a new API provider, allowing integration with Anthropic's Claude Code CLI tool and Claude Max Plan (Thanks @BarreiroT!)
+-   Add SAP AI Core as a new API provider with support for Claude and GPT models (Thanks @schardosin!)
+-   Add configurable default terminal profile setting, allowing users to specify which terminal Cline should use (Thanks @valinha!)
+-   Add terminal output size constraint setting to limit how much terminal output is processed
+-   Add MCP Rich Display settings to the settings page for persistent configuration (Thanks @Vl4diC0de!)
+-   Improve copy button functionality with refactored reusable components (Thanks @shouhanzen!)
+-   Improve AWS Bedrock provider by removing deprecated dependency and using standard AWS SDK (Thanks @watany-dev!)
+-   Fix list_files tool to properly return files when targeting hidden directories
+-   Fix search and replace edge case that could cause file deletion, making the algorithm more lenient for models using different diff formats
+-   Fix task restoration issues that could occur when resuming interrupted tasks
+-   Fix checkpoint saving to properly track all file changes
+-   Improve file context warnings to reduce diff edit errors when resuming restored tasks
+-   Clear chat input when switching between Plan/Act modes within a task
+-   Exclude .clinerules files from checkpoint tracking
+
+## [3.17.13]
+
+-   Add Thinking UX for Gemini models, providing visual feedback during model reasoning
+-   Add support for Notifications MCP integration with Cline
+-   Add prompt caching indicator for Grok 3 models
+-   Sort MCP marketplace by newest listings by default for easier discovery of recent servers
+-   Update O3 model family pricing to reflect latest OpenAI rates
+-   Remove '-beta' suffix from Grok model identifiers
+-   Fix AWS Bedrock provider by removing deprecated Anthropic-Bedrock SDK (Thanks @watany-dev!)
+-   Fix menu display issue for terminal timeout settings
+-   Improve chat input field styling and behavior
+
 ## [3.17.12]
 
 -   **Free Grok Model Available!** Access Grok 3 completely free through the Cline provider

CONTRIBUTING.md

@@ -18,6 +18,18 @@ We also welcome contributions to our [documentation](https://github.com/cline/cl
 
 If you're planning to work on a bigger feature, please create a [feature request](https://github.com/cline/cline/discussions/categories/feature-requests?discussions_q=is%3Aopen+category%3A%22Feature+Requests%22+sort%3Atop) first so we can discuss whether it aligns with Cline's vision.
 
+## Before Contributing
+
+All contributions must begin with a GitHub Issue, unless the change is for small bug fixes, typo corrections, minor wording improvements, or simple type fixes that don't change functionality.
+
+- **Check existing issues**: Search [GitHub Issues](https://github.com/cline/cline/issues).
+- **Create an issue**: Use appropriate templates:
+    - **Bugs:** "Bug Report" template.
+    - **Features:** "Detailed Feature Proposal" template. Approval from a core Cline contributor required before starting.
+- **Claim issues**: Comment your interest.
+
+**PRs without approved issues may be closed.**
+
 ## Development Setup
 
 1. **VS Code Extensions**
@@ -29,6 +41,7 @@ If you're planning to work on a bigger feature, please create a [feature request
 2. **Local Development**
     - Run `npm run install:all` to install dependencies
     - Run `npm run test` to run tests locally
+    - Run → Start Debugging or `>Debug: Select and Start Debugging` and wait for a new VS Code instance to open
     - Before submitting PR, run `npm run format:fix` to format your code
 
 3. **Linux-specific Setup**

buf.yaml

@@ -0,0 +1,22 @@
+version: v2
+modules:
+    - path: proto
+      name: cline/cline/lint
+
+lint:
+    use:
+        - STANDARD
+
+    except: # Add exceptions for current patterns that contradict STANDARD settings
+        - RPC_PASCAL_CASE # rpcs are camel case (start with lowercase)
+        - PACKAGE_DIRECTORY_MATCH # the protos in the cline package are not in a dir named cline.
+        - RPC_REQUEST_RESPONSE_UNIQUE # request messages are not unique.
+        - RPC_REQUEST_STANDARD_NAME # request messages dont all end with Request
+        - RPC_RESPONSE_STANDARD_NAME # response messages dont all end with Response
+        - PACKAGE_VERSION_SUFFIX # package name does not contain version.
+        - ENUM_VALUE_PREFIX # enum values dont start with the enum name.
+        - ENUM_ZERO_VALUE_SUFFIX # first value does not have to be UNSPECIFIED.
+
+# breaking:
+#   use:
+#     - WIRE_JSON # Detect changes that break the json wire format (this is the minimum recommended level.)

esbuild.js

@@ -125,9 +125,11 @@ const baseConfig = {
 	minify: production,
 	sourcemap: !production,
 	logLevel: "silent",
-	define: {
-		"process.env.IS_DEV": JSON.stringify(!production),
-	},
+	define: production
+		? {
+				"process.env.IS_DEV": JSON.stringify(!production),
+			}
+		: undefined,
 	tsconfig: path.resolve(__dirname, "tsconfig.json"),
 	plugins: [
 		copyWasmFiles,

evals/.gitignore

@@ -2,5 +2,21 @@ repositories
 
 results/evals.db
 
+diff-edits/cases/
+diff-edits/results/
+
+# Environment variables
+.env
+
+# backwards compatible
 diff_editing/test_cases/
-diff_editing/test_outputs/
+diff_editing/test_outputs/
+
+*.db
+*.db-wal
+*.db-shm
+
+.cache
+
+# Python bytecode cache
+*__pycache__/

evals/README.md

@@ -17,6 +17,7 @@ The evaluation system consists of two main components:
 
 1. **Test Server**: Enhanced HTTP server in `src/services/test/TestServer.ts` that provides detailed task results
 2. **CLI Tool**: Command-line interface in `evals/cli/` for orchestrating evaluations
+3. **Diff Edit Benchmark**: Separate command using the CLI tool that runs a comprehensive diff editing benchmark suite on real world cases, along with a streamlit dashboard displaying the results. For more details, see the [Diff Edit Benchmark README](./diff-edits/README.md). Make sure you add a `evals/diff-edits/cases` folder with all the conversation jsons. 
 
 ## Directory Structure
 
@@ -40,6 +41,12 @@ cline-repo/
 │   │   │   └── utils/                # Utility functions
 │   │   ├── package.json
 │   │   └── tsconfig.json
+│   ├── diff-edits/                   # Diff editing evaluation suite
+│   │   ├── cases/                    # Test case JSON files
+│   │   ├── results/                  # Evaluation results
+│   │   ├── diff-apply/               # Diff application logic
+│   │   ├── parsing/                  # Assistant message parsing
+│   │   └── prompts/                  # System prompts
 │   ├── repositories/                 # Cloned benchmark repositories
 │   │   ├── exercism/                 # Modified Exercism (from pashpashpash/evals)
 │   │   ├── swe-bench/                # SWE-Bench repository
@@ -148,6 +155,192 @@ Freelance-style programming tasks from the SWELancer benchmark.
 
 Multi-file software engineering tasks from the Multi-SWE-Bench repository.
 
+## Diff Edit Evaluations
+
+The Cline Evaluation System includes a specialized suite for evaluating how well models can make precise edits to files using the `replace_in_file` tool.
+
+### Overview
+
+Diff edit evaluations test a model's ability to:
+
+1. Understand file content and identify specific sections to modify
+2. Generate correct SEARCH/REPLACE blocks for targeted edits
+3. Successfully apply changes without introducing errors
+
+### Directory Structure
+
+```
+diff-edits/
+├── cases/                  # Test case JSON files
+├── results/                # Evaluation results
+├── ClineWrapper.ts         # Wrapper for model interaction
+├── TestRunner.ts           # Main test execution logic
+├── types.ts                # Type definitions
+├── diff-apply/             # Diff application logic
+├── parsing/                # Assistant message parsing
+└── prompts/                # System prompts
+```
+
+### Creating Test Cases
+
+Test cases are defined as JSON files in the `diff-edits/cases/` directory. Each test case should include:
+
+```json
+{
+  "test_id": "example_test_1",
+  "messages": [
+    {
+      "role": "user",
+      "text": "Please fix the bug in this code...",
+      "images": []
+    },
+    {
+      "role": "assistant",
+      "text": "I'll help you fix that bug..."
+    }
+  ],
+  "file_contents": "// Original file content here\nfunction example() {\n  // Code with bug\n}",
+  "file_path": "src/example.js",
+  "system_prompt_details": {
+    "mcp_string": "",
+    "cwd_value": "/path/to/working/directory",
+    "browser_use": false,
+    "width": 900,
+    "height": 600,
+    "os_value": "macOS",
+    "shell_value": "/bin/zsh",
+    "home_value": "/Users/username",
+    "user_custom_instructions": ""
+  },
+  "original_diff_edit_tool_call_message": ""
+}
+```
+
+### Running Diff Edit Evaluations
+
+#### Single Model Evaluation
+
+```bash
+cd evals/cli
+node dist/index.js run-diff-eval --model-ids "anthropic/claude-3-5-sonnet-20241022"
+```
+
+#### Multi-Model Evaluation
+
+Compare multiple models in a single evaluation run:
+
+```bash
+# Compare Claude and Grok models
+node dist/index.js run-diff-eval \
+  --model-ids "anthropic/claude-3-5-sonnet-20241022,x-ai/grok-beta" \
+  --max-cases 10 \
+  --valid-attempts-per-case 3 \
+  --verbose
+
+# Compare multiple Claude variants
+node dist/index.js run-diff-eval \
+  --model-ids "anthropic/claude-3-5-sonnet-20241022,anthropic/claude-3-5-haiku-20241022,anthropic/claude-3-opus-20240229" \
+  --max-cases 5 \
+  --valid-attempts-per-case 2 \
+  --parallel
+```
+
+#### Options
+
+- `--model-ids`: Comma-separated list of model IDs to evaluate (required)
+- `--system-prompt-name`: System prompt to use (default: "basicSystemPrompt")
+- `--valid-attempts-per-case`: Number of attempts per test case per model (default: 1)
+- `--max-cases`: Maximum number of test cases to run (default: all available)
+- `--parsing-function`: Function to parse assistant messages (default: "parseAssistantMessageV2")
+- `--diff-edit-function`: Function to apply diffs (default: "constructNewFileContentV2")
+- `--test-path`: Path to test cases (default: diff-edits/cases)
+- `--thinking-budget`: Tokens allocated for thinking (default: 0)
+- `--parallel`: Run tests in parallel (flag)
+- `--replay`: Use pre-recorded LLM output (flag)
+- `--verbose`: Enable detailed logging (flag)
+
+#### Examples
+
+```bash
+# Quick test with 2 models, 4 cases, 2 attempts each
+node dist/index.js run-diff-eval \
+  --model-ids "anthropic/claude-3-5-sonnet-20241022,x-ai/grok-beta" \
+  --max-cases 4 \
+  --valid-attempts-per-case 2 \
+  --verbose
+
+# Comprehensive evaluation with parallel execution
+node dist/index.js run-diff-eval \
+  --model-ids "anthropic/claude-3-5-sonnet-20241022,anthropic/claude-3-5-haiku-20241022" \
+  --system-prompt-name claude4SystemPrompt \
+  --valid-attempts-per-case 5 \
+  --max-cases 20 \
+  --parallel \
+  --verbose
+```
+
+### Database Storage & Analytics
+
+All evaluation results are automatically stored in a SQLite database (`diff-edits/evals.db`) for advanced analytics and comparison. The database includes:
+
+- **System Prompts**: Versioned system prompt content with hashing for deduplication
+- **Processing Functions**: Versioned parsing and diff-edit function configurations
+- **Files**: Original and edited file content with content-based hashing
+- **Runs**: Evaluation run metadata and configuration
+- **Cases**: Individual test case information with context tokens
+- **Results**: Detailed results with timing, cost, and success metrics
+
+### Interactive Dashboard
+
+Launch the Streamlit dashboard to visualize and analyze evaluation results:
+
+```bash
+cd diff-edits/dashboard
+streamlit run app.py
+```
+
+The dashboard provides:
+
+- **Model Performance Comparison**: Side-by-side comparison of success rates, latency, and costs
+- **Interactive Charts**: Success rate trends, latency vs cost analysis, and performance metrics
+- **Detailed Drill-Down**: Individual result analysis with file content viewing
+- **Run Selection**: Browse and compare different evaluation runs
+- **Real-time Updates**: Automatically refreshes with new evaluation data
+
+#### Dashboard Features
+
+1. **Hero Section**: Overview of current run with key metrics
+2. **Model Cards**: Performance cards with grades and detailed metrics
+3. **Comparison Charts**: Interactive Plotly charts for visual analysis
+4. **Result Explorer**: Detailed view of individual test results including:
+   - Original and edited file content
+   - Raw model output
+   - Parsed tool calls
+   - Timing and cost metrics
+   - Error analysis
+
+#### Quick Start Dashboard
+
+```bash
+# Run a quick evaluation
+node cli/dist/index.js run-diff-eval \
+  --model-ids "anthropic/claude-3-5-sonnet-20241022,x-ai/grok-beta" \
+  --max-cases 4 \
+  --valid-attempts-per-case 2 \
+  --verbose
+
+# Launch dashboard to view results
+cd diff-edits/dashboard && streamlit run app.py
+```
+
+### Legacy Results
+
+For backward compatibility, results are also saved as JSON files in the `diff-edits/results/` directory. The JSON results include:
+- Success/failure status
+- Extracted tool calls
+- Diff edit content
+- Token usage and cost metrics
+
 ## Metrics
 
 The evaluation system collects the following metrics:

evals/cli/package.json

@@ -1,39 +0,0 @@
-{
-	"name": "cline-evaluation-cli",
-	"version": "0.1.0",
-	"description": "CLI tool for orchestrating Cline evaluations across multiple benchmarks",
-	"main": "dist/index.js",
-	"scripts": {
-		"build": "tsc",
-		"start": "node dist/index.js",
-		"dev": "ts-node src/index.ts",
-		"test": "echo \"Error: no test specified\" && exit 1"
-	},
-	"keywords": [
-		"cline",
-		"evaluation",
-		"benchmark"
-	],
-	"author": "",
-	"license": "MIT",
-	"dependencies": {
-		"better-sqlite3": "^11.10.0",
-		"chalk": "^4.1.2",
-		"commander": "^9.4.1",
-		"execa": "^5.1.1",
-		"node-fetch": "^2.7.0",
-		"ora": "^5.4.1",
-		"sqlite": "^4.1.2",
-		"uuid": "^9.0.0",
-		"yargs": "^17.6.2"
-	},
-	"devDependencies": {
-		"@types/better-sqlite3": "^7.6.3",
-		"@types/node": "^18.11.18",
-		"@types/node-fetch": "^2.6.12",
-		"@types/uuid": "^9.0.0",
-		"@types/yargs": "^17.0.19",
-		"ts-node": "^10.9.1",
-		"typescript": "^4.9.4"
-	}
-}

evals/cli/src/commands/runDiffEval.ts

@@ -3,9 +3,9 @@ import chalk from "chalk"
 import path from "path"
 
 interface RunDiffEvalOptions {
-	modelId: string
+	modelIds: string
 	systemPromptName: string
-	numberOfRuns: number
+	validAttemptsPerCase: number
 	parsingFunction: string
 	diffEditFunction: string
 	thinkingBudget: number
@@ -14,22 +14,23 @@ interface RunDiffEvalOptions {
 	testPath: string
 	outputPath: string
 	replay: boolean
+	maxCases?: number
 }
 
 export async function runDiffEvalHandler(options: RunDiffEvalOptions) {
 	console.log(chalk.blue("Starting diff editing evaluation..."))
 
 	// Resolve the path to the TestRunner.ts script relative to the current file
-	const scriptPath = path.resolve(__dirname, "../../../diff_editing/TestRunner.ts")
+	const scriptPath = path.resolve(__dirname, "../../../diff-edits/TestRunner.ts")
 
 	// Construct the arguments array for the execa call
 	const args = [
-		"--model-id",
-		options.modelId,
+		"--model-ids",
+		options.modelIds,
 		"--system-prompt-name",
 		options.systemPromptName,
-		"--number-of-runs",
-		String(options.numberOfRuns),
+		"--valid-attempts-per-case",
+		String(options.validAttemptsPerCase),
 		"--parsing-function",
 		options.parsingFunction,
 		"--diff-edit-function",
@@ -59,12 +60,16 @@ export async function runDiffEvalHandler(options: RunDiffEvalOptions) {
 		args.push("--verbose")
 	}
 
+	if (options.maxCases) {
+		args.push("--max-cases", String(options.maxCases))
+	}
+
 	try {
 		console.log(chalk.gray(`Executing: npx tsx ${scriptPath} ${args.join(" ")}`))
 
 		// Execute the script as a child process
 		// We use 'inherit' to stream the stdout/stderr directly to the user's terminal
-		const subprocess = execa("npx", ["tsx", scriptPath, ...args], {
+		const subprocess = execa("npx", ["tsx", "--tsconfig", path.resolve(__dirname, "../../../tsconfig.json"), scriptPath, ...args], {
 			stdio: "inherit",
 		})
 

evals/cli/src/index.ts

@@ -84,9 +84,10 @@ program
 	.description("Run the diff editing evaluation suite")
 	.option("--test-path <path>", "Path to the directory containing test case JSON files")
 	.option("--output-path <path>", "Path to the directory to save the test output JSON files")
-	.option("--model-id <model_id>", "The model ID to use for the test")
+	.option("--model-ids <model_ids>", "Comma-separated list of model IDs to test")
 	.option("--system-prompt-name <name>", "The name of the system prompt to use", "basicSystemPrompt")
-	.option("-n, --number-of-runs <number>", "Number of times to run each test case", "1")
+	.option("-n, --valid-attempts-per-case <number>", "Number of valid attempts per test case per model (will retry until this many valid attempts are collected)", "1")
+	.option("--max-cases <number>", "Maximum number of test cases to run (limits total cases loaded)")
 	.option("--parsing-function <name>", "The parsing function to use", "parseAssistantMessageV2")
 	.option("--diff-edit-function <name>", "The diff editing function to use", "constructNewFileContentV2")
 	.option("--thinking-budget <tokens>", "Set the thinking tokens budget", "0")
@@ -95,11 +96,11 @@ program
 	.option("-v, --verbose", "Enable verbose logging", false)
 	.action(async (options) => {
 		try {
-			// The logic here simplifies slightly
 			const fullOptions = {
 				...options,
-				numberOfRuns: parseInt(options.numberOfRuns, 10),
+				validAttemptsPerCase: parseInt(options.validAttemptsPerCase, 10),
 				thinkingBudget: parseInt(options.thinkingBudget, 10),
+				maxCases: options.maxCases ? parseInt(options.maxCases, 10) : undefined,
 			}
 			await runDiffEvalHandler(fullOptions)
 		} catch (error) {

evals/diff-edits/ClineWrapper.ts

@@ -39,16 +39,22 @@ interface StreamResult {
 		cacheReadTokens: number
 		totalCost: number
 	}
+	timing?: {
+		timeToFirstTokenMs: number
+		timeToFirstEditMs?: number
+		totalRoundTripMs: number
+	}
 }
 
 /**
- * Process the stream and return full response
+ * Process the stream and return full response with timing data
  */
 async function processStream(
 	handler: OpenRouterHandler,
 	systemPrompt: string,
 	messages: Anthropic.Messages.MessageParam[],
 ): Promise<StreamResult> {
+	const startTime = Date.now()
 	const stream = handler.createMessage(systemPrompt, messages)
 
 	let assistantMessage = ""
@@ -58,12 +64,21 @@ async function processStream(
 	let cacheWriteTokens = 0
 	let cacheReadTokens = 0
 	let totalCost = 0
+	
+	// Timing tracking
+	let timeToFirstTokenMs: number | null = null
+	let timeToFirstEditMs: number | null = null
 
 	for await (const chunk of stream) {
 		if (!chunk) {
 			continue
 		}
 
+		// Capture time to first token (any chunk type)
+		if (timeToFirstTokenMs === null) {
+			timeToFirstTokenMs = Date.now() - startTime
+		}
+
 		switch (chunk.type) {
 			case "usage":
 				inputTokens += chunk.inputTokens
@@ -79,10 +94,25 @@ async function processStream(
 				break
 			case "text":
 				assistantMessage += chunk.text
+				
+				// Try to detect first tool call by parsing accumulated message
+				if (timeToFirstEditMs === null) {
+					try {
+						const parsed = parseAssistantMessageV2(assistantMessage)
+						const hasToolCall = parsed.some(block => block.type === "tool_use")
+						if (hasToolCall) {
+							timeToFirstEditMs = Date.now() - startTime
+						}
+					} catch {
+						// Parsing failed, continue accumulating
+					}
+				}
 				break
 		}
 	}
 
+	const totalRoundTripMs = Date.now() - startTime
+
 	return {
 		assistantMessage,
 		reasoningMessage,
@@ -93,6 +123,11 @@ async function processStream(
 			cacheReadTokens,
 			totalCost,
 		},
+		timing: {
+			timeToFirstTokenMs: timeToFirstTokenMs || 0,
+			timeToFirstEditMs: timeToFirstEditMs || undefined,
+			totalRoundTripMs,
+		},
 	}
 }
 
@@ -245,7 +280,22 @@ export async function runSingleEvaluation(input: TestInput): Promise<TestResult>
 		}
 
 		// check that we are editing the correct file path
+		console.log(`Expected file path: "${originalFilePath}"`);
+		console.log(`Actual file path used: "${diffToolPath}"`);
 		if (diffToolPath !== originalFilePath) {
+			console.log(`❌ File path mismatch detected!`);
+			// Enhanced logging:
+			if (streamResult?.assistantMessage) {
+				console.log(`   Full model output (assistantMessage):`);
+				console.log(`   -----------------------------------------`);
+				console.log(`   ${streamResult.assistantMessage}`);
+				console.log(`   -----------------------------------------`);
+			}
+			if (toolCall) {
+				console.log(`   Parsed tool call that caused mismatch:`);
+				console.log(`   ${JSON.stringify(toolCall, null, 2)}`);
+				console.log(`   -----------------------------------------`);
+			}
 			return {
 				success: false,
 				streamResult: streamResult,

evals/diff-edits/README.md

@@ -0,0 +1,71 @@
+# A Note on Cline's Diff Evaluation Setup
+
+Hey there, this note explains what we're doing with Cline's diff evaluation (evals) system. It's all about checking how well various AI models (which users connect to Cline via their own API keys), prompts, and diffing tools can handle file changes.
+
+## What We're Trying to Figure Out
+
+The main idea here is to figure out which AI models (configured by users) are best at making `replace_in_file` tool calls that work correctly. This helps us understand model capabilities and also speeds up our own experiments with prompts and diffing algorithms to make Cline better over time. We want to know a few key things.
+
+First, can the model create diffs, which are just sets of SEARCH and REPLACE blocks, that apply cleanly to a file? This is what we call `diffEditSuccess`.
+
+Second, how do different LLMs, like Claude or Grok, stack up against each other when they try to make these diff edits? We use a standard set of real-world test cases for this.
+
+Third, do different system prompts, say our `basicSystemPrompt` versus the `claude4SystemPrompt`, change how well a model does at diff editing?
+
+Fourth, we're also looking at different ways to apply the diffs themselves. We have a few algorithms like `constructNewFileContentV1`, `V2`, and `V3`, and we want to see which ones are more robust when fed model-generated diffs.
+
+Fifth, we track how fast the model starts making an edit. The `timeToFirstEditMs` metric gives us a hint about how quickly a user would see changes happening in their editor.
+
+And finally, we keep an eye on how many tokens are used and what it costs for each model and each try. This helps us compare how efficient they are.
+
+Right now, these evals are mostly about whether the diff *applies* correctly. That means, do the SEARCH blocks find a match, and can the REPLACE blocks be put in without an error? We're not yet deeply analyzing if the change is valid code or matches what the user *wanted* semantically. That's a problem for another day, and will require a lot more scaffolding.
+
+## How We Run These Tests
+
+Two prerequisites:
+
+1. Make sure you have an `evals/.env` file with `OPENROUTER_API_KEY=<your-openrouter-key>` 
+
+2. Make sure you add a `evals/diff-edits/cases` folder with all the conversation jsons prior to running this. 
+
+
+Our testing strategy is based on replaying situations from actual user sessions where diff edits were tried.
+
+It starts with our test cases. Each one is a JSON file in `./cases` that has the conversation history that led to a diff edit, the original file content and its path, and the info needed to rebuild the system prompt from that original session.
+
+Then, for every test run, we set up a specific configuration. This includes which LLM we're testing, which system prompt it gets, which function we use to parse the model's raw output, and which function we use to actually apply the diff. Here's the command I've been using:
+
+```bash
+npm run diff-eval -- --model-ids "anthropic/claude-3-5-sonnet-20241022,x-ai/grok-3-beta" --max-cases 4 --valid-attempts-per-case 2 --verbose --parallel
+```
+
+This will build the eval script, run it, and then open the streamlit dashboard to show the results.
+
+The `TestRunner.ts` script is the main coordinator. For each test case and setup, `ClineWrapper.ts` takes over and sends the conversation and system prompt to the LLM. We then watch the model's response as it streams in and parse it to find any tool calls.
+
+We're specifically looking for the model to make a single `replace_in_file` tool call. Multiple edits in one tool call are allowed, and recorded (in case you want to filter results by number of edits in a single tool call and compare success rate for that slice across different models/system prompts/etc). If it does, and it's for the correct file, we grab the diff content it produced. Then, the chosen diff application algorithm tries to apply that diff to the original file. We record whether this worked or not as `diffEditSuccess`.
+
+We record a bunch of data for every attempt into a database. This includes details about the model and prompt, token counts, costs, the raw output from the model, the parsed tool calls, whether it succeeded or failed, any error messages, and timing info. For a detailed explanation of the database schema, see [database.md](./database.md).
+
+A big part of this is how we handle "valid attempts," which I'll explain next.
+
+## Keeping it Fair with "Valid Attempts"
+
+LLMs can be unpredictable. If we replay an old scenario, a new model, or even the same model later, might do something completely different than what happened originally. It might call another tool or ask a question instead of trying a diff edit.
+
+Since we really want to test the *diff editing* part, we need a way to make sure we're comparing fairly. That's why we have this idea of "valid attempts."
+
+An attempt is "valid" for this benchmark if the model actually tries to do what we're interested in. This means two things. One, it must call the `replace_in_file` tool. Two, it must target the *same file path* that was targeted in the original recorded conversation for that test case.
+
+If the model does something else, like calling a different tool or picking the wrong file, we don't count that attempt against its diff editing score. Instead, we consider it an "invalid attempt" for *this specific benchmark* and simply re-run that test case with that model. We keep doing this until we've collected a set number of these "valid attempts."
+
+For example, if we ask for 5 valid attempts per test case, the system will keep re-rolling for that case until the model has tried to edit the correct file using the `replace_in_file` tool 5 times. Only then do we look at how many of those 5 valid attempts actually resulted in a successful diff application (`diffEditSuccess`).
+
+This way, if we're comparing two models and one gets a 10% success rate on its valid diff edit attempts, and another gets 90%, we have a much clearer picture of their actual diff-generating capabilities. It avoids muddying the waters with attempts where the model didn't even try to perform the specific action we're evaluating. This approach helps us isolate and measure the diff-editing skill more directly, despite the non-deterministic nature of these models.
+
+## Some known edge cases
+
+I noticed that some of the current conversation jsons in the `./cases` folder are a little big bogus. Here's a running list of these areas:
+
+- ~~What if the conversation json was using a model with a massive context window, like Google Gemini's 1M context window, and we're now re-rolling that case on a smaller context window model like claude 4 (200k) or grok-3 (128k)? It's def gonna fail, and we shouldn't just keep trying. We should have a smart system for selecting which cases we can use given the arguments passed in. For example, if we pass in claude and grok with `max cases = 2`, we shouldn't just pick the first two jsons in the folder. We should go through, use a tokenizer, and make sure it would fit with some padding like 20k tokens. Use tiktoken. 20k padding will be sufficient even though different models tokenize differently.~~
+- There are some weird jsons, where something weird happen, where essentially there's a fluke. Maybe the user was using an extremely dumb model that just hallucinated a fake filepath. Now when we try to reroll that case, we never get a valid case. This can easily be handled by making sure that the file is present before selecting that eval for testing. By "file is present" I mean, that file_contents is present in the eval. Additionally, we should in the streamlit dashboard show cases where getting a valid attempt is a challenge, so we can review those cases more easily and throw them out if they're bogus. Maybe a special tab/page in the dashboard for this purpose. Across all runs / cases, what the most consistently problematic cases are. Pop one open to see the case formatted json with all the user/assistant turns.

evals/diff-edits/TestRunner.ts

@@ -0,0 +1,930 @@
+import { runSingleEvaluation, TestInput, TestResult } from "./ClineWrapper"
+import { basicSystemPrompt } from "./prompts/basicSystemPrompt-06-06-25"
+import { claude4SystemPrompt } from "./prompts/claude4SystemPrompt-06-06-25"
+import { formatResponse } from "./helpers"
+import { Anthropic } from "@anthropic-ai/sdk"
+import * as fs from "fs"
+import * as path from "path"
+import { Command } from "commander"
+import { InputMessage, ProcessedTestCase, TestCase, TestConfig, SystemPromptDetails, ConstructSystemPromptFn } from "./types"
+import { loadOpenRouterModelData, EvalOpenRouterModelInfo } from "./openRouterModelsHelper" // Added import
+import {
+	getDatabase,
+	upsertSystemPrompt,
+	upsertProcessingFunctions,
+	upsertFile,
+	createBenchmarkRun,
+	createCase,
+	insertResult,
+	DatabaseClient,
+	CreateResultInput,
+} from "./database"
+
+// Load environment variables from .env file
+import * as dotenv from "dotenv"
+dotenv.config({ path: path.join(__dirname, "../.env") })
+
+// tiktoken for token counting
+import { get_encoding } from "tiktoken";
+const encoding = get_encoding("cl100k_base"); 
+
+let openRouterModelDataGlobal: Record<string, EvalOpenRouterModelInfo> = {}; // Global to store fetched data
+
+function log(isVerbose: boolean, message: string) {
+	if (isVerbose) {
+		console.log(message)
+	}
+}
+
+const systemPromptGeneratorLookup: Record<string, ConstructSystemPromptFn> = {
+	basicSystemPrompt: basicSystemPrompt,
+	claude4SystemPrompt: claude4SystemPrompt,
+}
+
+type TestResultSet = { [test_id: string]: (TestResult & { test_id?: string })[] }
+
+class NodeTestRunner {
+	private apiKey: string | undefined
+	private currentRunId: string | null = null
+	private systemPromptHash: string | null = null
+	private processingFunctionsHash: string | null = null
+	private caseIdMap: Map<string, string> = new Map() // test_id -> case_id mapping
+
+	constructor(isReplay: boolean) {
+		if (!isReplay) {
+			this.apiKey = process.env.OPENROUTER_API_KEY
+			if (!this.apiKey) {
+				throw new Error("OPENROUTER_API_KEY environment variable not set for a non-replay run.")
+			}
+		}
+	}
+
+	/**
+	 * Initialize database run and store system prompt and processing functions
+	 */
+	async initializeDatabaseRun(testConfig: TestConfig, testCases: ProcessedTestCase[], isVerbose: boolean): Promise<string> {
+		try {
+			// Generate a sample system prompt to hash (using first test case)
+			const sampleSystemPrompt = testCases.length > 0 
+				? this.constructSystemPrompt(testCases[0].system_prompt_details, testConfig.system_prompt_name)
+				: "default-system-prompt";
+
+			// Store system prompt
+			this.systemPromptHash = await upsertSystemPrompt({
+				name: testConfig.system_prompt_name,
+				content: sampleSystemPrompt
+			});
+
+			// Store processing functions
+			this.processingFunctionsHash = await upsertProcessingFunctions({
+				name: `${testConfig.parsing_function}-${testConfig.diff_edit_function}`,
+				parsing_function: testConfig.parsing_function,
+				diff_edit_function: testConfig.diff_edit_function
+			});
+
+			// Create benchmark run
+			const runDescription = `Model: ${testConfig.model_id}, Cases: ${testCases.length}, Runs per case: ${testConfig.number_of_runs}`;
+			this.currentRunId = await createBenchmarkRun({
+				description: runDescription,
+				system_prompt_hash: this.systemPromptHash
+			});
+
+			log(isVerbose, `✓ Database run initialized: ${this.currentRunId}`);
+			
+			// Create case records
+			await this.createDatabaseCases(testCases, isVerbose);
+
+			return this.currentRunId;
+		} catch (error) {
+			console.error("Failed to initialize database run:", error);
+			throw error;
+		}
+	}
+
+	/**
+	 * Initialize multi-model database run (one run for all models)
+	 */
+	async initializeMultiModelRun(testCases: ProcessedTestCase[], systemPromptName: string, parsingFunction: string, diffEditFunction: string, runDescription: string, isVerbose: boolean): Promise<string> {
+		try {
+			// Generate a sample system prompt to hash (using first test case)
+			const sampleSystemPrompt = testCases.length > 0 
+				? this.constructSystemPrompt(testCases[0].system_prompt_details, systemPromptName)
+				: "default-system-prompt";
+
+			// Store system prompt
+			this.systemPromptHash = await upsertSystemPrompt({
+				name: systemPromptName,
+				content: sampleSystemPrompt
+			});
+
+			// Store processing functions
+			this.processingFunctionsHash = await upsertProcessingFunctions({
+				name: `${parsingFunction}-${diffEditFunction}`,
+				parsing_function: parsingFunction,
+				diff_edit_function: diffEditFunction
+			});
+
+			// Create benchmark run
+			this.currentRunId = await createBenchmarkRun({
+				description: runDescription,
+				system_prompt_hash: this.systemPromptHash
+			});
+
+			log(isVerbose, `✓ Multi-model database run initialized: ${this.currentRunId}`);
+			
+			// Create case records
+			await this.createDatabaseCases(testCases, isVerbose);
+
+			return this.currentRunId;
+		} catch (error) {
+			console.error("Failed to initialize multi-model database run:", error);
+			throw error;
+		}
+	}
+
+	/**
+	 * Create database case records for all test cases
+	 */
+	async createDatabaseCases(testCases: ProcessedTestCase[], isVerbose: boolean): Promise<void> {
+		if (!this.currentRunId || !this.systemPromptHash) {
+			throw new Error("Database run not initialized");
+		}
+
+		for (const testCase of testCases) {
+			try {
+				// Store file content if available
+				let fileHash: string | undefined;
+				if (testCase.file_contents && testCase.file_path) {
+					fileHash = await upsertFile({
+						filepath: testCase.file_path,
+						content: testCase.file_contents
+					});
+				}
+
+				// Calculate tokens in context (approximate)
+				const tokensInContext = this.estimateTokens(testCase.messages);
+
+				// Create case record
+				const caseId = await createCase({
+					run_id: this.currentRunId,
+					description: testCase.test_id,
+					system_prompt_hash: this.systemPromptHash,
+					task_id: testCase.test_id,
+					tokens_in_context: tokensInContext,
+					file_hash: fileHash
+				});
+
+				this.caseIdMap.set(testCase.test_id, caseId);
+			} catch (error) {
+				console.error(`Failed to create database case for ${testCase.test_id}:`, error);
+				// Continue with other cases
+			}
+		}
+
+		log(isVerbose, `✓ Created ${this.caseIdMap.size} database case records`);
+	}
+
+	/**
+	 * Store test result in database
+	 */
+	async storeResultInDatabase(result: TestResult, testId: string, modelId: string): Promise<void> {
+		if (!this.currentRunId || !this.processingFunctionsHash) {
+			return; // Skip if database not initialized
+		}
+
+		const caseId = this.caseIdMap.get(testId);
+		if (!caseId) {
+			return; // Skip if case not found
+		}
+
+		try {
+			// Map error string to error enum (simple mapping)
+			const errorEnum = this.mapErrorToEnum(result.error);
+
+			// Store diff edit content if available
+			let fileEditedHash: string | undefined;
+			if (result.diffEdit) {
+				fileEditedHash = await upsertFile({
+					filepath: `diff-edit-${testId}`,
+					content: result.diffEdit
+				});
+			}
+
+			// Calculate basic metrics from diff edit if available
+			let numEdits = 0;
+			let numLinesAdded = 0;
+			let numLinesDeleted = 0;
+			
+			if (result.diffEdit) {
+				// Simple parsing to count edits - count SEARCH/REPLACE blocks
+				const searchBlocks = (result.diffEdit.match(/------- SEARCH/g) || []).length;
+				numEdits = searchBlocks;
+				
+				// Count added/deleted lines (rough approximation)
+				const lines = result.diffEdit.split('\n');
+				for (const line of lines) {
+					if (line.startsWith('+') && !line.startsWith('+++')) {
+						numLinesAdded++;
+					} else if (line.startsWith('-') && !line.startsWith('---')) {
+						numLinesDeleted++;
+					}
+				}
+			}
+
+			const resultInput: CreateResultInput = {
+				run_id: this.currentRunId,
+				case_id: caseId,
+				model_id: modelId,
+				processing_functions_hash: this.processingFunctionsHash,
+				succeeded: result.success && (result.diffEditSuccess ?? false),
+				error_enum: errorEnum,
+				num_edits: numEdits || undefined,
+				num_lines_deleted: numLinesDeleted || undefined,
+				num_lines_added: numLinesAdded || undefined,
+				time_to_first_token_ms: result.streamResult?.timing?.timeToFirstTokenMs,
+				time_to_first_edit_ms: result.streamResult?.timing?.timeToFirstEditMs,
+				time_round_trip_ms: result.streamResult?.timing?.totalRoundTripMs,
+				cost_usd: result.streamResult?.usage?.totalCost,
+				completion_tokens: result.streamResult?.usage?.outputTokens,
+				raw_model_output: result.streamResult?.assistantMessage,
+				file_edited_hash: fileEditedHash,
+				parsed_tool_call_json: result.toolCalls ? JSON.stringify(result.toolCalls) : undefined
+			};
+
+			await insertResult(resultInput);
+		} catch (error) {
+			console.error(`Failed to store result in database for ${testId}:`, error);
+			// Continue execution - don't fail the test run
+		}
+	}
+
+	/**
+	 * Estimate token count for messages (rough approximation)
+	 */
+	public estimateTokens(messages: Anthropic.Messages.MessageParam[]): number { // Made public
+		let totalText = "";
+		for (const message of messages) {
+			if (Array.isArray(message.content)) {
+				for (const block of message.content) {
+					if (block.type === 'text') {
+						totalText += block.text + "\n";
+					}
+				}
+			} else if (typeof message.content === 'string') {
+				totalText += message.content + "\n";
+			}
+		}
+		return encoding.encode(totalText).length;
+	}
+
+	/**
+	 * Map error string to error enum
+	 */
+	private mapErrorToEnum(error?: string): number | undefined {
+		if (!error) return undefined;
+		
+		const errorMap: Record<string, number> = {
+			'no_tool_calls': 1,
+			'parsing_error': 2,
+			'diff_edit_error': 3,
+			'missing_original_diff_edit_tool_call_message': 4,
+			'api_error': 5,
+			'wrong_tool_call': 6,
+			'wrong_file_edited': 7,
+			'multi_tool_calls': 8,
+			'tool_call_params_undefined': 9,
+			'other_error': 99
+		};
+
+		return errorMap[error] || 99; // 99 for unknown errors
+	}
+
+	/**
+	 * convert our messages array into a properly formatted Anthropic messages array
+	 */
+	transformMessages(messages: InputMessage[]): Anthropic.Messages.MessageParam[] {
+		return messages.map((msg) => {
+			// Use TextBlockParam here for constructing the input message
+			const content: (Anthropic.TextBlockParam | Anthropic.ImageBlockParam)[] = []
+
+			if (msg.text) {
+				// This object now correctly matches the TextBlockParam type
+				content.push({ type: "text", text: msg.text })
+			}
+
+			if (msg.images && Array.isArray(msg.images)) {
+				const imageBlocks = formatResponse.imageBlocks(msg.images)
+				content.push(...imageBlocks)
+			}
+
+			return {
+				role: msg.role,
+				content: content,
+			}
+		})
+	}
+
+	/**
+	 * Generate the system prompt on the fly
+	 */
+	constructSystemPrompt(systemPromptDetails: SystemPromptDetails, systemPromptName: string) {
+		const systemPromptGenerator = systemPromptGeneratorLookup[systemPromptName]
+
+		const { cwd_value, browser_use, width, height, os_value, shell_value, home_value, mcp_string, user_custom_instructions } =
+			systemPromptDetails
+
+		const systemPrompt = systemPromptGenerator(
+			cwd_value,
+			browser_use,
+			width,
+			height,
+			os_value,
+			shell_value,
+			home_value,
+			mcp_string,
+			user_custom_instructions,
+		)
+
+		return systemPrompt
+	}
+
+	/**
+	 * Loads our test cases from a directory of json files
+	 */
+	loadTestCases(testDirectoryPath: string, isVerbose: boolean): TestCase[] {
+		const testCasesArray: TestCase[] = []
+		const dirents = fs.readdirSync(testDirectoryPath, { withFileTypes: true })
+
+		for (const dirent of dirents) {
+			if (dirent.isFile() && dirent.name.endsWith(".json")) {
+				const testFilePath = path.join(testDirectoryPath, dirent.name)
+				const fileContent = fs.readFileSync(testFilePath, "utf8")
+				const testCase: TestCase = JSON.parse(fileContent)
+
+				// Use the filename (without extension) as the test_id if not provided
+				if (!testCase.test_id) {
+					testCase.test_id = path.parse(dirent.name).name
+				}
+
+				// Filter out cases with missing file_contents
+				if (!testCase.file_contents || testCase.file_contents.trim() === "") {
+					log(isVerbose, `Skipping case ${testCase.test_id}: missing or empty file_contents.`);
+					continue;
+				}
+				testCasesArray.push(testCase)
+			}
+		}
+		return testCasesArray
+	}
+
+	/**
+	 * Saves the test results to the specified output directory.
+	 */
+	saveTestResults(results: TestResultSet, outputPath: string) {
+		// Ensure output directory exists
+		if (!fs.existsSync(outputPath)) {
+			fs.mkdirSync(outputPath, { recursive: true })
+		}
+
+		// Write each test result to its own file
+		for (const testId in results) {
+			const outputFilePath = path.join(outputPath, `${testId}.json`)
+			const testResult = results[testId]
+			fs.writeFileSync(outputFilePath, JSON.stringify(testResult, null, 2))
+		}
+	}
+
+	/**
+	 * Run a single test example
+	 */
+	async runSingleTest(testCase: ProcessedTestCase, testConfig: TestConfig, isVerbose: boolean = false): Promise<TestResult> {
+		if (testConfig.replay && !testCase.original_diff_edit_tool_call_message) {
+			return {
+				success: false,
+				error: "missing_original_diff_edit_tool_call_message",
+				errorString: `Test case ${testCase.test_id} is missing 'original_diff_edit_tool_call_message' for replay.`,
+			}
+		}
+
+		const customSystemPrompt = this.constructSystemPrompt(testCase.system_prompt_details, testConfig.system_prompt_name)
+
+		// messages don't include system prompt and are everything up to the first replace_in_file tool call which results in a diff edit error
+		const input: TestInput = {
+			apiKey: this.apiKey,
+			systemPrompt: customSystemPrompt,
+			messages: testCase.messages,
+			modelId: testConfig.model_id,
+			originalFile: testCase.file_contents,
+			originalFilePath: testCase.file_path,
+			parsingFunction: testConfig.parsing_function,
+			diffEditFunction: testConfig.diff_edit_function,
+			thinkingBudgetTokens: testConfig.thinking_tokens_budget,
+			originalDiffEditToolCallMessage: testConfig.replay ? testCase.original_diff_edit_tool_call_message : undefined,
+		}
+
+		if (isVerbose) {
+			log(isVerbose, `    Sending request to ${testConfig.model_id} for test case ${testCase.test_id}...`);
+		}
+		
+		return await runSingleEvaluation(input)
+	}
+
+	/**
+	 * Runs all the text examples synchonously
+	 */
+	async runAllTests(testCases: ProcessedTestCase[], testConfig: TestConfig, isVerbose: boolean): Promise<TestResultSet> {
+		const results: TestResultSet = {}
+
+		// Initialize database run
+		try {
+			await this.initializeDatabaseRun(testConfig, testCases, isVerbose);
+		} catch (error) {
+			log(isVerbose, `Warning: Failed to initialize database: ${error}`);
+		}
+
+		for (const testCase of testCases) {
+			results[testCase.test_id] = []
+
+			log(isVerbose, `-Running test: ${testCase.test_id}`)
+			for (let i = 0; i < testConfig.number_of_runs; i++) {
+				log(isVerbose, `  Attempt ${i+1}/${testConfig.number_of_runs} for ${testCase.test_id}...`);
+				const result = await this.runSingleTest(testCase, testConfig, isVerbose)
+				results[testCase.test_id].push(result)
+				
+				// Log result status
+				if (isVerbose) {
+					if (result.success) {
+						log(isVerbose, `  ✓ Attempt ${i+1} completed successfully`);
+					} else {
+						log(isVerbose, `  ✗ Attempt ${i+1} failed (error: ${result.error || 'unknown'})`);
+					}
+				}
+				
+				// Store result in database
+				try {
+					await this.storeResultInDatabase(result, testCase.test_id, testConfig.model_id);
+				} catch (error) {
+					log(isVerbose, `Warning: Failed to store result in database: ${error}`);
+				}
+			}
+		}
+		return results
+	}
+
+	/**
+	 * Runs all of the text examples asynchronously, with concurrency limit
+	 */
+	async runAllTestsParallel(
+		testCases: ProcessedTestCase[],
+		testConfig: TestConfig,
+		isVerbose: boolean,
+		maxConcurrency: number = 20,
+	): Promise<TestResultSet> {
+		const results: TestResultSet = {}
+		testCases.forEach((tc) => {
+			results[tc.test_id] = []
+		})
+
+		// Initialize database run
+		try {
+			await this.initializeDatabaseRun(testConfig, testCases, isVerbose);
+		} catch (error) {
+			log(isVerbose, `Warning: Failed to initialize database: ${error}`);
+		}
+
+		// Create a flat list of all individual runs we need to execute
+		const allRuns = testCases.flatMap((testCase) =>
+			Array(testConfig.number_of_runs)
+				.fill(null)
+				.map(() => testCase),
+		)
+
+		for (let i = 0; i < allRuns.length; i += maxConcurrency) {
+			const batch = allRuns.slice(i, i + maxConcurrency)
+
+			const batchPromises = batch.map((testCase) => {
+				log(isVerbose, `  Running test for ${testCase.test_id}...`);
+				return this.runSingleTest(testCase, testConfig, isVerbose).then((result) => ({
+					...result,
+					test_id: testCase.test_id,
+				}))
+			})
+
+			const batchResults = await Promise.all(batchPromises)
+
+			// Calculate the total cost for this batch
+			const batchCost = batchResults.reduce((total, result) => {
+				return total + (result.streamResult?.usage?.totalCost || 0)
+			}, 0)
+
+			// Populate the results dictionary and store in database
+			for (const result of batchResults) {
+				if (result.test_id) {
+					results[result.test_id].push(result)
+					
+					// Store result in database
+					try {
+						await this.storeResultInDatabase(result, result.test_id, testConfig.model_id);
+					} catch (error) {
+						log(isVerbose, `Warning: Failed to store result in database: ${error}`);
+					}
+				}
+			}
+
+			const batchNumber = i / maxConcurrency + 1
+			const totalBatches = Math.ceil(allRuns.length / maxConcurrency)
+			log(isVerbose, `-Completed batch ${batchNumber} of ${totalBatches}... (Batch Cost: $${batchCost.toFixed(6)})`)
+		}
+
+		return results
+	}
+
+	/**
+	 * Check if a test result is a valid attempt (no error_enum 1, 6, or 7)
+	 */
+	isValidAttempt(result: TestResult): boolean {
+		// Invalid if error is one of: no_tool_calls, wrong_tool_call, wrong_file_edited
+		const invalidErrors = ['no_tool_calls', 'wrong_tool_call', 'wrong_file_edited'];
+		return !invalidErrors.includes(result.error || '');
+	}
+
+	/**
+	 * Runs all tests for a specific model (assumes database run already initialized)
+	 * Keeps retrying until we get the requested number of valid attempts per case
+	 */
+	async runAllTestsForModel(testCases: ProcessedTestCase[], testConfig: TestConfig, isVerbose: boolean): Promise<TestResultSet> {
+		const results: TestResultSet = {}
+
+		for (const testCase of testCases) {
+			results[testCase.test_id] = []
+			let validAttempts = 0;
+			let totalAttempts = 0;
+
+			log(isVerbose, `-Running test: ${testCase.test_id}`)
+			
+			// Keep trying until we get the requested number of valid attempts
+			while (validAttempts < testConfig.number_of_runs) {
+				totalAttempts++;
+				log(isVerbose, `  Attempt ${totalAttempts} for ${testCase.test_id} (${validAttempts}/${testConfig.number_of_runs} valid so far)...`);
+				
+				const result = await this.runSingleTest(testCase, testConfig, isVerbose)
+				results[testCase.test_id].push(result)
+				
+				// Check if this was a valid attempt
+				const isValid = this.isValidAttempt(result);
+				if (isValid) {
+					validAttempts++;
+					log(isVerbose, `  ✓ Valid attempt ${validAttempts}/${testConfig.number_of_runs} completed (${result.success ? 'SUCCESS' : 'FAILED'})`);
+				} else {
+					log(isVerbose, `  ✗ Invalid attempt (error: ${result.error || 'unknown'})`);
+				}
+				
+				// Store result in database
+				try {
+					await this.storeResultInDatabase(result, testCase.test_id, testConfig.model_id);
+				} catch (error) {
+					log(isVerbose, `Warning: Failed to store result in database: ${error}`);
+				}
+				
+				// Safety check to prevent infinite loops - limit to 10 attempts per valid attempt requested
+				if (totalAttempts >= testConfig.number_of_runs * 10) {
+					log(isVerbose, `  ⚠️ Reached maximum attempts (${totalAttempts}) for test case ${testCase.test_id}. Only got ${validAttempts}/${testConfig.number_of_runs} valid attempts.`);
+					break;
+				}
+			}
+			
+			log(isVerbose, `  ✓ Completed test case ${testCase.test_id}: ${validAttempts}/${testConfig.number_of_runs} valid attempts (${totalAttempts} total attempts)`);
+		}
+		return results
+	}
+
+	/**
+	 * Print output of the tests
+	 */
+	printSummary(results: TestResultSet, isVerbose: boolean) {
+		let totalRuns = 0
+		let totalPasses = 0
+		let totalInputTokens = 0
+		let totalOutputTokens = 0
+		let totalCost = 0
+		let runsWithUsageData = 0
+		let totalDiffEditSuccesses = 0
+		let totalRunsWithToolCalls = 0
+		const testCaseIds = Object.keys(results)
+
+		log(isVerbose, "\n=== TEST SUMMARY ===")
+
+		for (const testId of testCaseIds) {
+			const testResults = results[testId]
+			const passedCount = testResults.filter((r) => r.success && r.diffEditSuccess).length
+			const runCount = testResults.length
+
+			totalRuns += runCount
+			totalPasses += passedCount
+
+			const runsWithToolCalls = testResults.filter((r) => r.success === true).length
+			const diffEditSuccesses = passedCount
+			totalRunsWithToolCalls += runsWithToolCalls
+			totalDiffEditSuccesses += diffEditSuccesses
+
+			// Accumulate token and cost data
+			for (const result of testResults) {
+				if (result.streamResult?.usage) {
+					totalInputTokens += result.streamResult.usage.inputTokens
+					totalOutputTokens += result.streamResult.usage.outputTokens
+					totalCost += result.streamResult.usage.totalCost
+					runsWithUsageData++
+				}
+			}
+
+			log(isVerbose, `\n--- Test Case: ${testId} ---`)
+			log(isVerbose, `  Runs: ${runCount}`)
+			log(isVerbose, `  Passed: ${passedCount}`)
+			log(isVerbose, `  Success Rate: ${runCount > 0 ? ((passedCount / runCount) * 100).toFixed(1) : "N/A"}%`)
+		}
+
+		log(isVerbose, "\n\n=== OVERALL SUMMARY ===")
+		log(isVerbose, `Total Test Cases: ${testCaseIds.length}`)
+		log(isVerbose, `Total Runs Executed: ${totalRuns}`)
+		log(isVerbose, `Overall Passed: ${totalPasses}`)
+		log(isVerbose, `Overall Failed: ${totalRuns - totalPasses}`)
+		log(isVerbose, `Overall Success Rate: ${totalRuns > 0 ? ((totalPasses / totalRuns) * 100).toFixed(1) : "N/A"}%`)
+
+		log(isVerbose, "\n\n=== OVERALL DIFF EDIT SUCCESS RATE ===")
+		if (totalRunsWithToolCalls > 0) {
+			const diffSuccessRate = (totalDiffEditSuccesses / totalRunsWithToolCalls) * 100
+			log(isVerbose, `Total Runs with Successful Tool Calls: ${totalRunsWithToolCalls}`)
+			log(isVerbose, `Total Runs with Successful Diff Edits: ${totalDiffEditSuccesses}`)
+			log(isVerbose, `Diff Edit Success Rate: ${diffSuccessRate.toFixed(1)}%`)
+		} else {
+			log(isVerbose, "No successful tool calls to analyze for diff edit success.")
+		}
+
+		log(isVerbose, "\n\n=== TOKEN & COST ANALYSIS ===")
+		if (runsWithUsageData > 0) {
+			log(isVerbose, `Total Input Tokens: ${totalInputTokens.toLocaleString()}`)
+			log(isVerbose, `Total Output Tokens: ${totalOutputTokens.toLocaleString()}`)
+			log(isVerbose, `Total Cost: $${totalCost.toFixed(6)}`)
+			log(isVerbose, "---")
+			log(
+				isVerbose,
+				`Avg Input Tokens / Run: ${(totalInputTokens / runsWithUsageData).toLocaleString(undefined, {
+					maximumFractionDigits: 0,
+				})}`,
+			)
+			log(
+				isVerbose,
+				`Avg Output Tokens / Run: ${(totalOutputTokens / runsWithUsageData).toLocaleString(undefined, {
+					maximumFractionDigits: 0,
+				})}`,
+			)
+			log(isVerbose, `Avg Cost / Run: $${(totalCost / runsWithUsageData).toFixed(6)}`)
+		} else {
+			log(isVerbose, "No usage data available to analyze.")
+		}
+	}
+}
+
+async function main() {
+	interface EvaluationTask {
+		modelId: string;
+		testCase: ProcessedTestCase;
+		testConfig: TestConfig;
+	}
+
+	const program = new Command()
+
+	const defaultTestPath = path.join(__dirname, "cases")
+	const defaultOutputPath = path.join(__dirname, "results")
+
+	program
+		.name("TestRunner")
+		.description("Run evaluation tests for diff editing")
+		.version("1.0.0")
+		.option("--test-path <path>", "Path to the directory containing test case JSON files", defaultTestPath)
+		.option("--output-path <path>", "Path to the directory to save the test output JSON files", defaultOutputPath)
+		.option("--model-ids <model_ids>", "Comma-separated list of model IDs to test")
+		.option("--system-prompt-name <name>", "The name of the system prompt to use", "basicSystemPrompt")
+		.option("-n, --valid-attempts-per-case <number>", "Number of valid attempts per test case per model (will retry until this many valid attempts are collected)", "1")
+		.option("--max-cases <number>", "Maximum number of test cases to run (limits total cases loaded)")
+		.option("--parsing-function <name>", "The parsing function to use", "parseAssistantMessageV2")
+		.option("--diff-edit-function <name>", "The diff editing function to use", "constructNewFileContentV2")
+		.option("--thinking-budget <tokens>", "Set the thinking tokens budget", "0")
+		.option("--parallel", "Run tests in parallel", false)
+		.option("--replay", "Run evaluation from a pre-recorded LLM output, skipping the API call", false)
+		.option("-v, --verbose", "Enable verbose logging", false)
+		.option("--max-concurrency <number>", "Maximum number of parallel requests", "80")
+
+
+	program.parse(process.argv)
+
+	const options = program.opts()
+	const isVerbose = options.verbose
+	const testPath = options.testPath
+	const outputPath = options.outputPath
+	const maxConcurrency = parseInt(options.maxConcurrency, 10);
+
+	// Parse model IDs from comma-separated string
+	const modelIds = options.modelIds ? options.modelIds.split(',').map(id => id.trim()) : [];
+	if (modelIds.length === 0) {
+		console.error("Error: --model-ids is required and must contain at least one model ID");
+		process.exit(1);
+	}
+
+	const validAttemptsPerCase = parseInt(options.validAttemptsPerCase, 10);
+	
+	try {
+		const startTime = Date.now()
+
+		// Load OpenRouter model data first
+		openRouterModelDataGlobal = await loadOpenRouterModelData(isVerbose);
+		if (Object.keys(openRouterModelDataGlobal).length === 0 && isVerbose) {
+			log(isVerbose, "Warning: Could not load OpenRouter model data. Context window filtering might be affected for OpenRouter models.");
+		}
+
+		const runner = new NodeTestRunner(options.replay)
+		let allLoadedTestCases = runner.loadTestCases(testPath, isVerbose) // Pass isVerbose
+		
+		const allProcessedTestCasesGlobal: ProcessedTestCase[] = allLoadedTestCases.map((tc) => ({
+			...tc,
+			messages: runner.transformMessages(tc.messages),
+		}));
+
+		log(isVerbose, `-Loaded ${allLoadedTestCases.length} initial test cases.`)
+		log(isVerbose, `-Testing ${modelIds.length} model(s): ${modelIds.join(', ')}`)
+		log(isVerbose, `-Target: ${validAttemptsPerCase} valid attempts per test case per model (will retry until this many valid attempts are collected)`)
+		if (options.replay) {
+			log(isVerbose, `-Running in REPLAY mode. No API calls will be made.`)
+		}
+		log(isVerbose, "Starting tests...\n")
+
+		// Determine the smallest context window among all specified models
+		let smallestContextWindow = Infinity;
+		for (const modelId of modelIds) {
+			let modelInfo = openRouterModelDataGlobal[modelId];
+			if (!modelInfo) {
+				const foundKey = Object.keys(openRouterModelDataGlobal).find(
+					key => key.includes(modelId) || modelId.includes(key)
+				);
+				if (foundKey) modelInfo = openRouterModelDataGlobal[foundKey];
+			}
+			const currentModelContext = modelInfo?.contextWindow;
+			if (currentModelContext && currentModelContext > 0) {
+				if (currentModelContext < smallestContextWindow) {
+					smallestContextWindow = currentModelContext;
+				}
+			} else {
+				log(isVerbose, `Warning: Context window for model ${modelId} is unknown or zero. It will not constrain the test case selection.`);
+			}
+		}
+
+		if (smallestContextWindow === Infinity) {
+			log(isVerbose, "Warning: Could not determine a common smallest context window. Proceeding with all loaded cases, context issues may occur.");
+		} else {
+			log(isVerbose, `Smallest common context window (with padding consideration) across specified models: ${smallestContextWindow} (target for filtering: ${smallestContextWindow - 20000})`);
+		}
+		
+		let eligibleCasesForThisRun = [...allLoadedTestCases];
+		if (smallestContextWindow !== Infinity && smallestContextWindow > 20000) { // Only filter if a valid smallest window is found
+			const originalCaseCount = eligibleCasesForThisRun.length;
+			eligibleCasesForThisRun = eligibleCasesForThisRun.filter(tc => {
+				const systemPromptText = runner.constructSystemPrompt(tc.system_prompt_details, options.systemPromptName);
+				const systemPromptTokens = encoding.encode(systemPromptText).length;
+				const messagesTokens = runner.estimateTokens(runner.transformMessages(tc.messages));
+				const totalInputTokens = systemPromptTokens + messagesTokens;
+				return totalInputTokens + 20000 <= smallestContextWindow; // 20k padding
+			});
+			log(isVerbose, `Filtered to ${eligibleCasesForThisRun.length} cases (from ${originalCaseCount}) to fit smallest context window of ${smallestContextWindow} (with padding).`);
+		}
+
+		// Apply max-cases limit if specified, to the context-filtered list
+		if (options.maxCases && options.maxCases > 0 && eligibleCasesForThisRun.length > options.maxCases) {
+			log(isVerbose, `Limiting to ${options.maxCases} test cases (out of ${eligibleCasesForThisRun.length} eligible).`);
+			eligibleCasesForThisRun = eligibleCasesForThisRun.slice(0, options.maxCases);
+		}
+
+		if (eligibleCasesForThisRun.length === 0) {
+			log(isVerbose, `No eligible test cases found after filtering for all specified models. Exiting.`);
+			process.exit(0);
+		}
+		
+		const processedEligibleCasesForRun: ProcessedTestCase[] = eligibleCasesForThisRun.map((tc) => ({
+			...tc,
+			messages: runner.transformMessages(tc.messages),
+		}));
+
+		// Initialize ONE database run for ALL models using the commonly eligible cases
+		const runDescription = `Models: ${modelIds.join(', ')}, Common Cases: ${processedEligibleCasesForRun.length}, Valid attempts per case: ${validAttemptsPerCase}`;
+		await runner.initializeMultiModelRun(processedEligibleCasesForRun, options.systemPromptName, options.parsingFunction, options.diffEditFunction, runDescription, isVerbose);
+
+		// Create a global task queue
+		const globalTaskQueue: EvaluationTask[] = modelIds.flatMap(modelId => 
+			processedEligibleCasesForRun.map(testCase => ({
+				modelId,
+				testCase,
+				testConfig: {
+					model_id: modelId,
+					system_prompt_name: options.systemPromptName,
+					number_of_runs: validAttemptsPerCase,
+					parsing_function: options.parsingFunction,
+					diff_edit_function: options.diffEditFunction,
+					thinking_tokens_budget: parseInt(options.thinkingBudget, 10),
+					replay: options.replay,
+				}
+			}))
+		);
+
+		const results: TestResultSet = {};
+		const taskStates: Record<string, { valid: number; total: number; pending: number }> = {};
+
+		globalTaskQueue.forEach(({ modelId, testCase }) => {
+			const taskId = `${modelId}-${testCase.test_id}`;
+			taskStates[taskId] = { valid: 0, total: 0, pending: 0 };
+			if (!results[testCase.test_id]) {
+				results[testCase.test_id] = [];
+			}
+		});
+
+		let remainingTasks = [...globalTaskQueue];
+
+		while (remainingTasks.length > 0) {
+			const batch: EvaluationTask[] = [];
+			for (const task of remainingTasks) {
+				if (batch.length >= maxConcurrency) break;
+				const taskId = `${task.modelId}-${task.testCase.test_id}`;
+				if ((taskStates[taskId].valid + taskStates[taskId].pending) < validAttemptsPerCase) {
+					batch.push(task);
+					taskStates[taskId].pending++;
+				}
+			}
+
+			if (batch.length === 0) {
+				await new Promise(resolve => setTimeout(resolve, 100));
+				continue;
+			}
+
+			const batchPromises = batch.map(task => {
+				const taskId = `${task.modelId}-${task.testCase.test_id}`;
+				taskStates[taskId].total++;
+				log(isVerbose, `  Attempt ${taskStates[taskId].total} for ${task.testCase.test_id} with ${task.modelId} (${taskStates[taskId].valid} valid, ${taskStates[taskId].pending - 1} pending)...`);
+				return runner.runSingleTest(task.testCase, task.testConfig, isVerbose).then(result => ({
+					...result,
+					test_id: task.testCase.test_id,
+					modelId: task.modelId,
+				}));
+			});
+
+			const batchResults = await Promise.all(batchPromises);
+
+			for (const result of batchResults) {
+				const taskId = `${result.modelId}-${result.test_id}`;
+				taskStates[taskId].pending--;
+				results[result.test_id].push(result);
+
+				if (runner.isValidAttempt(result)) {
+					taskStates[taskId].valid++;
+					log(isVerbose, `  ✓ Valid attempt ${taskStates[taskId].valid}/${validAttemptsPerCase} for ${result.test_id} with ${result.modelId} completed (${result.success ? 'SUCCESS' : 'FAILED'})`);
+				} else {
+					log(isVerbose, `  ✗ Invalid attempt for ${result.test_id} with ${result.modelId} (error: ${result.error || 'unknown'})`);
+				}
+
+				await runner.storeResultInDatabase(result, result.test_id, result.modelId);
+			}
+
+			remainingTasks = remainingTasks.filter(task => {
+				const taskId = `${task.modelId}-${task.testCase.test_id}`;
+				if (taskStates[taskId].total >= validAttemptsPerCase * 10) {
+					log(isVerbose, `  ⚠️ Reached maximum attempts for ${task.testCase.test_id} with ${task.modelId}.`);
+					return false;
+				}
+				return taskStates[taskId].valid < validAttemptsPerCase;
+			});
+
+			const batchCost = batchResults.reduce((total, result) => total + (result.streamResult?.usage?.totalCost || 0), 0);
+			log(isVerbose, `-Completed batch... (Batch Cost: $${batchCost.toFixed(6)}, Remaining tasks: ${remainingTasks.length})`);
+		}
+
+		// Print summary for each model
+		for (const modelId of modelIds) {
+			const modelResults: TestResultSet = {};
+			Object.keys(results).forEach(testId => {
+				modelResults[testId] = results[testId].filter(r => (r as any).modelId === modelId);
+			});
+			log(isVerbose, `\n=== Results for Model: ${modelId} ===`);
+			runner.printSummary(modelResults, isVerbose);
+		}
+
+		const endTime = Date.now()
+		const durationSeconds = ((endTime - startTime) / 1000).toFixed(2)
+		log(isVerbose, `\n-Total execution time: ${durationSeconds} seconds`)
+
+		log(isVerbose, `\n✓ All results stored in database. Use the dashboard to view results.`)
+	} catch (error) {
+		console.error("\nError running tests:", error)
+		process.exit(1)
+	}
+}
+
+if (require.main === module) {
+	main()
+}

evals/diff-edits/dashboard/.streamlit/config.toml

@@ -0,0 +1,8 @@
+[theme]
+base="dark"
+
+[browser]
+gatherUsageStats = false
+
+[server]
+headless = true

evals/diff-edits/dashboard/README.md

@@ -0,0 +1,159 @@
+# 🚀 The Sickest Diff Edits Evaluation Dashboard Ever!
+
+A beautiful, modern Streamlit dashboard for visualizing and analyzing diff editing evaluation results with deep drill-down capabilities.
+
+## ✨ Features
+
+### 🎯 **Smart Model Comparison**
+- **Latest Run Focus**: Automatically loads and displays your most recent evaluation run
+- **Beautiful Performance Cards**: Each model gets a stunning card with performance grades (A+ to C)
+- **Best Performer Highlighting**: The top model gets special styling and a trophy 🏆
+- **Interactive Charts**: Success rate comparisons and latency vs cost analysis
+
+### 🔍 **Deep Drill-Down Analysis**
+- **Individual Result Inspection**: Click any model to see detailed results
+- **Side-by-Side File Views**: See original file content with line numbers
+- **Parsed Tool Call Analysis**: View exactly what the model tried to do
+- **Error Analysis**: Detailed error information for failed attempts
+- **Success Metrics**: Line changes, edit counts, and timing breakdowns
+
+### 🎨 **Aesthetic Design**
+- **Modern UI**: Custom CSS with Inter font, gradients, and shadows
+- **Responsive Layout**: Looks great on any screen size
+- **Color-Coded Performance**: Green for excellent, yellow for good, red for poor
+- **Smooth Animations**: Hover effects and transitions
+- **Professional Styling**: Clean, modern design that looks amazing
+
+### 📊 **Comprehensive Metrics**
+- **Success Rates**: Color-coded percentages with performance grades
+- **Timing Analysis**: First token, first edit, and round trip times
+- **Cost Tracking**: Per-result and total cost analysis
+- **Token Metrics**: Context tokens and completion tokens
+- **Edit Statistics**: Number of edits, lines added/deleted
+
+## 🚀 Quick Start
+
+1. **Install dependencies**:
+   ```bash
+   cd diff-edits/dashboard
+   pip install -r requirements.txt
+   ```
+
+2. **Launch the dashboard**:
+   ```bash
+   streamlit run app.py
+   ```
+   
+   Or use the convenient launch script:
+   ```bash
+   ./launch.sh
+   ```
+
+3. **Open your browser** to http://localhost:8501
+
+## 🎯 Dashboard Sections
+
+### **Hero Section**
+- Beautiful gradient header with run information
+- Key metrics overview (models tested, total results, success rate, cost)
+
+### **Model Performance Cards**
+- Each model displayed as a beautiful card
+- Large success rate display with color coding
+- Performance grade badges (A+, A, B+, B, C+, C)
+- Key metrics: latency, cost, results count, first token time
+- "Drill Down" button for detailed analysis
+
+### **Performance Analytics**
+- Interactive bar chart showing success rates
+- Scatter plot of latency vs cost with bubble sizes
+- Hover details and zoom capabilities
+
+### **Detailed Analysis (Drill-Down)**
+- Model-specific success rate, latency, and cost metrics
+- Individual result selector with status icons
+- Tabbed interface for different views:
+
+#### 📄 **File & Edits Tab**
+- **Side-by-side view**: Original file content with line numbers
+- **Edit analysis**: Success/failure status with detailed metrics
+- **Error display**: Clear error information for failed attempts
+- **Success metrics**: Lines added/deleted, number of edits
+- **Parsed tool calls**: JSON view of what the model attempted
+
+#### 🤖 **Raw Output Tab**
+- Complete raw model output in a code viewer
+- Monospace font for easy reading
+
+#### 🔧 **Parsed Tool Call Tab**
+- Pretty-printed JSON of parsed tool calls
+- Diff block visualization for replace_in_file calls
+- Error handling for malformed JSON
+
+#### 📊 **Metrics Tab**
+- Detailed timing metrics (first token, first edit, round trip)
+- Token and cost information
+- Context size and completion tokens
+
+## 🛠 **Technical Features**
+
+### **Smart Data Loading**
+- Automatic latest run detection
+- Efficient SQL queries with proper JOINs
+- Streamlit caching for performance
+- Error handling for missing data
+
+### **Interactive Navigation**
+- Session state management for drill-down views
+- Back button to return to overview
+- Smooth transitions between views
+
+### **Beautiful Styling**
+- Custom CSS with Google Fonts (Inter)
+- Gradient backgrounds and shadows
+- Hover effects and animations
+- Color-coded performance indicators
+- Professional card-based layout
+
+### **Responsive Design**
+- Works on desktop, tablet, and mobile
+- Flexible column layouts
+- Scalable text and metrics
+
+## 🎨 **Design Philosophy**
+
+This dashboard follows modern design principles:
+- **Clarity**: Information is easy to find and understand
+- **Beauty**: Visually appealing with professional styling
+- **Functionality**: Deep drill-down capabilities for detailed analysis
+- **Performance**: Fast loading with efficient data queries
+- **Usability**: Intuitive navigation and clear visual hierarchy
+
+## 📊 **Data Visualization**
+
+- **Plotly Charts**: Interactive, professional-looking visualizations
+- **Color Coding**: Consistent color scheme for performance levels
+- **Performance Badges**: A+ to C grading system
+- **Status Icons**: ✅ for success, ❌ for failure
+- **Metric Cards**: Clean, card-based metric display
+
+## 🔧 **Customization**
+
+The dashboard is highly customizable:
+- **CSS Styling**: Easy to modify colors, fonts, and layouts
+- **Performance Grades**: Adjustable thresholds for A/B/C grades
+- **Metrics Display**: Add or remove metrics as needed
+- **Chart Types**: Easily swap chart types or add new visualizations
+
+## 🚀 **Future Enhancements**
+
+Potential additions:
+- **Historical Trends**: Compare performance across multiple runs
+- **Export Functionality**: Download results as CSV/PDF
+- **Real-time Updates**: Auto-refresh for ongoing evaluations
+- **Custom Filters**: Filter by date range, model type, etc.
+- **Comparison Mode**: Side-by-side model comparisons
+
+---
+
+**This is the sickest eval dashboard ever!** 🔥 It combines beautiful design with powerful analysis capabilities, making it easy to understand model performance at a glance while providing deep drill-down capabilities for detailed investigation.

evals/diff-edits/dashboard/app.py

@@ -0,0 +1,948 @@
+import streamlit as st
+import sqlite3
+import pandas as pd
+import plotly.express as px
+import plotly.graph_objects as go
+from plotly.subplots import make_subplots
+import numpy as np
+from datetime import datetime
+import os
+import json
+import difflib
+# import mimetypes # No longer needed here if guess_language_from_filepath handles it
+from utils import get_database_connection, guess_language_from_filepath # Import from utils
+
+# Page config
+st.set_page_config(
+    page_title="Diff Edits Evaluation Dashboard",
+    page_icon="📊",
+    layout="wide",
+    initial_sidebar_state="expanded"
+)
+
+# Custom CSS for beautiful styling
+st.markdown("""
+<style>
+    /* Import Google Fonts */
+    @import url('https://fonts.googleapis.com/css2?family=Azeret+Mono:wght@400;700&display=swap');
+    
+    /* Global Styles */
+    .main {
+        font-family: 'Azeret Mono', monospace;
+    }
+    
+    /* Hero Section */
+    .hero-container {
+        background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
+        padding: 2rem;
+        border-radius: 15px;
+        margin-bottom: 2rem;
+        color: white;
+        text-align: center;
+    }
+    
+    .hero-title {
+        font-size: 3rem;
+        font-weight: 700;
+        margin-bottom: 0.5rem;
+        text-shadow: 2px 2px 4px rgba(0,0,0,0.3);
+    }
+    
+    .hero-subtitle {
+        font-size: 1.2rem;
+        font-weight: 300;
+        opacity: 0.9;
+    }
+    
+    /* Model Performance Cards */
+    .model-card {
+        background: white;
+        border-radius: 15px;
+        padding: 1.5rem;
+        margin: 1rem 0;
+        box-shadow: 0 8px 32px rgba(0,0,0,0.1);
+        border: 1px solid rgba(255,255,255,0.2);
+        transition: transform 0.3s ease, box-shadow 0.3s ease;
+    }
+    
+    .model-card:hover {
+        transform: translateY(-5px);
+        box-shadow: 0 12px 40px rgba(0,0,0,0.15);
+    }
+    
+    .model-card.best-performer {
+        border: 2px solid #00D4AA;
+        background: linear-gradient(135deg, #f0fdf4 0%, #ecfdf5 100%);
+    }
+    
+    .model-name {
+        font-size: 1.5rem;
+        font-weight: 600;
+        margin-bottom: 1rem;
+        color: #1f2937;
+    }
+    
+    .success-rate {
+        font-size: 3rem;
+        font-weight: 700;
+        margin-bottom: 0.5rem;
+    }
+    
+    .success-rate.excellent { color: #10b981; }
+    .success-rate.good { color: #f59e0b; }
+    .success-rate.poor { color: #ef4444; }
+    
+    .metric-row {
+        display: flex;
+        justify-content: space-between;
+        margin: 0.5rem 0;
+        padding: 0.5rem;
+        background: rgba(0,0,0,0.02);
+        border-radius: 8px;
+    }
+    
+    .metric-label {
+        font-weight: 500;
+        color: #6b7280;
+    }
+    
+    .metric-value {
+        font-weight: 600;
+        color: #1f2937;
+    }
+    
+    /* Performance Badge */
+    .performance-badge {
+        display: inline-block;
+        padding: 0.25rem 0.75rem;
+        border-radius: 20px;
+        font-weight: 600;
+        font-size: 0.875rem;
+        margin-left: 1rem;
+    }
+    
+    .badge-a { background: #10b981; color: white; }
+    .badge-b { background: #f59e0b; color: white; }
+    .badge-c { background: #ef4444; color: white; }
+    
+    /* Comparison Charts */
+    .chart-container {
+        background: white;
+        border-radius: 15px;
+        padding: 1.5rem;
+        margin: 1rem 0;
+        box-shadow: 0 4px 20px rgba(0,0,0,0.08);
+    }
+    
+    /* Result Detail Modal */
+    .result-detail {
+        background: white;
+        border-radius: 15px;
+        padding: 2rem;
+        margin: 1rem 0;
+        box-shadow: 0 8px 32px rgba(0,0,0,0.1);
+    }
+    
+    .file-viewer {
+        background: #f8fafc;
+        border: 1px solid #e2e8f0;
+        border-radius: 8px;
+        padding: 1rem;
+        font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', monospace;
+        font-size: 0.875rem;
+        line-height: 1.5;
+        overflow-x: auto;
+    }
+    
+    .diff-added {
+        background-color: #dcfce7;
+        color: #166534;
+    }
+    
+    .diff-removed {
+        background-color: #fef2f2;
+        color: #dc2626;
+    }
+    
+    .error-display {
+        background: #fef2f2;
+        border: 1px solid #fecaca;
+        border-radius: 8px;
+        padding: 1rem;
+        color: #dc2626;
+        font-family: monospace;
+    }
+    
+    /* Sidebar Styling */
+    .sidebar .sidebar-content {
+        background: linear-gradient(180deg, #f8fafc 0%, #f1f5f9 100%);
+    }
+    
+    /* Custom Metrics */
+    .custom-metric {
+        text-align: center;
+        padding: 1rem;
+        background: white;
+        border-radius: 10px;
+        box-shadow: 0 2px 10px rgba(0,0,0,0.05);
+        margin: 0.5rem 0;
+    }
+    
+    .custom-metric-value {
+        font-size: 2rem;
+        font-weight: 700;
+        color: #1f2937;
+    }
+    
+    .custom-metric-label {
+        font-size: 0.875rem;
+        color: #6b7280;
+        font-weight: 500;
+        margin-top: 0.25rem;
+    }
+</style>
+""", unsafe_allow_html=True)
+
+# Enhanced data loading functions
+@st.cache_data
+def load_all_runs():
+    """Load all evaluation runs"""
+    conn = get_database_connection()
+    
+    query = """
+    SELECT run_id, description, created_at, system_prompt_hash
+    FROM runs 
+    ORDER BY created_at DESC
+    """
+    
+    return pd.read_sql_query(query, conn)
+
+@st.cache_data
+def load_run_comparison(run_id):
+    """Load a specific run with model comparison data"""
+    conn = get_database_connection()
+    
+    # Get the run details
+    run_query = f"""
+    SELECT run_id, description, created_at, system_prompt_hash
+    FROM runs 
+    WHERE run_id = '{run_id}'
+    """
+    run_data = pd.read_sql_query(run_query, conn)
+    
+    if run_data.empty:
+        return None, None
+    
+    # Get model performance for this run
+    model_perf_query = f"""
+    SELECT 
+        res.model_id,
+        COUNT(*) as total_results,
+        AVG(CASE WHEN res.succeeded THEN 1.0 ELSE 0.0 END) as success_rate,
+        AVG(res.cost_usd) as avg_cost,
+        SUM(res.cost_usd) as total_cost,
+        AVG(res.time_to_first_token_ms) as avg_first_token_ms,
+        AVG(res.time_to_first_edit_ms) as avg_first_edit_ms,
+        AVG(res.time_round_trip_ms) as avg_round_trip_ms,
+        AVG(res.completion_tokens) as avg_completion_tokens,
+        AVG(res.num_edits) as avg_num_edits,
+        MIN(res.time_round_trip_ms) as min_round_trip_ms,
+        MAX(res.time_round_trip_ms) as max_round_trip_ms
+    FROM results res
+    JOIN cases c ON res.case_id = c.case_id
+    WHERE c.run_id = '{run_id}'
+      AND (res.error_enum NOT IN (1, 6, 7) OR res.error_enum IS NULL)  -- Exclude: no_tool_calls, wrong_tool_call, wrong_file_edited
+    GROUP BY res.model_id
+    ORDER BY success_rate DESC, avg_round_trip_ms ASC
+    """
+    
+    model_performance = pd.read_sql_query(model_perf_query, conn)
+    
+    return run_data.iloc[0], model_performance
+
+@st.cache_data
+def load_latest_run_comparison():
+    """Load the latest run with model comparison data"""
+    conn = get_database_connection()
+    
+    # Get the latest run
+    latest_run_query = """
+    SELECT run_id, description, created_at, system_prompt_hash
+    FROM runs 
+    ORDER BY created_at DESC 
+    LIMIT 1
+    """
+    latest_run = pd.read_sql_query(latest_run_query, conn)
+    
+    if latest_run.empty:
+        return None, None
+    
+    return load_run_comparison(latest_run.iloc[0]['run_id'])
+
+@st.cache_data
+def load_detailed_results(run_id, model_id=None, valid_only=False):
+    """Load detailed results for drill-down analysis"""
+    conn = get_database_connection()
+    
+    where_clause = f"WHERE c.run_id = '{run_id}'"
+    if model_id:
+        where_clause += f" AND res.model_id = '{model_id}'"
+    
+    # Option to filter out invalid attempts
+    if valid_only:
+        where_clause += " AND (res.error_enum NOT IN (1, 6, 7) OR res.error_enum IS NULL)"
+    
+    query = f"""
+    SELECT 
+        res.*,
+        c.task_id,
+        c.description as case_description,
+        c.tokens_in_context,
+        sp.name as system_prompt_name,
+        pf.name as processing_functions_name,
+        orig_f.filepath as original_filepath,
+        orig_f.content as original_file_content,
+        edit_f.filepath as edited_filepath,
+        edit_f.content as edited_file_content
+    FROM results res
+    JOIN cases c ON res.case_id = c.case_id
+    LEFT JOIN system_prompts sp ON c.system_prompt_hash = sp.hash
+    LEFT JOIN processing_functions pf ON res.processing_functions_hash = pf.hash
+    LEFT JOIN files orig_f ON c.file_hash = orig_f.hash
+    LEFT JOIN files edit_f ON res.file_edited_hash = edit_f.hash
+    {where_clause}
+    ORDER BY res.created_at DESC
+    """
+    
+    return pd.read_sql_query(query, conn)
+
+def get_performance_grade(success_rate):
+    """Get performance grade based on success rate"""
+    if success_rate >= 0.9:
+        return "A+", "excellent"
+    elif success_rate >= 0.8:
+        return "A", "excellent"
+    elif success_rate >= 0.7:
+        return "B+", "good"
+    elif success_rate >= 0.6:
+        return "B", "good"
+    elif success_rate >= 0.5:
+        return "C+", "good"
+    else:
+        return "C", "poor"
+
+def render_hero_section(current_run, model_performance):
+    """Render the hero section with key metrics"""
+    run_title = current_run['description'] if current_run['description'] else f"Run {current_run['run_id'][:8]}..."
+    st.markdown(f"""
+    <div class="hero-container">
+        <div class="hero-title">Diff Edit Evaluation Results</div>
+        <div class="hero-subtitle">A comprehensive analysis of model performance on code editing tasks.</div>
+        <div class="hero-subtitle" style="font-size: 0.9rem; margin-top: 10px;">
+            <strong>Current Run:</strong> {run_title} • {current_run['created_at']}
+        </div>
+    </div>
+    """, unsafe_allow_html=True)
+    
+    # Key metrics row
+    col1, col2, col3, col4 = st.columns(4)
+    
+    total_results = model_performance['total_results'].sum()
+    overall_success = model_performance['success_rate'].mean()
+    total_cost = model_performance['total_cost'].sum()
+    avg_latency = model_performance['avg_round_trip_ms'].mean()
+    
+    with col1:
+        st.markdown(f"""
+        <div class="custom-metric">
+            <div class="custom-metric-value">{len(model_performance)}</div>
+            <div class="custom-metric-label">Models Tested</div>
+        </div>
+        """, unsafe_allow_html=True)
+    
+    with col2:
+        st.markdown(f"""
+        <div class="custom-metric">
+            <div class="custom-metric-value">{total_results}</div>
+            <div class="custom-metric-label">Valid Results</div>
+        </div>
+        """, unsafe_allow_html=True)
+    
+    with col3:
+        success_color = "#10b981" if overall_success > 0.8 else "#f59e0b" if overall_success > 0.6 else "#ef4444"
+        st.markdown(f"""
+        <div class="custom-metric">
+            <div class="custom-metric-value" style="color: {success_color}">{overall_success:.1%}</div>
+            <div class="custom-metric-label">Avg Success Rate</div>
+        </div>
+        """, unsafe_allow_html=True)
+    
+    with col4:
+        st.markdown(f"""
+        <div class="custom-metric">
+            <div class="custom-metric-value">${total_cost:.3f}</div>
+            <div class="custom-metric-label">Total Cost</div>
+        </div>
+        """, unsafe_allow_html=True)
+
+def render_model_comparison_cards(model_performance):
+    """Render beautiful model comparison cards"""
+    st.markdown("## Model Leaderboard")
+    
+    # Find best performer
+    best_model = model_performance.iloc[0]['model_id']
+    
+    for idx, model in model_performance.iterrows():
+        is_best = model['model_id'] == best_model
+        grade, grade_class = get_performance_grade(model['success_rate'])
+        
+        # Create a container for each model
+        with st.container():
+            col1, col2 = st.columns([3, 1])
+            
+            with col1:
+                # Use Streamlit's native components instead of raw HTML
+                if is_best:
+                    st.success(f"**{model['model_id']}** - Best Performer")
+                else:
+                    st.info(f"**{model['model_id']}**")
+                
+                # Success rate with color coding
+                success_rate = model['success_rate']
+                if success_rate >= 0.8:
+                    st.success(f"**Success Rate:** {success_rate:.1%} ({grade})")
+                elif success_rate >= 0.6:
+                    st.warning(f"**Success Rate:** {success_rate:.1%} ({grade})")
+                else:
+                    st.error(f"**Success Rate:** {success_rate:.1%} ({grade})")
+                
+                # Metrics in columns
+                metric_col1, metric_col2, metric_col3, metric_col4 = st.columns(4)
+                
+                with metric_col1:
+                    st.metric("Avg Latency", f"{model['avg_round_trip_ms']:.0f}ms")
+                
+                with metric_col2:
+                    st.metric("Avg Cost", f"${model['avg_cost']:.4f}")
+                
+                with metric_col3:
+                    st.metric("Valid Results", f"{model['total_results']}")
+                
+                with metric_col4:
+                    st.metric("First Token", f"{model['avg_first_token_ms']:.0f}ms")
+            
+            with col2:
+                st.write("")  # Add some spacing
+                if st.button(f"Drill Down", key=f"drill_{model['model_id']}", use_container_width=True):
+                    st.session_state.drill_down_model = model['model_id']
+            
+            st.divider()  # Add a divider between models
+
+def render_comparison_charts(model_performance):
+    """Render interactive comparison charts"""
+    st.markdown("## Performance Analysis")
+    
+    col1, col2 = st.columns(2)
+
+    with col1:
+        # Time to First Edit
+        fig_first_edit = px.bar(
+            model_performance,
+            x='model_id',
+            y='avg_first_edit_ms',
+            title="Time to First Edit",
+            labels={'avg_first_edit_ms': 'Time to First Edit (ms)', 'model_id': 'Model'},
+            color='avg_first_edit_ms',
+            color_continuous_scale='bluered',
+            text='avg_first_edit_ms',
+            template='plotly_dark'
+        )
+        fig_first_edit.update_traces(texttemplate='%{text:.0f}ms', textposition='outside')
+        fig_first_edit.update_layout(
+            showlegend=False,
+            plot_bgcolor='rgba(0,0,0,0)',
+            paper_bgcolor='rgba(0,0,0,0)',
+            font=dict(family="Azeret Mono, monospace"),
+            margin=dict(t=50)
+        )
+        st.plotly_chart(fig_first_edit, use_container_width=True)
+
+    with col2:
+        # Latency vs Cost Scatter
+        fig_scatter = px.scatter(
+            model_performance,
+            x='avg_round_trip_ms',
+            y='avg_cost',
+            size='total_results',
+            color='success_rate',
+            hover_name='model_id',
+            title="Latency vs Cost Analysis",
+            labels={
+                'avg_round_trip_ms': 'Avg Round Trip (ms)',
+                'avg_cost': 'Avg Cost ($)',
+                'success_rate': 'Success Rate',
+                'total_results': 'Valid Results'
+            },
+            color_continuous_scale='RdYlGn',
+            template='plotly_dark'
+        )
+        fig_scatter.update_layout(
+            plot_bgcolor='rgba(0,0,0,0)',
+            paper_bgcolor='rgba(0,0,0,0)',
+            font=dict(family="Azeret Mono, monospace")
+        )
+        st.plotly_chart(fig_scatter, use_container_width=True)
+
+def render_detailed_analysis(run_id, model_id):
+    """Render detailed drill-down analysis"""
+    st.markdown(f"## Detailed Analysis: {model_id}")
+    
+    # Load all results (including invalid attempts)
+    detailed_results = load_detailed_results(run_id, model_id)
+    
+    # Also load only valid results for metrics
+    valid_results = load_detailed_results(run_id, model_id, valid_only=True)
+    
+    if detailed_results.empty:
+        st.warning("No detailed results found.")
+        return
+    
+    # Show total vs valid results
+    st.info(f"Showing all {len(detailed_results)} results ({len(valid_results)} valid, {len(detailed_results) - len(valid_results)} invalid)")
+    
+    # Results overview
+    col1, col2, col3 = st.columns(3)
+    
+    with col1:
+        success_count = valid_results['succeeded'].sum()
+        total_count = len(valid_results)
+        st.metric("Success Rate", f"{success_count}/{total_count} ({success_count/total_count:.1%} of valid results)")
+    
+    with col2:
+        avg_latency = detailed_results['time_round_trip_ms'].mean()
+        st.metric("Avg Latency", f"{avg_latency:.0f}ms")
+    
+    with col3:
+        total_cost = detailed_results['cost_usd'].sum()
+        st.metric("Total Cost", f"${total_cost:.4f}")
+    
+    # Interactive results table
+    st.markdown("### 📋 Individual Results")
+    
+    # Add result selector with indicators for valid/invalid attempts
+    result_options = []
+    for idx, row in detailed_results.iterrows():
+        # Check if this is a valid result
+        is_valid = (row['error_enum'] not in [1, 6, 7]) if not pd.isna(row['error_enum']) else True
+        
+        # Create status indicator
+        if is_valid:
+            status = "✅" if row['succeeded'] else "❌"
+        else:
+            status = "⚠️"  # Warning symbol for invalid results
+            
+        # Add validity indicator to the option text
+        validity_text = "" if is_valid else " [INVALID RESULT]"
+        result_options.append(f"{status} {row['task_id']} - {row['time_round_trip_ms']:.0f}ms{validity_text}")
+    
+    selected_result_idx = st.selectbox(
+        "Select a result to analyze:",
+        range(len(result_options)),
+        format_func=lambda x: result_options[x]
+    )
+    
+    if selected_result_idx is not None:
+        render_result_detail(detailed_results.iloc[selected_result_idx])
+
+def render_result_detail(result):
+    """Render detailed view of a single result"""
+    st.markdown("### 🔬 Result Deep Dive")
+    
+    # Check if this is a valid result
+    is_valid = (result['error_enum'] not in [1, 6, 7]) if not pd.isna(result['error_enum']) else True
+    
+    # Show validity warning if needed
+    if not is_valid:
+        st.warning("⚠️ **This is an invalid result** - The model didn't properly call the diff edit tool or edited the wrong file. This result is excluded from success rate calculations.")
+    
+    # Result metadata
+    col1, col2, col3, col4 = st.columns(4)
+    
+    with col1:
+        status_icon = "✅" if result['succeeded'] else "❌"
+        st.markdown(f"**Status:** {status_icon} {'Success' if result['succeeded'] else 'Failed'}")
+    
+    with col2:
+        st.markdown(f"**Task ID:** {result['task_id']}")
+    
+    with col3:
+        st.markdown(f"**Round Trip:** {result['time_round_trip_ms']:.0f}ms")
+    
+    with col4:
+        st.markdown(f"**Cost:** ${result['cost_usd']:.4f}")
+    
+    # Tabbed interface for different views
+    tab1, tab2, tab3, tab4 = st.tabs(["📄 File & Edits", "🤖 Raw Output", "🔧 Parsed Tool Call", "📊 Metrics"])
+    
+    with tab1:
+        render_file_and_edits_view(result)
+    
+    with tab2:
+        render_raw_output_view(result)
+    
+    with tab3:
+        render_parsed_tool_call_view(result)
+    
+    with tab4:
+        render_metrics_view(result)
+
+def render_file_and_edits_view(result):
+    """Render side-by-side file and edits view"""
+    st.markdown("#### 📄 File Content & Edit Analysis")
+    
+    # Check if we have original file content
+    has_original = not pd.isna(result['original_file_content']) and result['original_file_content']
+    has_edited = not pd.isna(result['edited_file_content']) and result['edited_file_content']
+    
+    if not has_original and not has_edited:
+        st.warning("No file content available for this result.")
+        return
+    
+    col1, col2 = st.columns(2)
+    
+    with col1:
+        st.markdown("**Original File:**")
+        if has_original:
+            filepath = result['original_filepath'] if not pd.isna(result['original_filepath']) else 'Unknown file'
+            st.markdown(f"📁 `{filepath}`")
+            
+            # Display full original file content in a scrollable code block
+            with st.expander("View Original File Content", expanded=True):
+                # Prepare content for the copy button (needs JS-specific escaping)
+                raw_content_for_copy = result['original_file_content']
+                # Escape for JavaScript template literal: backticks, backslashes, newlines
+                js_escaped_content = raw_content_for_copy.replace('\\', '\\\\') \
+                                                       .replace('`', '\\`') \
+                                                       .replace('\r\n', '\\n') \
+                                                       .replace('\n', '\\n') \
+                                                       .replace('\r', '\\n')
+
+                unique_suffix = str(result.name if hasattr(result, 'name') else result['task_id']).replace('-', '_').replace('.', '_')
+                button_id = f"copyBtnOriginal_{unique_suffix}"
+                
+                copy_button_html = f"""
+                    <button id="{button_id}" onclick="copyOriginalToClipboard(`{js_escaped_content}`, '{button_id}')" style="margin-bottom: 10px; padding: 5px 10px; border-radius: 5px; border: 1px solid #ccc; cursor: pointer;">Copy Original File</button>
+                    <script>
+                        if (!window.copyOriginalToClipboard) {{
+                            window.copyOriginalToClipboard = async function(text, buttonId) {{
+                                try {{
+                                    await navigator.clipboard.writeText(text);
+                                    const button = document.getElementById(buttonId);
+                                    button.innerText = 'Copied!';
+                                    button.style.backgroundColor = '#d4edda'; // Optional: success feedback
+                                    setTimeout(() => {{ 
+                                        button.innerText = 'Copy Original File'; 
+                                        button.style.backgroundColor = '';
+                                    }}, 2000);
+                                }} catch (err) {{
+                                    console.error('Failed to copy original: ', err);
+                                    const button = document.getElementById(buttonId);
+                                    button.innerText = 'Copy Failed!';
+                                    button.style.backgroundColor = '#f8d7da'; // Optional: error feedback
+                                    setTimeout(() => {{ 
+                                        button.innerText = 'Copy Original File'; 
+                                        button.style.backgroundColor = '';
+                                    }}, 2000);
+                                }}
+                            }}
+                        }}
+                    </script>
+                    """
+                st.components.v1.html(copy_button_html, height=50)
+
+                # Prepare content for st.code (needs actual newlines)
+                content_for_display = result['original_file_content']
+                # Iteratively replace common escaped newline sequences with actual newlines
+                # This handles cases like "\\n" -> "\n" and then "\n" (if it was literally "\n")
+                # Order might matter if there are multiple levels of escaping, but this covers common ones.
+                content_for_display = content_for_display.replace('\\\\r\\\\n', '\r\n').replace('\\\\n', '\n') # Double escaped
+                content_for_display = content_for_display.replace('\\r\\n', '\r\n').replace('\\n', '\n')     # Single escaped
+
+                language = guess_language_from_filepath(filepath)
+                st.code(content_for_display, language=language, line_numbers=False)
+
+        else:
+            st.warning("Original file content not available")
+    
+    with col2:
+        st.markdown("**Edit Analysis:**")
+        
+        if not result['succeeded']:
+            # Show error information
+            st.error("❌ **Edit Failed**")
+            
+            if not pd.isna(result['error_enum']):
+                st.markdown(f"**Error Code:** {result['error_enum']}")
+        else:
+            # Show successful edit information
+            st.success("✅ **Edit Successful**")
+            
+            # Show edit metrics
+            metric_col1, metric_col2, metric_col3 = st.columns(3)
+            
+            with metric_col1:
+                if not pd.isna(result['num_edits']):
+                    st.metric("Edits", int(result['num_edits']))
+            
+            with metric_col2:
+                if not pd.isna(result['num_lines_added']):
+                    st.metric("Added", int(result['num_lines_added']))
+            
+            with metric_col3:
+                if not pd.isna(result['num_lines_deleted']):
+                    st.metric("Deleted", int(result['num_lines_deleted']))
+            
+            # Show edited file if available
+            if has_edited:
+                st.markdown("**Edited File:**")
+                with st.expander("View Edited File Content"):
+                    edited_lines = result['edited_file_content'].split('\n')
+                    for i, line in enumerate(edited_lines[:50], 1):
+                        st.text(f"{i:3d} | {line}")
+                    
+                    if len(edited_lines) > 50:
+                        st.text(f"... ({len(edited_lines) - 50} more lines)")
+        
+        # Show parsed tool call if available
+        if not pd.isna(result['parsed_tool_call_json']):
+            with st.expander("View Parsed Tool Call"):
+                try:
+                    parsed_call = json.loads(result['parsed_tool_call_json'])
+                    st.json(parsed_call)
+                except:
+                    st.text(result['parsed_tool_call_json'])
+
+def render_raw_output_view(result):
+    """Render raw model output"""
+    st.markdown("#### 🤖 Raw Model Output")
+    
+    if pd.isna(result['raw_model_output']) or not result['raw_model_output']:
+        st.warning("No raw output available for this result.")
+        return
+    
+    st.markdown("""
+    <div class="file-viewer">
+    """, unsafe_allow_html=True)
+    
+    st.text(result['raw_model_output'])
+    
+    st.markdown("</div>", unsafe_allow_html=True)
+
+def render_parsed_tool_call_view(result):
+    """Render parsed tool call analysis"""
+    st.markdown("#### 🔧 Parsed Tool Call Analysis")
+    
+    if pd.isna(result['parsed_tool_call_json']) or not result['parsed_tool_call_json']:
+        st.warning("No parsed tool call available for this result.")
+        return
+    
+    try:
+        parsed_call = json.loads(result['parsed_tool_call_json'])
+        
+        # Pretty print the JSON
+        st.json(parsed_call)
+        
+        # If it's a replace_in_file call, show the diff blocks
+        if isinstance(parsed_call, dict) and 'diff' in parsed_call:
+            st.markdown("**Diff Blocks:**")
+            st.code(parsed_call['diff'], language='diff')
+            
+    except json.JSONDecodeError:
+        st.markdown("**Raw Parsed Call (Invalid JSON):**")
+        st.text(result['parsed_tool_call_json'])
+
+def render_metrics_view(result):
+    """Render detailed metrics for the result"""
+    st.markdown("#### 📊 Detailed Metrics")
+    
+    col1, col2 = st.columns(2)
+    
+    with col1:
+        st.markdown("**Timing Metrics:**")
+        if not pd.isna(result['time_to_first_token_ms']):
+            st.metric("Time to First Token", f"{result['time_to_first_token_ms']:.0f}ms")
+        
+        if not pd.isna(result['time_to_first_edit_ms']):
+            st.metric("Time to First Edit", f"{result['time_to_first_edit_ms']:.0f}ms")
+        
+        if not pd.isna(result['time_round_trip_ms']):
+            st.metric("Round Trip Time", f"{result['time_round_trip_ms']:.0f}ms")
+    
+    with col2:
+        st.markdown("**Token & Cost Metrics:**")
+        if not pd.isna(result['completion_tokens']):
+            st.metric("Completion Tokens", int(result['completion_tokens']))
+        
+        if not pd.isna(result['cost_usd']):
+            st.metric("Cost", f"${result['cost_usd']:.4f}")
+        
+        if not pd.isna(result['tokens_in_context']):
+            st.metric("Context Tokens", int(result['tokens_in_context']))
+
+def guess_language_from_filepath(filepath):
+    """Guess the language for syntax highlighting from filepath."""
+    if not filepath or pd.isna(filepath):
+        return None
+    
+    extension_map = {
+        '.py': 'python',
+        '.js': 'javascript',
+        '.ts': 'typescript',
+        '.java': 'java',
+        '.cs': 'csharp',
+        '.cpp': 'cpp',
+        '.c': 'c',
+        '.html': 'html',
+        '.css': 'css',
+        '.json': 'json',
+        '.sql': 'sql',
+        '.md': 'markdown',
+        '.rb': 'ruby',
+        '.php': 'php',
+        '.go': 'go',
+        '.rs': 'rust',
+        '.swift': 'swift',
+        '.kt': 'kotlin',
+        '.sh': 'bash',
+        '.yaml': 'yaml',
+        '.yml': 'yaml',
+        '.xml': 'xml',
+    }
+    
+    _, ext = os.path.splitext(filepath)
+def main():
+    # Add a note about valid attempts
+    st.sidebar.markdown("""
+    ### Note on Metrics
+    Success rates are calculated based on **valid results only**. 
+    
+    Invalid results (where the model didn't call the diff edit tool or edited the wrong file) are excluded from calculations.
+    """)
+    
+    # Initialize session state
+    if 'drill_down_model' not in st.session_state:
+        st.session_state.drill_down_model = None
+    if 'selected_run_id' not in st.session_state:
+        st.session_state.selected_run_id = None
+    
+    # Load all runs for sidebar
+    all_runs = load_all_runs()
+    
+    if all_runs.empty:
+        st.error("No evaluation runs found in the database.")
+        st.stop()
+    
+    # Sidebar for run selection
+    with st.sidebar:
+        st.markdown("## 📊 Evaluation Runs")
+        st.markdown("Select a run to analyze:")
+        
+        # Create run options with nice formatting
+        run_options = []
+        run_ids = []
+        
+        for idx, run in all_runs.iterrows():
+            # Format the run description nicely
+            date_str = run['created_at'][:10]  # Get just the date part
+            time_str = run['created_at'][11:16]  # Get just the time part
+            
+            if run['description']:
+                display_name = f"🚀 {run['description']}"
+            else:
+                display_name = f"📅 Run {run['run_id'][:8]}..."
+            
+            run_options.append(f"{display_name}\n📅 {date_str} {time_str}")
+            run_ids.append(run['run_id'])
+        
+        # Default to latest run if no selection
+        if st.session_state.selected_run_id is None:
+            default_index = 0  # Latest run is first
+            st.session_state.selected_run_id = run_ids[0]
+        else:
+            try:
+                default_index = run_ids.index(st.session_state.selected_run_id)
+            except ValueError:
+                default_index = 0
+                st.session_state.selected_run_id = run_ids[0]
+        
+        selected_run_idx = st.selectbox(
+            "Choose run:",
+            range(len(run_options)),
+            format_func=lambda x: run_options[x],
+            index=default_index,
+            key="run_selector"
+        )
+        
+        # Update selected run if changed
+        if run_ids[selected_run_idx] != st.session_state.selected_run_id:
+            st.session_state.selected_run_id = run_ids[selected_run_idx]
+            st.session_state.drill_down_model = None  # Reset drill down when changing runs
+            st.rerun()
+        
+        # Show run details in sidebar
+        selected_run = all_runs.iloc[selected_run_idx]
+        st.markdown("---")
+        st.markdown("### 📋 Run Details")
+        st.markdown(f"**Run ID:** `{selected_run['run_id'][:12]}...`")
+        st.markdown(f"**Created:** {selected_run['created_at']}")
+        if selected_run['description']:
+            st.markdown(f"**Description:** {selected_run['description']}")
+    
+    # Load data for selected run
+    current_run, model_performance = load_run_comparison(st.session_state.selected_run_id)
+    
+    if current_run is None or model_performance.empty:
+        st.error("No data found for the selected run.")
+        st.stop()
+    
+    # Render main dashboard
+    render_hero_section(current_run, model_performance)
+    
+    # Check if we're in drill-down mode
+    if st.session_state.drill_down_model:
+        col1, col2 = st.columns([1, 4])
+        with col1:
+            if st.button("Back to Overview", use_container_width=True):
+                st.session_state.drill_down_model = None
+                st.rerun()
+        
+        render_detailed_analysis(current_run['run_id'], st.session_state.drill_down_model)
+    else:
+        # Success Rate Comparison
+        fig_success = px.bar(
+            model_performance,
+            x='model_id',
+            y='success_rate',
+            title="Success Rate by Model",
+            labels={'success_rate': 'Success Rate', 'model_id': 'Model'},
+            color='success_rate',
+            color_continuous_scale='RdYlGn',
+            text='success_rate',
+            template='plotly_dark'
+        )
+        fig_success.update_traces(texttemplate='%{text:.1%}', textposition='outside')
+        fig_success.update_layout(
+            showlegend=False,
+            plot_bgcolor='rgba(0,0,0,0)',
+            paper_bgcolor='rgba(0,0,0,0)',
+            font=dict(family="Azeret Mono, monospace"),
+            yaxis_range=[0,1],  # Set y-axis from 0% to 100%
+            margin=dict(t=50)  # Add top margin to prevent clipping
+        )
+        st.plotly_chart(fig_success, use_container_width=True)
+        
+        render_model_comparison_cards(model_performance)
+        render_comparison_charts(model_performance)
+
+if __name__ == "__main__":
+    main()

evals/diff-edits/dashboard/launch.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# Diff Edits Evaluation Dashboard Launcher
+echo "🚀 Starting Diff Edits Evaluation Dashboard..."
+
+# Check if we're in the right directory
+if [ ! -f "app.py" ]; then
+    echo "❌ Error: app.py not found. Please run this script from the dashboard directory."
+    exit 1
+fi
+
+# Check if database exists
+if [ ! -f "../evals.db" ]; then
+    echo "⚠️  Warning: Database file ../evals.db not found."
+    echo "   Make sure you've run some evaluations first to populate the database."
+    echo "   You can run: node ../cli/dist/index.js run-diff-eval --model-id anthropic/claude-sonnet-4 --max-cases 1"
+    echo ""
+fi
+
+# Check if requirements are installed
+echo "📦 Checking Python dependencies..."
+if ! python -c "import streamlit, plotly, pandas" 2>/dev/null; then
+    echo "📥 Installing required packages..."
+    pip install -r requirements.txt
+fi
+
+echo "🌐 Launching Streamlit dashboard..."
+echo "   Dashboard will open in your browser at http://localhost:8501"
+echo "   Press Ctrl+C to stop the dashboard"
+echo ""
+
+# Launch Streamlit
+streamlit run app.py

evals/diff-edits/dashboard/pages/02_Bad_Cases.py

@@ -0,0 +1,183 @@
+import streamlit as st
+import pandas as pd
+import json
+import os # Need to import os for load_case_raw_data
+from utils import get_database_connection, guess_language_from_filepath # Absolute import
+
+st.set_page_config(
+    page_title="Case Health Inspector",
+    page_icon="🧑‍⚕️",
+    layout="wide"
+)
+
+st.title("Case Health Inspector")
+st.markdown("Identify test cases that are frequently problematic across different models and runs.")
+
+@st.cache_data
+def load_problematic_cases_summary():
+    conn = get_database_connection()
+    query = """
+    WITH case_attempts AS (
+        SELECT
+            c.task_id,
+            c.description AS case_description,
+            f_orig.filepath AS original_filepath, -- Get from files table
+            r.run_id,
+            r.model_id,
+            r.result_id,
+            (CASE WHEN (r.error_enum NOT IN (1, 6, 7) OR r.error_enum IS NULL) THEN 1 ELSE 0 END) AS is_valid_attempt,
+            (CASE WHEN (r.error_enum NOT IN (1, 6, 7) OR r.error_enum IS NULL) THEN r.succeeded ELSE NULL END) AS succeeded_on_valid
+        FROM cases c
+        JOIN results r ON c.case_id = r.case_id
+        LEFT JOIN files f_orig ON c.file_hash = f_orig.hash -- Join to get original filepath
+    ),
+    case_summary AS (
+        SELECT
+            task_id,
+            case_description,
+            original_filepath, -- This is now f_orig.filepath
+            COUNT(DISTINCT run_id) AS num_benchmark_runs,
+            COUNT(result_id) AS total_attempts,
+            SUM(is_valid_attempt) AS total_valid_attempts,
+            SUM(succeeded_on_valid) AS total_successful_valid_attempts
+        FROM case_attempts
+        GROUP BY task_id, case_description, original_filepath -- original_filepath is f_orig.filepath
+    )
+    SELECT
+        task_id,
+        case_description,
+        original_filepath, -- This is f_orig.filepath from case_summary
+        num_benchmark_runs,
+        total_attempts,
+        total_valid_attempts,
+        CAST(total_valid_attempts AS REAL) * 100.0 / total_attempts AS percent_valid_attempts,
+        CASE 
+            WHEN total_valid_attempts > 0 THEN CAST(total_successful_valid_attempts AS REAL) * 100.0 / total_valid_attempts
+            ELSE 0 
+        END AS success_rate_on_valid
+    FROM case_summary
+    ORDER BY percent_valid_attempts ASC, success_rate_on_valid ASC;
+    """
+    df = pd.read_sql_query(query, conn)
+    return df
+
+@st.cache_data
+def load_case_raw_data(task_id):
+    """Loads the original JSON data for a given task_id."""
+    # This assumes test cases are stored in ../cases relative to this script's parent (dashboard)
+    # So, ../../cases from this script's location (pages/02_Bad_Cases.py)
+    # Correct path from this script (pages/02_Bad_Cases.py) to cases/
+    # os.path.dirname(__file__) -> pages
+    # os.path.join(..., '..') -> dashboard
+    # os.path.join(..., '..', '..') -> diff-edits
+    # os.path.join(..., '..', '..', 'cases') -> diff-edits/cases
+    cases_dir = os.path.join(os.path.dirname(__file__), '..', '..', 'cases')
+    
+    # The task_id is usually the filename without .json
+    # However, some task_ids might have suffixes or be different.
+    # We need a robust way to find the file. For now, assume task_id is filename base.
+    # This might need adjustment if task_id format varies significantly from filename.
+    
+    # Try direct match first
+    potential_filename = f"{task_id}.json"
+    filepath = os.path.join(cases_dir, potential_filename)
+
+    if not os.path.exists(filepath):
+        # If direct match fails, list files and try to find one that starts with task_id
+        # This is a simple fallback, might need more robust matching if task_ids are complex
+        try:
+            for f_name in os.listdir(cases_dir):
+                if f_name.startswith(task_id) and f_name.endswith(".json"):
+                    filepath = os.path.join(cases_dir, f_name)
+                    break
+            else: # No break means no file found
+                 return None # File not found
+        except FileNotFoundError:
+            return None # Cases directory itself not found
+
+    if not os.path.exists(filepath): # Check again after potential find
+        return None
+
+    try:
+        with open(filepath, 'r') as f:
+            return json.load(f)
+    except Exception as e:
+        st.error(f"Error loading case file {filepath}: {e}")
+        return None
+
+def render_problematic_cases_page():
+    summary_df = load_problematic_cases_summary()
+
+    if summary_df.empty:
+        st.warning("No case summary data found. Run some evaluations first.")
+        return
+
+    st.markdown("### Cases Overview")
+    st.dataframe(summary_df.style.format({
+        "percent_valid_attempts": "{:.1f}%",
+        "success_rate_on_valid": "{:.1f}%"
+    }), use_container_width=True)
+
+    st.markdown("---")
+    st.markdown("### Case Drill Down")
+    
+    selected_task_id = st.selectbox(
+        "Select a Case ID (task_id) to inspect:",
+        options=[""] + summary_df['task_id'].tolist() # Add a blank option
+    )
+
+    if selected_task_id:
+        case_data = summary_df[summary_df['task_id'] == selected_task_id].iloc[0]
+        st.subheader(f"Details for Case: {case_data['task_id']}")
+        st.markdown(f"**Description:** {case_data['case_description']}")
+        st.markdown(f"**Original Filepath:** `{case_data['original_filepath']}`")
+        
+        raw_json_data = load_case_raw_data(selected_task_id)
+        if raw_json_data:
+            with st.expander("View Raw Case JSON Data", expanded=False):
+                st.json(raw_json_data)
+            
+            if 'file_contents' in raw_json_data and raw_json_data['file_contents']:
+                with st.expander("View Original File Content (from Case JSON)", expanded=True):
+                    # Prepare content for the copy button
+                    raw_content_for_copy = raw_json_data['file_contents']
+                    js_escaped_content = raw_content_for_copy.replace('\\', '\\\\') \
+                                                           .replace('`', '\\`') \
+                                                           .replace('\r\n', '\\n') \
+                                                           .replace('\n', '\\n') \
+                                                           .replace('\r', '\\n')
+                    button_id = f"copyBtnCase_{selected_task_id.replace('-', '_').replace('.', '_')}"
+                    copy_button_html = f"""
+                        <button id="{button_id}" onclick="copyCaseContentToClipboard(`{js_escaped_content}`, '{button_id}')" style="margin-bottom: 10px; padding: 5px 10px; border-radius: 5px; border: 1px solid #ccc; cursor: pointer;">Copy File Content</button>
+                        <script>
+                            if (!window.copyCaseContentToClipboard) {{
+                                window.copyCaseContentToClipboard = async function(text, buttonId) {{
+                                    try {{
+                                        await navigator.clipboard.writeText(text);
+                                        const button = document.getElementById(buttonId);
+                                        button.innerText = 'Copied!';
+                                        setTimeout(() => {{ button.innerText = 'Copy File Content'; }}, 2000);
+                                    }} catch (err) {{ console.error('Failed to copy: ', err); const button = document.getElementById(buttonId); button.innerText = 'Copy Failed!'; setTimeout(() => {{ button.innerText = 'Copy File Content'; }}, 2000); }}
+                                }}
+                            }}
+                        </script>
+                        """
+                    st.components.v1.html(copy_button_html, height=50)
+
+                    # Prepare content for st.code
+                    content_for_display = raw_json_data['file_contents']
+                    content_for_display = content_for_display.replace('\\\\r\\\\n', '\r\n').replace('\\\\n', '\n')
+                    content_for_display = content_for_display.replace('\\r\\n', '\r\n').replace('\\n', '\n')
+                    
+                    language = guess_language_from_filepath(case_data['original_filepath'])
+                    st.code(content_for_display, language=language, line_numbers=False)
+            else:
+                st.warning("Original file content not found in case JSON.")
+        else:
+            st.error(f"Could not load raw JSON data for case: {selected_task_id}")
+        
+        # Placeholder for more detailed stats (per-model performance on this case, error breakdown)
+        st.markdown("*(Further per-model statistics and error breakdowns for this case can be added here.)*")
+
+if __name__ == "__main__":
+    render_problematic_cases_page()

evals/diff-edits/dashboard/requirements.txt

@@ -0,0 +1,4 @@
+streamlit>=1.28.0
+plotly>=5.17.0
+pandas>=2.0.0
+numpy>=1.24.0

evals/diff-edits/dashboard/utils.py

@@ -0,0 +1,51 @@
+import streamlit as st
+import sqlite3
+import pandas as pd
+import os
+
+@st.cache_resource
+def get_database_connection():
+    # Assuming the script is run from the dashboard directory,
+    # evals.db is two levels up from there.
+    # __file__ is utils.py, its dirname is dashboard.
+    # os.path.dirname(__file__) -> dashboard/
+    # os.path.join(..., '..') -> diff-edits/
+    # os.path.join(..., '..', 'evals.db') -> diff-edits/evals.db
+    db_path = os.path.join(os.path.dirname(__file__), '..', 'evals.db')
+    if not os.path.exists(db_path):
+        st.error(f"Database not found. Expected at: {os.path.abspath(db_path)}")
+        st.stop()
+    return sqlite3.connect(db_path, check_same_thread=False)
+
+def guess_language_from_filepath(filepath):
+    """Guess the language for syntax highlighting from filepath."""
+    if not filepath or pd.isna(filepath):
+        return None
+    
+    extension_map = {
+        '.py': 'python',
+        '.js': 'javascript',
+        '.ts': 'typescript',
+        '.java': 'java',
+        '.cs': 'csharp',
+        '.cpp': 'cpp',
+        '.c': 'c',
+        '.html': 'html',
+        '.css': 'css',
+        '.json': 'json',
+        '.sql': 'sql',
+        '.md': 'markdown',
+        '.rb': 'ruby',
+        '.php': 'php',
+        '.go': 'go',
+        '.rs': 'rust',
+        '.swift': 'swift',
+        '.kt': 'kotlin',
+        '.sh': 'bash',
+        '.yaml': 'yaml',
+        '.yml': 'yaml',
+        '.xml': 'xml',
+    }
+    
+    _, ext = os.path.splitext(str(filepath)) # Ensure filepath is string
+    return extension_map.get(ext.lower(), None)

evals/diff-edits/database.md

@@ -0,0 +1,84 @@
+# Diff Edit Evaluation Database Schema
+
+This document provides an overview of the SQLite database schema used for the diff edit evaluation suite. The database is designed to capture every aspect of the evaluation runs in a structured way, allowing for detailed, multi-dimensional analysis and ensuring full reproducibility of our findings.
+
+## Data Model Overview
+
+The database is composed of several interconnected tables that work together to provide a comprehensive picture of each evaluation. The core of the model revolves around `runs`, `cases`, and `results`.
+
+### `runs`
+
+A `run` represents a single, top-level execution of the evaluation script (e.g., one invocation of `npm run diff-eval`). It serves as the main container for a complete benchmark session.
+
+-   **Purpose**: To group all the results from a single benchmark execution, allowing for high-level comparison between different runs over time.
+-   **Key Columns**:
+    -   `run_id`: A unique identifier for the entire run.
+    -   `description`: A human-readable summary of the run's configuration (e.g., which models were tested, how many cases, etc.).
+    -   `system_prompt_hash`: A foreign key that links this run to the specific system prompt that was used, ensuring we can track performance changes based on prompt modifications.
+
+### `cases`
+
+A `case` represents a single test scenario that is presented to a model. It corresponds to one of the JSON files in the `cases/` directory and links that static definition to a specific benchmark `run`.
+
+-   **Purpose**: To track the individual test scenarios within a given run.
+-   **Key Columns**:
+    -   `case_id`: A unique identifier for the case *within* a specific run.
+    -   `run_id`: A foreign key linking back to the parent `run`.
+    -   `task_id`: The original, persistent identifier for the test case (typically from the JSON filename).
+    -   `file_hash`: A foreign key linking to the original, un-edited file content for this case.
+
+### `results`
+
+This is the most granular and important table in the database. A `result` represents the outcome of a single attempt by a specific model on a specific case.
+
+-   **Purpose**: To store the detailed outcome of every single model attempt, providing the raw data for all quantitative and qualitative analysis.
+-   **Key Columns**:
+    -   `result_id`: A unique identifier for the individual attempt.
+    -   `run_id`, `case_id`, `model_id`, `processing_functions_hash`: A set of foreign keys that precisely situate this result within the context of a specific run, case, model, and set of helper functions.
+    -   `succeeded`: A boolean indicating if the generated diff was applied successfully.
+    -   `error_enum`: A numeric code representing the specific type of error if the attempt failed (e.g., `1` for `no_tool_calls`, `7` for `wrong_file_edited`).
+    -   `num_edits`, `num_lines_deleted`, `num_lines_added`: Quantitative metrics about the structure of the generated diff.
+    -   `time_to_first_token_ms`, `time_to_first_edit_ms`, `time_round_trip_ms`: High-precision timing data to measure model latency.
+    -   `cost_usd`, `completion_tokens`: Cost and token usage metrics for efficiency analysis.
+    -   `raw_model_output`, `file_edited_hash`, `parsed_tool_call_json`: The rich, qualitative data. This includes the model's full, raw response and the parsed tool calls, which are invaluable for debugging and understanding the model's reasoning.
+
+---
+
+## Supporting Tables
+
+The following tables store versioned, deduplicated content to ensure data integrity and efficiency.
+
+### `system_prompts`
+
+-   **Purpose**: Stores the versioned content of the system prompts used in evaluations.
+-   **Key Columns**:
+    -   `hash`: A unique hash of the prompt's content, which acts as the primary key. This prevents duplicate storage of the same prompt.
+    -   `name`: A human-readable name for the prompt (e.g., `basicSystemPrompt`, `claude4SystemPrompt`).
+    -   `content`: The full text of the system prompt.
+
+### `processing_functions`
+
+-   **Purpose**: Stores the versioned combinations of parsing and diff-editing functions.
+-   **Key Columns**:
+    -   `hash`: A unique hash of the function combination name.
+    -   `name`: A human-readable name (e.g., `parseV2-diffV2`).
+    -   `parsing_function`: The name of the function used to parse the model's output.
+    -   `diff_edit_function`: The name of the function used to apply the diff.
+
+### `files`
+
+-   **Purpose**: Stores the content of all files involved in the tests, including the original source files and the diffs generated by the models.
+-   **Key Columns**:
+    -   `hash`: A content-based hash of the file, ensuring that identical files are only stored once.
+    -   `filepath`: The original path of the file.
+    -   `content`: The full content of the file.
+
+## The Bigger Picture
+
+This relational schema provides a powerful foundation for sophisticated analysis. It moves beyond simple pass/fail metrics and allows us to explore the nuanced interactions between models, prompts, and the code they operate on. With this database, we can answer critical questions like:
+
+-   "How does prompt engineering affect not just success rate, but also latency and cost?"
+-   "Are certain models more prone to specific types of errors (e.g., hallucinating file paths vs. failing to call a tool)?"
+-   "Which of our internal diffing algorithms is the most robust against a wide range of model-generated edits?"
+
+Ultimately, this data model enables us to move from simply *measuring* performance to truly *understanding* it, providing the insights needed to build more capable and reliable AI engineering systems.

evals/diff-edits/database/client.ts

@@ -0,0 +1,135 @@
+import Database from 'better-sqlite3';
+import * as fs from 'fs';
+import * as path from 'path';
+import * as crypto from 'crypto';
+
+export class DatabaseClient {
+  private static instance: DatabaseClient;
+  private db: Database.Database;
+  private dbPath: string;
+
+  private constructor() {
+    // Get database path from environment or use default
+    this.dbPath = process.env.DIFF_EVALS_DB_PATH || path.join(__dirname, '../evals.db');
+    
+    // Ensure directory exists
+    const dbDir = path.dirname(this.dbPath);
+    if (!fs.existsSync(dbDir)) {
+      fs.mkdirSync(dbDir, { recursive: true });
+    }
+
+    // Initialize database connection
+    this.db = new Database(this.dbPath);
+    
+    // Enable WAL mode for concurrent access
+    this.db.pragma('journal_mode = WAL');
+    
+    // Enable foreign key constraints
+    this.db.pragma('foreign_keys = ON');
+    
+    // Initialize schema if needed
+    this.initializeSchema();
+  }
+
+  static getInstance(): DatabaseClient {
+    if (!DatabaseClient.instance) {
+      DatabaseClient.instance = new DatabaseClient();
+    }
+    return DatabaseClient.instance;
+  }
+
+  private initializeSchema(): void {
+    // Check if tables exist by trying to query one of them
+    try {
+      this.db.prepare('SELECT COUNT(*) FROM system_prompts LIMIT 1').get();
+      // If we get here, tables exist
+      return;
+    } catch (error) {
+      // Tables don't exist, create them
+      console.log('Initializing database schema...');
+      this.createTables();
+    }
+  }
+
+  private createTables(): void {
+    const schemaPath = path.join(__dirname, 'schema.sql');
+    const schema = fs.readFileSync(schemaPath, 'utf8');
+    
+    // Execute the entire schema as one block
+    this.db.transaction(() => {
+      this.db.exec(schema);
+    })();
+
+    console.log('Database schema initialized successfully');
+  }
+
+  getDatabase(): Database.Database {
+    return this.db;
+  }
+
+  getDatabasePath(): string {
+    return this.dbPath;
+  }
+
+  // Utility method to generate SHA-256 hash
+  static generateHash(content: string): string {
+    return crypto.createHash('sha256').update(content).digest('hex');
+  }
+
+  // Utility method to generate UUID-like ID
+  static generateId(): string {
+    return crypto.randomUUID();
+  }
+
+  // Transaction wrapper
+  transaction<T>(fn: () => T): T {
+    return this.db.transaction(fn)();
+  }
+
+  // Close database connection (for cleanup)
+  close(): void {
+    if (this.db) {
+      this.db.close();
+    }
+  }
+
+  // Get database info
+  getInfo(): { path: string; size: number; tables: string[] } {
+    const stats = fs.statSync(this.dbPath);
+    const tables = this.db
+      .prepare("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name")
+      .all()
+      .map((row: any) => row.name);
+
+    return {
+      path: this.dbPath,
+      size: stats.size,
+      tables
+    };
+  }
+
+  // Vacuum database (cleanup and optimize)
+  vacuum(): void {
+    this.db.exec('VACUUM');
+  }
+
+  // Get database statistics
+  getStats(): { [tableName: string]: number } {
+    const tables = ['system_prompts', 'processing_functions', 'files', 'runs', 'cases', 'results'];
+    const stats: { [tableName: string]: number } = {};
+
+    for (const table of tables) {
+      try {
+        const result = this.db.prepare(`SELECT COUNT(*) as count FROM ${table}`).get() as { count: number };
+        stats[table] = result.count;
+      } catch (error) {
+        stats[table] = 0;
+      }
+    }
+
+    return stats;
+  }
+}
+
+// Export singleton instance getter
+export const getDatabase = () => DatabaseClient.getInstance();

evals/diff-edits/database/index.ts

@@ -0,0 +1,23 @@
+// Main database module exports
+export { DatabaseClient, getDatabase } from './client';
+export * from './types';
+export * from './operations';
+export * from './queries';
+
+// Re-export commonly used functions for convenience
+export {
+  upsertSystemPrompt,
+  upsertProcessingFunctions,
+  upsertFile,
+  createBenchmarkRun,
+  createCase,
+  insertResult,
+  getRunStats
+} from './operations';
+
+export {
+  getSuccessRatesByModel,
+  getModelComparisons,
+  getDatabaseSummary,
+  getErrorDistribution
+} from './queries';

evals/diff-edits/database/operations.ts

@@ -0,0 +1,348 @@
+import { DatabaseClient } from './client';
+import {
+  SystemPrompt,
+  ProcessingFunctions,
+  FileRecord,
+  BenchmarkRun,
+  Case,
+  Result,
+  CreateSystemPromptInput,
+  CreateProcessingFunctionsInput,
+  CreateFileInput,
+  CreateBenchmarkRunInput,
+  CreateCaseInput,
+  CreateResultInput
+} from './types';
+
+const db = DatabaseClient.getInstance();
+
+// System Prompts Operations
+export async function upsertSystemPrompt(input: CreateSystemPromptInput): Promise<string> {
+  const hash = DatabaseClient.generateHash(input.content);
+  
+  const stmt = db.getDatabase().prepare(`
+    INSERT OR IGNORE INTO system_prompts (hash, name, content)
+    VALUES (?, ?, ?)
+  `);
+  
+  stmt.run(hash, input.name, input.content);
+  return hash;
+}
+
+export async function getSystemPromptByHash(hash: string): Promise<SystemPrompt | null> {
+  const stmt = db.getDatabase().prepare(`
+    SELECT * FROM system_prompts WHERE hash = ?
+  `);
+  
+  const result = stmt.get(hash) as SystemPrompt | undefined;
+  return result || null;
+}
+
+// Processing Functions Operations
+export async function upsertProcessingFunctions(input: CreateProcessingFunctionsInput): Promise<string> {
+  const hash = DatabaseClient.generateHash(input.parsing_function + input.diff_edit_function);
+  
+  const stmt = db.getDatabase().prepare(`
+    INSERT OR IGNORE INTO processing_functions (hash, name, parsing_function, diff_edit_function)
+    VALUES (?, ?, ?, ?)
+  `);
+  
+  stmt.run(hash, input.name, input.parsing_function, input.diff_edit_function);
+  return hash;
+}
+
+export async function getProcessingFunctionsByHash(hash: string): Promise<ProcessingFunctions | null> {
+  const stmt = db.getDatabase().prepare(`
+    SELECT * FROM processing_functions WHERE hash = ?
+  `);
+  
+  const result = stmt.get(hash) as ProcessingFunctions | undefined;
+  return result || null;
+}
+
+// Files Operations
+export async function upsertFile(input: CreateFileInput): Promise<string> {
+  const hash = DatabaseClient.generateHash(input.content);
+  
+  const stmt = db.getDatabase().prepare(`
+    INSERT OR IGNORE INTO files (hash, filepath, content, tokens)
+    VALUES (?, ?, ?, ?)
+  `);
+  
+  stmt.run(hash, input.filepath, input.content, input.tokens || null);
+  return hash;
+}
+
+export async function getFileByHash(hash: string): Promise<FileRecord | null> {
+  const stmt = db.getDatabase().prepare(`
+    SELECT * FROM files WHERE hash = ?
+  `);
+  
+  const result = stmt.get(hash) as FileRecord | undefined;
+  return result || null;
+}
+
+// Benchmark Runs Operations
+export async function createBenchmarkRun(input: CreateBenchmarkRunInput): Promise<string> {
+  const runId = DatabaseClient.generateId();
+  
+  const stmt = db.getDatabase().prepare(`
+    INSERT INTO runs (run_id, description, system_prompt_hash)
+    VALUES (?, ?, ?)
+  `);
+  
+  stmt.run(runId, input.description || null, input.system_prompt_hash);
+  return runId;
+}
+
+export async function getBenchmarkRun(runId: string): Promise<BenchmarkRun | null> {
+  const stmt = db.getDatabase().prepare(`
+    SELECT * FROM runs WHERE run_id = ?
+  `);
+  
+  const result = stmt.get(runId) as BenchmarkRun | undefined;
+  return result || null;
+}
+
+export async function getAllBenchmarkRuns(): Promise<BenchmarkRun[]> {
+  const stmt = db.getDatabase().prepare(`
+    SELECT * FROM runs ORDER BY created_at DESC
+  `);
+  
+  return stmt.all() as BenchmarkRun[];
+}
+
+// Cases Operations
+export async function createCase(input: CreateCaseInput): Promise<string> {
+  const caseId = DatabaseClient.generateId();
+  
+  const stmt = db.getDatabase().prepare(`
+    INSERT INTO cases (case_id, run_id, description, system_prompt_hash, task_id, tokens_in_context, file_hash)
+    VALUES (?, ?, ?, ?, ?, ?, ?)
+  `);
+  
+  stmt.run(
+    caseId,
+    input.run_id,
+    input.description,
+    input.system_prompt_hash,
+    input.task_id,
+    input.tokens_in_context,
+    input.file_hash || null
+  );
+  
+  return caseId;
+}
+
+export async function getCasesByRun(runId: string): Promise<Case[]> {
+  const stmt = db.getDatabase().prepare(`
+    SELECT * FROM cases WHERE run_id = ? ORDER BY created_at
+  `);
+  
+  return stmt.all(runId) as Case[];
+}
+
+export async function getCaseById(caseId: string): Promise<Case | null> {
+  const stmt = db.getDatabase().prepare(`
+    SELECT * FROM cases WHERE case_id = ?
+  `);
+  
+  const result = stmt.get(caseId) as Case | undefined;
+  return result || null;
+}
+
+// Results Operations
+export async function insertResult(input: CreateResultInput): Promise<string> {
+  const resultId = DatabaseClient.generateId();
+  
+  const stmt = db.getDatabase().prepare(`
+    INSERT INTO results (
+      result_id, run_id, case_id, model_id, processing_functions_hash,
+      succeeded, error_enum, num_edits, num_lines_deleted, num_lines_added,
+      time_to_first_token_ms, time_to_first_edit_ms, time_round_trip_ms,
+      cost_usd, completion_tokens, raw_model_output, file_edited_hash,
+      parsed_tool_call_json
+    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+  `);
+  
+  stmt.run(
+    resultId,
+    input.run_id,
+    input.case_id,
+    input.model_id,
+    input.processing_functions_hash,
+    input.succeeded ? 1 : 0, // Convert boolean to integer
+    input.error_enum || null,
+    input.num_edits || null,
+    input.num_lines_deleted || null,
+    input.num_lines_added || null,
+    input.time_to_first_token_ms || null,
+    input.time_to_first_edit_ms || null,
+    input.time_round_trip_ms || null,
+    input.cost_usd || null,
+    input.completion_tokens || null,
+    input.raw_model_output || null,
+    input.file_edited_hash || null,
+    input.parsed_tool_call_json || null
+  );
+  
+  return resultId;
+}
+
+export async function getResultsByRun(runId: string): Promise<Result[]> {
+  const stmt = db.getDatabase().prepare(`
+    SELECT * FROM results WHERE run_id = ? ORDER BY created_at
+  `);
+  
+  return stmt.all(runId) as Result[];
+}
+
+export async function getResultsByCase(caseId: string): Promise<Result[]> {
+  const stmt = db.getDatabase().prepare(`
+    SELECT * FROM results WHERE case_id = ? ORDER BY created_at
+  `);
+  
+  return stmt.all(caseId) as Result[];
+}
+
+export async function getResultById(resultId: string): Promise<Result | null> {
+  const stmt = db.getDatabase().prepare(`
+    SELECT * FROM results WHERE result_id = ?
+  `);
+  
+  const result = stmt.get(resultId) as Result | undefined;
+  return result || null;
+}
+
+// Batch operations for performance
+export async function insertResultsBatch(inputs: CreateResultInput[]): Promise<string[]> {
+  const stmt = db.getDatabase().prepare(`
+    INSERT INTO results (
+      result_id, run_id, case_id, model_id, processing_functions_hash,
+      succeeded, error_enum, num_edits, num_lines_deleted, num_lines_added,
+      time_to_first_token_ms, time_to_first_edit_ms, time_round_trip_ms,
+      cost_usd, completion_tokens, raw_model_output, file_edited_hash,
+      parsed_tool_call_json
+    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+  `);
+  
+  return db.transaction(() => {
+    const resultIds: string[] = [];
+    
+    for (const input of inputs) {
+      const resultId = DatabaseClient.generateId();
+      
+      stmt.run(
+        resultId,
+        input.run_id,
+        input.case_id,
+        input.model_id,
+        input.processing_functions_hash,
+        input.succeeded ? 1 : 0, // Convert boolean to integer
+        input.error_enum || null,
+        input.num_edits || null,
+        input.num_lines_deleted || null,
+        input.num_lines_added || null,
+        input.time_to_first_token_ms || null,
+        input.time_to_first_edit_ms || null,
+        input.time_round_trip_ms || null,
+        input.cost_usd || null,
+        input.completion_tokens || null,
+        input.raw_model_output || null,
+        input.file_edited_hash || null,
+        input.parsed_tool_call_json || null
+      );
+      
+      resultIds.push(resultId);
+    }
+    
+    return resultIds;
+  });
+}
+
+export async function createCasesBatch(inputs: CreateCaseInput[]): Promise<string[]> {
+  const stmt = db.getDatabase().prepare(`
+    INSERT INTO cases (case_id, run_id, description, system_prompt_hash, task_id, tokens_in_context)
+    VALUES (?, ?, ?, ?, ?, ?)
+  `);
+  
+  return db.transaction(() => {
+    const caseIds: string[] = [];
+    
+    for (const input of inputs) {
+      const caseId = DatabaseClient.generateId();
+      
+      stmt.run(
+        caseId,
+        input.run_id,
+        input.description,
+        input.system_prompt_hash,
+        input.task_id,
+        input.tokens_in_context
+      );
+      
+      caseIds.push(caseId);
+    }
+    
+    return caseIds;
+  });
+}
+
+// Utility functions
+export async function getRunStats(runId: string): Promise<{
+  total_cases: number;
+  total_results: number;
+  success_rate: number;
+  avg_cost: number;
+  avg_latency: number;
+}> {
+  const stmt = db.getDatabase().prepare(`
+    SELECT 
+      COUNT(DISTINCT c.case_id) as total_cases,
+      COUNT(r.result_id) as total_results,
+      AVG(CASE WHEN r.succeeded THEN 1.0 ELSE 0.0 END) as success_rate,
+      AVG(r.cost_usd) as avg_cost,
+      AVG(r.time_round_trip_ms) as avg_latency
+    FROM cases c
+    LEFT JOIN results r ON c.case_id = r.case_id
+    WHERE c.run_id = ?
+  `);
+  
+  const result = stmt.get(runId) as any;
+  return {
+    total_cases: result.total_cases || 0,
+    total_results: result.total_results || 0,
+    success_rate: result.success_rate || 0,
+    avg_cost: result.avg_cost || 0,
+    avg_latency: result.avg_latency || 0
+  };
+}
+
+// Count valid attempts for a specific case and model
+export async function getValidAttemptCount(caseId: string, modelId: string): Promise<number> {
+  const stmt = db.getDatabase().prepare(`
+    SELECT COUNT(*) as count
+    FROM results 
+    WHERE case_id = ? 
+      AND model_id = ?
+      AND error_enum NOT IN (1, 6, 7)  -- Exclude: no_tool_calls, wrong_tool_call, wrong_file_edited
+  `);
+  
+  const result = stmt.get(caseId, modelId) as { count: number };
+  return result.count;
+}
+
+// Get valid results for a specific case and model (for analysis)
+export async function getValidResults(caseId: string, modelId: string, limit?: number): Promise<Result[]> {
+  const limitClause = limit ? `LIMIT ${limit}` : '';
+  const stmt = db.getDatabase().prepare(`
+    SELECT * FROM results 
+    WHERE case_id = ? 
+      AND model_id = ?
+      AND error_enum NOT IN (1, 6, 7)  -- Only valid attempts
+    ORDER BY created_at
+    ${limitClause}
+  `);
+  
+  return stmt.all(caseId, modelId) as Result[];
+}

evals/diff-edits/database/queries.ts

@@ -0,0 +1,309 @@
+import { DatabaseClient } from './client';
+import {
+  ModelSuccessRate,
+  ModelLatency,
+  CostAnalysis,
+  ErrorDistribution,
+  FailedCase,
+  PerformanceTrend,
+  ModelComparison
+} from './types';
+
+const db = DatabaseClient.getInstance();
+
+// Performance analysis queries
+export async function getSuccessRatesByModel(): Promise<ModelSuccessRate[]> {
+  const stmt = db.getDatabase().prepare(`
+    SELECT 
+      model_id,
+      COUNT(*) as total_runs,
+      SUM(CASE WHEN succeeded THEN 1 ELSE 0 END) as successful_runs,
+      ROUND(AVG(CASE WHEN succeeded THEN 1.0 ELSE 0.0 END) * 100, 2) as success_rate
+    FROM results
+    WHERE error_enum NOT IN (1, 6, 7) OR error_enum IS NULL  -- Exclude: no_tool_calls, wrong_tool_call, wrong_file_edited
+    GROUP BY model_id
+    ORDER BY success_rate DESC, total_runs DESC
+  `);
+  
+  return stmt.all() as ModelSuccessRate[];
+}
+
+export async function getAverageLatencyByModel(): Promise<ModelLatency[]> {
+  const stmt = db.getDatabase().prepare(`
+    SELECT 
+      model_id,
+      ROUND(AVG(time_to_first_token_ms), 2) as avg_time_to_first_token_ms,
+      ROUND(AVG(time_to_first_edit_ms), 2) as avg_time_to_first_edit_ms,
+      ROUND(AVG(time_round_trip_ms), 2) as avg_time_round_trip_ms
+    FROM results
+    WHERE time_to_first_token_ms IS NOT NULL
+    GROUP BY model_id
+    ORDER BY avg_time_round_trip_ms ASC
+  `);
+  
+  return stmt.all() as ModelLatency[];
+}
+
+export async function getCostAnalysisByRun(): Promise<CostAnalysis[]> {
+  const stmt = db.getDatabase().prepare(`
+    SELECT 
+      run_id,
+      model_id,
+      ROUND(SUM(cost_usd), 4) as total_cost_usd,
+      ROUND(AVG(cost_usd), 4) as avg_cost_per_case,
+      SUM(completion_tokens) as total_completion_tokens
+    FROM results
+    WHERE cost_usd IS NOT NULL
+    GROUP BY run_id, model_id
+    ORDER BY total_cost_usd DESC
+  `);
+  
+  return stmt.all() as CostAnalysis[];
+}
+
+// Error analysis queries
+export async function getErrorDistribution(): Promise<ErrorDistribution[]> {
+  const stmt = db.getDatabase().prepare(`
+    SELECT 
+      error_enum,
+      COUNT(*) as count,
+      ROUND(COUNT(*) * 100.0 / (SELECT COUNT(*) FROM results WHERE succeeded = 0), 2) as percentage
+    FROM results
+    WHERE succeeded = 0 AND error_enum IS NOT NULL
+    GROUP BY error_enum
+    ORDER BY count DESC
+  `);
+  
+  return stmt.all() as ErrorDistribution[];
+}
+
+export async function getFailedCasesByError(errorEnum?: number): Promise<FailedCase[]> {
+  let query = `
+    SELECT 
+      r.case_id,
+      r.model_id,
+      r.error_enum,
+      c.description,
+      r.raw_model_output
+    FROM results r
+    JOIN cases c ON r.case_id = c.case_id
+    WHERE r.succeeded = 0
+  `;
+  
+  const params: any[] = [];
+  if (errorEnum !== undefined) {
+    query += ` AND r.error_enum = ?`;
+    params.push(errorEnum);
+  }
+  
+  query += ` ORDER BY r.created_at DESC LIMIT 100`;
+  
+  const stmt = db.getDatabase().prepare(query);
+  return stmt.all(...params) as FailedCase[];
+}
+
+// Trend analysis queries
+export async function getPerformanceTrends(days: number = 30): Promise<PerformanceTrend[]> {
+  const stmt = db.getDatabase().prepare(`
+    SELECT 
+      DATE(r.created_at) as date,
+      r.model_id,
+      ROUND(AVG(CASE WHEN r.succeeded THEN 1.0 ELSE 0.0 END) * 100, 2) as success_rate,
+      ROUND(AVG(r.time_round_trip_ms), 2) as avg_latency_ms,
+      ROUND(AVG(r.cost_usd), 4) as avg_cost_usd
+    FROM results r
+    WHERE r.created_at >= datetime('now', '-' || ? || ' days')
+      AND (r.error_enum NOT IN (1, 6, 7) OR r.error_enum IS NULL)  -- Exclude: no_tool_calls, wrong_tool_call, wrong_file_edited
+    GROUP BY DATE(r.created_at), r.model_id
+    ORDER BY date DESC, model_id
+  `);
+  
+  return stmt.all(days) as PerformanceTrend[];
+}
+
+export async function getModelComparisons(): Promise<ModelComparison[]> {
+  const stmt = db.getDatabase().prepare(`
+    SELECT 
+      model_id,
+      ROUND(AVG(CASE WHEN succeeded THEN 1.0 ELSE 0.0 END) * 100, 2) as success_rate,
+      ROUND(AVG(time_round_trip_ms), 2) as avg_latency_ms,
+      ROUND(AVG(cost_usd), 4) as avg_cost_usd,
+      COUNT(*) as total_runs
+    FROM results
+    WHERE error_enum NOT IN (1, 6, 7) OR error_enum IS NULL  -- Exclude: no_tool_calls, wrong_tool_call, wrong_file_edited
+    GROUP BY model_id
+    HAVING total_runs >= 10
+    ORDER BY success_rate DESC, avg_latency_ms ASC
+  `);
+  
+  return stmt.all() as ModelComparison[];
+}
+
+// Advanced analysis queries
+export async function getTopPerformingCases(limit: number = 10): Promise<Array<{
+  case_id: string;
+  description: string;
+  success_rate: number;
+  avg_latency_ms: number;
+  total_runs: number;
+}>> {
+  const stmt = db.getDatabase().prepare(`
+    SELECT 
+      c.case_id,
+      c.description,
+      ROUND(AVG(CASE WHEN r.succeeded THEN 1.0 ELSE 0.0 END) * 100, 2) as success_rate,
+      ROUND(AVG(r.time_round_trip_ms), 2) as avg_latency_ms,
+      COUNT(r.result_id) as total_runs
+    FROM cases c
+    JOIN results r ON c.case_id = r.case_id
+    WHERE r.error_enum NOT IN (1, 6, 7) OR r.error_enum IS NULL  -- Exclude: no_tool_calls, wrong_tool_call, wrong_file_edited
+    GROUP BY c.case_id, c.description
+    HAVING total_runs >= 5
+    ORDER BY success_rate DESC, avg_latency_ms ASC
+    LIMIT ?
+  `);
+  
+  return stmt.all(limit) as Array<{
+    case_id: string;
+    description: string;
+    success_rate: number;
+    avg_latency_ms: number;
+    total_runs: number;
+  }>;
+}
+
+export async function getWorstPerformingCases(limit: number = 10): Promise<Array<{
+  case_id: string;
+  description: string;
+  success_rate: number;
+  avg_latency_ms: number;
+  total_runs: number;
+}>> {
+  const stmt = db.getDatabase().prepare(`
+    SELECT 
+      c.case_id,
+      c.description,
+      ROUND(AVG(CASE WHEN r.succeeded THEN 1.0 ELSE 0.0 END) * 100, 2) as success_rate,
+      ROUND(AVG(r.time_round_trip_ms), 2) as avg_latency_ms,
+      COUNT(r.result_id) as total_runs
+    FROM cases c
+    JOIN results r ON c.case_id = r.case_id
+    WHERE r.error_enum NOT IN (1, 6, 7) OR r.error_enum IS NULL  -- Exclude: no_tool_calls, wrong_tool_call, wrong_file_edited
+    GROUP BY c.case_id, c.description
+    HAVING total_runs >= 5
+    ORDER BY success_rate ASC, avg_latency_ms DESC
+    LIMIT ?
+  `);
+  
+  return stmt.all(limit) as Array<{
+    case_id: string;
+    description: string;
+    success_rate: number;
+    avg_latency_ms: number;
+    total_runs: number;
+  }>;
+}
+
+export async function getModelPerformanceByTimeOfDay(): Promise<Array<{
+  model_id: string;
+  hour: number;
+  success_rate: number;
+  avg_latency_ms: number;
+  total_runs: number;
+}>> {
+  const stmt = db.getDatabase().prepare(`
+    SELECT 
+      model_id,
+      CAST(strftime('%H', created_at) AS INTEGER) as hour,
+      ROUND(AVG(CASE WHEN succeeded THEN 1.0 ELSE 0.0 END) * 100, 2) as success_rate,
+      ROUND(AVG(time_round_trip_ms), 2) as avg_latency_ms,
+      COUNT(*) as total_runs
+    FROM results
+    GROUP BY model_id, hour
+    HAVING total_runs >= 5
+    ORDER BY model_id, hour
+  `);
+  
+  return stmt.all() as Array<{
+    model_id: string;
+    hour: number;
+    success_rate: number;
+    avg_latency_ms: number;
+    total_runs: number;
+  }>;
+}
+
+export async function getRunComparison(runId1: string, runId2: string): Promise<{
+  run1: { run_id: string; success_rate: number; avg_latency_ms: number; avg_cost_usd: number; total_cases: number };
+  run2: { run_id: string; success_rate: number; avg_latency_ms: number; avg_cost_usd: number; total_cases: number };
+}> {
+  const stmt = db.getDatabase().prepare(`
+    SELECT 
+      run_id,
+      ROUND(AVG(CASE WHEN succeeded THEN 1.0 ELSE 0.0 END) * 100, 2) as success_rate,
+      ROUND(AVG(time_round_trip_ms), 2) as avg_latency_ms,
+      ROUND(AVG(cost_usd), 4) as avg_cost_usd,
+      COUNT(DISTINCT case_id) as total_cases
+    FROM results
+    WHERE run_id IN (?, ?)
+    GROUP BY run_id
+  `);
+  
+  const results = stmt.all(runId1, runId2) as Array<{
+    run_id: string;
+    success_rate: number;
+    avg_latency_ms: number;
+    avg_cost_usd: number;
+    total_cases: number;
+  }>;
+  
+  const run1 = results.find(r => r.run_id === runId1);
+  const run2 = results.find(r => r.run_id === runId2);
+  
+  if (!run1 || !run2) {
+    throw new Error('One or both runs not found');
+  }
+  
+  return { run1, run2 };
+}
+
+// Summary statistics
+export async function getDatabaseSummary(): Promise<{
+  total_runs: number;
+  total_cases: number;
+  total_results: number;
+  valid_results: number;
+  unique_models: number;
+  overall_success_rate: number;
+  date_range: { earliest: string; latest: string };
+}> {
+  const stmt = db.getDatabase().prepare(`
+    SELECT 
+      (SELECT COUNT(*) FROM runs) as total_runs,
+      (SELECT COUNT(*) FROM cases) as total_cases,
+      (SELECT COUNT(*) FROM results) as total_results,
+      (SELECT COUNT(*) FROM results WHERE error_enum NOT IN (1, 6, 7) OR error_enum IS NULL) as valid_results,
+      (SELECT COUNT(DISTINCT model_id) FROM results) as unique_models,
+      (SELECT ROUND(AVG(CASE WHEN succeeded THEN 1.0 ELSE 0.0 END) * 100, 2) 
+       FROM results 
+       WHERE error_enum NOT IN (1, 6, 7) OR error_enum IS NULL) as overall_success_rate,
+      (SELECT MIN(created_at) FROM results) as earliest,
+      (SELECT MAX(created_at) FROM results) as latest
+    FROM results
+    LIMIT 1
+  `);
+  
+  const result = stmt.get() as any;
+  return {
+    total_runs: result.total_runs || 0,
+    total_cases: result.total_cases || 0,
+    total_results: result.total_results || 0,
+    valid_results: result.valid_results || 0,
+    unique_models: result.unique_models || 0,
+    overall_success_rate: result.overall_success_rate || 0,
+    date_range: {
+      earliest: result.earliest || '',
+      latest: result.latest || ''
+    }
+  };
+}

evals/diff-edits/database/schema.sql

@@ -0,0 +1,78 @@
+PRAGMA foreign_keys = ON;
+
+CREATE TABLE system_prompts (
+    hash TEXT PRIMARY KEY,
+    name TEXT NOT NULL,
+    content TEXT NOT NULL,
+    created_at DATETIME DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE TABLE processing_functions (
+    hash TEXT PRIMARY KEY,
+    name TEXT NOT NULL,
+    parsing_function TEXT NOT NULL,
+    diff_edit_function TEXT NOT NULL,
+    created_at DATETIME DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE TABLE files (
+    hash TEXT PRIMARY KEY,
+    filepath TEXT NOT NULL,
+    content TEXT NOT NULL,
+    tokens INTEGER,
+    created_at DATETIME DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE TABLE runs (
+    run_id TEXT PRIMARY KEY,
+    created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+    description TEXT,
+    system_prompt_hash TEXT NOT NULL,
+    FOREIGN KEY (system_prompt_hash) REFERENCES system_prompts(hash)
+);
+
+CREATE TABLE cases (
+    case_id TEXT PRIMARY KEY,
+    run_id TEXT NOT NULL,
+    created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+    description TEXT NOT NULL,
+    system_prompt_hash TEXT NOT NULL,
+    task_id TEXT NOT NULL,
+    tokens_in_context INTEGER,
+    file_hash TEXT,
+    FOREIGN KEY (run_id) REFERENCES runs(run_id),
+    FOREIGN KEY (system_prompt_hash) REFERENCES system_prompts(hash),
+    FOREIGN KEY (file_hash) REFERENCES files(hash)
+);
+
+CREATE TABLE results (
+    result_id TEXT PRIMARY KEY,
+    run_id TEXT NOT NULL,
+    case_id TEXT NOT NULL,
+    model_id TEXT NOT NULL,
+    processing_functions_hash TEXT NOT NULL,
+    succeeded BOOLEAN NOT NULL,
+    error_enum INTEGER,
+    num_edits INTEGER,
+    num_lines_deleted INTEGER,
+    num_lines_added INTEGER,
+    time_to_first_token_ms INTEGER,
+    time_to_first_edit_ms INTEGER,
+    time_round_trip_ms INTEGER,
+    cost_usd REAL,
+    completion_tokens INTEGER,
+    raw_model_output TEXT,
+    file_edited_hash TEXT,
+    parsed_tool_call_json TEXT,
+    created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+    FOREIGN KEY (run_id) REFERENCES runs(run_id),
+    FOREIGN KEY (case_id) REFERENCES cases(case_id),
+    FOREIGN KEY (processing_functions_hash) REFERENCES processing_functions(hash)
+);
+
+CREATE INDEX idx_results_run_model ON results(run_id, model_id);
+CREATE INDEX idx_results_case_model ON results(case_id, model_id);
+CREATE INDEX idx_results_success ON results(succeeded);
+CREATE INDEX idx_cases_run ON cases(run_id);
+CREATE INDEX idx_results_created_at ON results(created_at);
+CREATE INDEX idx_runs_created_at ON runs(created_at);

evals/diff-edits/database/test.ts

@@ -0,0 +1,53 @@
+// Simple test to verify database functionality
+import { getDatabase } from './client';
+import { upsertSystemPrompt, createBenchmarkRun, getDatabaseSummary } from './index';
+
+async function testDatabase() {
+  console.log('Testing database functionality...');
+  
+  try {
+    // Test database connection
+    const db = getDatabase();
+    console.log('✓ Database connection established');
+    console.log('Database path:', db.getDatabasePath());
+    
+    // Test database info
+    const info = db.getInfo();
+    console.log('✓ Database info:', info);
+    
+    // Test database stats
+    const stats = db.getStats();
+    console.log('✓ Database stats:', stats);
+    
+    // Test system prompt creation
+    const systemPromptHash = await upsertSystemPrompt({
+      name: 'test-prompt',
+      content: 'This is a test system prompt for database verification.'
+    });
+    console.log('✓ System prompt created with hash:', systemPromptHash);
+    
+    // Test benchmark run creation
+    const runId = await createBenchmarkRun({
+      description: 'Test run for database verification',
+      system_prompt_hash: systemPromptHash
+    });
+    console.log('✓ Benchmark run created with ID:', runId);
+    
+    // Test database summary
+    const summary = await getDatabaseSummary();
+    console.log('✓ Database summary:', summary);
+    
+    console.log('\n🎉 All database tests passed!');
+    
+  } catch (error) {
+    console.error('❌ Database test failed:', error);
+    process.exit(1);
+  }
+}
+
+// Run test if this file is executed directly
+if (require.main === module) {
+  testDatabase();
+}
+
+export { testDatabase };

evals/diff-edits/database/types.ts

@@ -0,0 +1,169 @@
+// Database type definitions for diff-edits evaluation system
+
+export interface SystemPrompt {
+  hash: string;
+  name: string;
+  content: string;
+  created_at: string;
+}
+
+export interface ProcessingFunctions {
+  hash: string;
+  name: string;
+  parsing_function: string;
+  diff_edit_function: string;
+  created_at: string;
+}
+
+export interface FileRecord {
+  hash: string;
+  filepath: string;
+  content: string;
+  tokens?: number;
+  created_at: string;
+}
+
+export interface BenchmarkRun {
+  run_id: string;
+  created_at: string;
+  description?: string;
+  system_prompt_hash: string;
+}
+
+export interface Case {
+	case_id: string
+	run_id: string
+	created_at: string
+	description: string
+	system_prompt_hash: string
+	task_id: string
+	tokens_in_context: number
+	file_hash?: string
+}
+
+export interface Result {
+  result_id: string;
+  run_id: string;
+  case_id: string;
+  model_id: string;
+  processing_functions_hash: string;
+  succeeded: boolean;
+  error_enum?: number;
+  num_edits?: number;
+  num_lines_deleted?: number;
+  num_lines_added?: number;
+  time_to_first_token_ms?: number;
+  time_to_first_edit_ms?: number;
+  time_round_trip_ms?: number;
+  cost_usd?: number;
+  completion_tokens?: number;
+  raw_model_output?: string;
+  file_edited_hash?: string;
+  parsed_tool_call_json?: string;
+  created_at: string;
+}
+
+// Input types for creating records
+export interface CreateSystemPromptInput {
+  name: string;
+  content: string;
+}
+
+export interface CreateProcessingFunctionsInput {
+  name: string;
+  parsing_function: string;
+  diff_edit_function: string;
+}
+
+export interface CreateFileInput {
+  filepath: string;
+  content: string;
+  tokens?: number;
+}
+
+export interface CreateBenchmarkRunInput {
+  description?: string;
+  system_prompt_hash: string;
+}
+
+export interface CreateCaseInput {
+  run_id: string;
+  description: string;
+  system_prompt_hash: string;
+  task_id: string;
+  tokens_in_context: number;
+  file_hash?: string;
+}
+
+export interface CreateResultInput {
+  run_id: string;
+  case_id: string;
+  model_id: string;
+  processing_functions_hash: string;
+  succeeded: boolean;
+  error_enum?: number;
+  num_edits?: number;
+  num_lines_deleted?: number;
+  num_lines_added?: number;
+  time_to_first_token_ms?: number;
+  time_to_first_edit_ms?: number;
+  time_round_trip_ms?: number;
+  cost_usd?: number;
+  completion_tokens?: number;
+  raw_model_output?: string;
+  file_edited_hash?: string;
+  parsed_tool_call_json?: string;
+}
+
+// Analysis result types
+export interface ModelSuccessRate {
+  model_id: string;
+  total_runs: number;
+  successful_runs: number;
+  success_rate: number;
+}
+
+export interface ModelLatency {
+  model_id: string;
+  avg_time_to_first_token_ms: number;
+  avg_time_to_first_edit_ms: number;
+  avg_time_round_trip_ms: number;
+}
+
+export interface CostAnalysis {
+  run_id: string;
+  model_id: string;
+  total_cost_usd: number;
+  avg_cost_per_case: number;
+  total_completion_tokens: number;
+}
+
+export interface ErrorDistribution {
+  error_enum: number;
+  count: number;
+  percentage: number;
+}
+
+export interface FailedCase {
+  case_id: string;
+  model_id: string;
+  error_enum: number;
+  description: string;
+  raw_model_output?: string;
+}
+
+export interface PerformanceTrend {
+  date: string;
+  model_id: string;
+  success_rate: number;
+  avg_latency_ms: number;
+  avg_cost_usd: number;
+}
+
+export interface ModelComparison {
+  model_id: string;
+  success_rate: number;
+  avg_latency_ms: number;
+  avg_cost_usd: number;
+  total_runs: number;
+}

evals/diff-edits/openRouterModelsHelper.ts

@@ -0,0 +1,98 @@
+import axios from "axios";
+import path from "path";
+import fs from "fs/promises";
+
+// Minimal type for what we need from OpenRouter model info in evals
+export interface EvalOpenRouterModelInfo {
+    id: string;
+    contextWindow: number;
+    inputPrice?: number; // Price per million tokens
+    outputPrice?: number; // Price per million tokens
+    // Add any other fields if they become necessary for evals
+}
+
+function logHelper(isVerbose: boolean, message: string) {
+    if (isVerbose) {
+        console.log(`[OpenRouterModelsHelper] ${message}`);
+    }
+}
+
+/**
+ * Ensures the cache directory exists within evals and returns its path
+ */
+async function ensureEvalCacheDirectoryExists(): Promise<string> {
+    // Cache directory within evals, e.g., evals/.cache/
+    const cacheDir = path.join(__dirname, "..", ".cache");
+    await fs.mkdir(cacheDir, { recursive: true });
+    return cacheDir;
+}
+
+/**
+ * Fetches, parses, and caches OpenRouter model data.
+ * Tries to read from a local cache first.
+ * @param isVerbose Enable verbose logging
+ * @returns A record of model IDs to their info.
+ */
+export async function loadOpenRouterModelData(isVerbose: boolean = false): Promise<Record<string, EvalOpenRouterModelInfo>> {
+    const cacheDir = await ensureEvalCacheDirectoryExists();
+    const cacheFilePath = path.join(cacheDir, "openRouterModels.json");
+    let models: Record<string, EvalOpenRouterModelInfo> = {};
+
+    try {
+        const stats = await fs.stat(cacheFilePath).catch(() => null);
+        // Use cache if less than 24 hours old
+        if (stats && (Date.now() - stats.mtimeMs < 24 * 60 * 60 * 1000)) {
+            logHelper(isVerbose, "Using cached OpenRouter model data.");
+            const fileContents = await fs.readFile(cacheFilePath, "utf8");
+            models = JSON.parse(fileContents);
+            if (Object.keys(models).length > 0) {
+                return models;
+            }
+            logHelper(isVerbose, "Cache was empty or invalid, fetching fresh data.");
+        } else if (stats) {
+            logHelper(isVerbose, "Cached OpenRouter model data is stale, fetching fresh data.");
+        } else {
+            logHelper(isVerbose, "No cached OpenRouter model data found, fetching fresh data.");
+        }
+    } catch (e) {
+        logHelper(isVerbose, `Error accessing cache, fetching fresh data: ${e}`);
+    }
+
+    try {
+        const response = await axios.get("https://openrouter.ai/api/v1/models");
+        if (response.data?.data) {
+            const rawModels = response.data.data;
+            const parsedModels: Record<string, EvalOpenRouterModelInfo> = {};
+            const parsePrice = (price: any) => price ? parseFloat(price) * 1_000_000 : undefined;
+
+            for (const rawModel of rawModels) {
+                parsedModels[rawModel.id] = {
+                    id: rawModel.id,
+                    contextWindow: rawModel.context_length ?? 0,
+                    inputPrice: parsePrice(rawModel.pricing?.prompt),
+                    outputPrice: parsePrice(rawModel.pricing?.completion),
+                };
+            }
+            await fs.writeFile(cacheFilePath, JSON.stringify(parsedModels, null, 2));
+            logHelper(isVerbose, `Fetched and cached ${Object.keys(parsedModels).length} OpenRouter models.`);
+            return parsedModels;
+        } else {
+            logHelper(isVerbose, "Invalid response structure from OpenRouter API.");
+        }
+    } catch (error) {
+        logHelper(isVerbose, `Error fetching OpenRouter models: ${error}. Attempting to use stale cache if available.`);
+        // Attempt to read stale cache as a last resort if fetching failed
+        try {
+            const fileContents = await fs.readFile(cacheFilePath, "utf8");
+            models = JSON.parse(fileContents);
+            if (Object.keys(models).length > 0) {
+                logHelper(isVerbose, "Successfully loaded stale cache after fetch failure.");
+                return models;
+            }
+        } catch (cacheError) {
+            logHelper(isVerbose, `Failed to read stale cache: ${cacheError}. Proceeding without OpenRouter model data.`);
+        }
+    }
+    // Return empty if all attempts fail, so the caller can decide how to handle it
+    return {};
+}

evals/diff-edits/run_and_open_dashboard.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+# Get the directory of this script to make paths robust
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+# The 'evals' directory is the parent of the script's directory
+EVALS_DIR=$(dirname "$SCRIPT_DIR")
+
+# Navigate to the evals directory to ensure npm commands run correctly
+cd "$EVALS_DIR"
+
+# Re-install dependencies and build the CLI
+echo "Ensuring dependencies are up to date and building CLI..."
+npm install && npm run build:cli
+
+# Check if the build was successful before proceeding
+if [ $? -ne 0 ]; then
+    echo "CLI build failed. Aborting evaluation."
+    exit 1
+fi
+
+# Run the evaluation script, passing all arguments from the command line
+echo "Running evaluation..."
+node ./cli/dist/index.js run-diff-eval "$@"
+
+# Check the exit code of the evaluation script
+if [ $? -eq 0 ]; then
+  # If the script succeeded, open the dashboard in the background
+  echo "Evaluation complete. Starting dashboard..."
+  (cd "$SCRIPT_DIR/dashboard" && streamlit run app.py &)
+else
+  # If the script failed, print an error message and exit
+  echo "Evaluation failed. Dashboard will not be started."
+  exit 1
+fi

evals/diff-edits/types.ts

@@ -61,7 +61,22 @@ export type ConstructSystemPromptFn = (
 
 export interface TestResult {
 	success: boolean
-	streamResult?: any
+	streamResult?: {
+		assistantMessage: string
+		reasoningMessage: string
+		usage: {
+			inputTokens: number
+			outputTokens: number
+			cacheWriteTokens: number
+			cacheReadTokens: number
+			totalCost: number
+		}
+		timing?: {
+			timeToFirstTokenMs: number
+			timeToFirstEditMs?: number
+			totalRoundTripMs: number
+		}
+	}
 	diffEdit?: string
 	toolCalls?: ExtractedToolCall[]
 	diffEditSuccess?: boolean

evals/diff_editing/TestRunner.ts

@@ -1,391 +0,0 @@
-import { runSingleEvaluation, TestInput, TestResult } from "./ClineWrapper"
-import { basicSystemPrompt } from "./prompts/basicSystemPrompt-06-06-25"
-import { claude4SystemPrompt } from "./prompts/claude4SystemPrompt-06-06-25"
-import { formatResponse } from "./helpers"
-import { Anthropic } from "@anthropic-ai/sdk"
-import * as fs from "fs"
-import * as path from "path"
-import { Command } from "commander"
-import { InputMessage, ProcessedTestCase, TestCase, TestConfig, SystemPromptDetails, ConstructSystemPromptFn } from "./types"
-
-function log(isVerbose: boolean, message: string) {
-	if (isVerbose) {
-		console.log(message)
-	}
-}
-
-const systemPromptGeneratorLookup: Record<string, ConstructSystemPromptFn> = {
-	basicSystemPrompt: basicSystemPrompt,
-	claude4SystemPrompt: claude4SystemPrompt,
-}
-
-type TestResultSet = { [test_id: string]: (TestResult & { test_id?: string })[] }
-
-class NodeTestRunner {
-	private apiKey: string | undefined
-
-	constructor(isReplay: boolean) {
-		if (!isReplay) {
-			this.apiKey = process.env.OPENROUTER_API_KEY
-			if (!this.apiKey) {
-				throw new Error("OPENROUTER_API_KEY environment variable not set for a non-replay run.")
-			}
-		}
-	}
-
-	/**
-	 * convert our messages array into a properly formatted Anthropic messages array
-	 */
-	transformMessages(messages: InputMessage[]): Anthropic.Messages.MessageParam[] {
-		return messages.map((msg) => {
-			// Use TextBlockParam here for constructing the input message
-			const content: (Anthropic.TextBlockParam | Anthropic.ImageBlockParam)[] = []
-
-			if (msg.text) {
-				// This object now correctly matches the TextBlockParam type
-				content.push({ type: "text", text: msg.text })
-			}
-
-			if (msg.images && Array.isArray(msg.images)) {
-				const imageBlocks = formatResponse.imageBlocks(msg.images)
-				content.push(...imageBlocks)
-			}
-
-			return {
-				role: msg.role,
-				content: content,
-			}
-		})
-	}
-
-	/**
-	 * Generate the system prompt on the fly
-	 */
-	constructSystemPrompt(systemPromptDetails: SystemPromptDetails, systemPromptName: string) {
-		const systemPromptGenerator = systemPromptGeneratorLookup[systemPromptName]
-
-		const { cwd_value, browser_use, width, height, os_value, shell_value, home_value, mcp_string, user_custom_instructions } =
-			systemPromptDetails
-
-		const systemPrompt = systemPromptGenerator(
-			cwd_value,
-			browser_use,
-			width,
-			height,
-			os_value,
-			shell_value,
-			home_value,
-			mcp_string,
-			user_custom_instructions,
-		)
-
-		return systemPrompt
-	}
-
-	/**
-	 * Loads our test cases from a directory of json files
-	 */
-	loadTestCases(testDirectoryPath: string): TestCase[] {
-		const testCasesArray: TestCase[] = []
-		const dirents = fs.readdirSync(testDirectoryPath, { withFileTypes: true })
-
-		for (const dirent of dirents) {
-			if (dirent.isFile() && dirent.name.endsWith(".json")) {
-				const testFilePath = path.join(testDirectoryPath, dirent.name)
-				const fileContent = fs.readFileSync(testFilePath, "utf8")
-				const testCase: TestCase = JSON.parse(fileContent)
-
-				// Use the filename (without extension) as the test_id if not provided
-				if (!testCase.test_id) {
-					testCase.test_id = path.parse(dirent.name).name
-				}
-				testCasesArray.push(testCase)
-			}
-		}
-
-		return testCasesArray
-	}
-
-	/**
-	 * Saves the test results to the specified output directory.
-	 */
-	saveTestResults(results: TestResultSet, outputPath: string) {
-		// Ensure output directory exists
-		if (!fs.existsSync(outputPath)) {
-			fs.mkdirSync(outputPath, { recursive: true })
-		}
-
-		// Write each test result to its own file
-		for (const testId in results) {
-			const outputFilePath = path.join(outputPath, `${testId}.json`)
-			const testResult = results[testId]
-			fs.writeFileSync(outputFilePath, JSON.stringify(testResult, null, 2))
-		}
-	}
-
-	/**
-	 * Run a single test example
-	 */
-	async runSingleTest(testCase: ProcessedTestCase, testConfig: TestConfig): Promise<TestResult> {
-		if (testConfig.replay && !testCase.original_diff_edit_tool_call_message) {
-			return {
-				success: false,
-				error: "missing_original_diff_edit_tool_call_message",
-				errorString: `Test case ${testCase.test_id} is missing 'original_diff_edit_tool_call_message' for replay.`,
-			}
-		}
-
-		const customSystemPrompt = this.constructSystemPrompt(testCase.system_prompt_details, testConfig.system_prompt_name)
-
-		// messages don't include system prompt and are everything up to the first replace_in_file tool call which results in a diff edit error
-		const input: TestInput = {
-			apiKey: this.apiKey,
-			systemPrompt: customSystemPrompt,
-			messages: testCase.messages,
-			modelId: testConfig.model_id,
-			originalFile: testCase.file_contents,
-			originalFilePath: testCase.file_path,
-			parsingFunction: testConfig.parsing_function,
-			diffEditFunction: testConfig.diff_edit_function,
-			thinkingBudgetTokens: testConfig.thinking_tokens_budget,
-			originalDiffEditToolCallMessage: testConfig.replay ? testCase.original_diff_edit_tool_call_message : undefined,
-		}
-
-		return await runSingleEvaluation(input)
-	}
-
-	/**
-	 * Runs all the text examples synchonously
-	 */
-	async runAllTests(testCases: ProcessedTestCase[], testConfig: TestConfig, isVerbose: boolean): Promise<TestResultSet> {
-		const results: TestResultSet = {}
-
-		for (const testCase of testCases) {
-			results[testCase.test_id] = []
-
-			log(isVerbose, `-Running test: ${testCase.test_id}`)
-			for (let i = 0; i < testConfig.number_of_runs; i++) {
-				const result = await this.runSingleTest(testCase, testConfig)
-				results[testCase.test_id].push(result)
-			}
-		}
-		return results
-	}
-
-	/**
-	 * Runs all of the text examples asynchronously, with concurrency limit
-	 */
-	async runAllTestsParallel(
-		testCases: ProcessedTestCase[],
-		testConfig: TestConfig,
-		isVerbose: boolean,
-		maxConcurrency: number = 20,
-	): Promise<TestResultSet> {
-		const results: TestResultSet = {}
-		testCases.forEach((tc) => {
-			results[tc.test_id] = []
-		})
-
-		// Create a flat list of all individual runs we need to execute
-		const allRuns = testCases.flatMap((testCase) =>
-			Array(testConfig.number_of_runs)
-				.fill(null)
-				.map(() => testCase),
-		)
-
-		for (let i = 0; i < allRuns.length; i += maxConcurrency) {
-			const batch = allRuns.slice(i, i + maxConcurrency)
-
-			const batchPromises = batch.map((testCase) =>
-				this.runSingleTest(testCase, testConfig).then((result) => ({
-					...result,
-					test_id: testCase.test_id,
-				})),
-			)
-
-			const batchResults = await Promise.all(batchPromises)
-
-			// Calculate the total cost for this batch
-			const batchCost = batchResults.reduce((total, result) => {
-				return total + (result.streamResult?.usage?.totalCost || 0)
-			}, 0)
-
-			// Populate the results dictionary
-			for (const result of batchResults) {
-				if (result.test_id) {
-					results[result.test_id].push(result)
-				}
-			}
-
-			const batchNumber = i / maxConcurrency + 1
-			const totalBatches = Math.ceil(allRuns.length / maxConcurrency)
-			log(isVerbose, `-Completed batch ${batchNumber} of ${totalBatches}... (Batch Cost: $${batchCost.toFixed(6)})`)
-		}
-
-		return results
-	}
-
-	/**
-	 * Print output of the tests
-	 */
-	printSummary(results: TestResultSet, isVerbose: boolean) {
-		let totalRuns = 0
-		let totalPasses = 0
-		let totalInputTokens = 0
-		let totalOutputTokens = 0
-		let totalCost = 0
-		let runsWithUsageData = 0
-		let totalDiffEditSuccesses = 0
-		let totalRunsWithToolCalls = 0
-		const testCaseIds = Object.keys(results)
-
-		log(isVerbose, "\n=== TEST SUMMARY ===")
-
-		for (const testId of testCaseIds) {
-			const testResults = results[testId]
-			const passedCount = testResults.filter((r) => r.success && r.diffEditSuccess).length
-			const runCount = testResults.length
-
-			totalRuns += runCount
-			totalPasses += passedCount
-
-			const runsWithToolCalls = testResults.filter((r) => r.success === true).length
-			const diffEditSuccesses = passedCount
-			totalRunsWithToolCalls += runsWithToolCalls
-			totalDiffEditSuccesses += diffEditSuccesses
-
-			// Accumulate token and cost data
-			for (const result of testResults) {
-				if (result.streamResult?.usage) {
-					totalInputTokens += result.streamResult.usage.inputTokens
-					totalOutputTokens += result.streamResult.usage.outputTokens
-					totalCost += result.streamResult.usage.totalCost
-					runsWithUsageData++
-				}
-			}
-
-			log(isVerbose, `\n--- Test Case: ${testId} ---`)
-			log(isVerbose, `  Runs: ${runCount}`)
-			log(isVerbose, `  Passed: ${passedCount}`)
-			log(isVerbose, `  Success Rate: ${runCount > 0 ? ((passedCount / runCount) * 100).toFixed(1) : "N/A"}%`)
-		}
-
-		log(isVerbose, "\n\n=== OVERALL SUMMARY ===")
-		log(isVerbose, `Total Test Cases: ${testCaseIds.length}`)
-		log(isVerbose, `Total Runs Executed: ${totalRuns}`)
-		log(isVerbose, `Overall Passed: ${totalPasses}`)
-		log(isVerbose, `Overall Failed: ${totalRuns - totalPasses}`)
-		log(isVerbose, `Overall Success Rate: ${totalRuns > 0 ? ((totalPasses / totalRuns) * 100).toFixed(1) : "N/A"}%`)
-
-		log(isVerbose, "\n\n=== OVERALL DIFF EDIT SUCCESS RATE ===")
-		if (totalRunsWithToolCalls > 0) {
-			const diffSuccessRate = (totalDiffEditSuccesses / totalRunsWithToolCalls) * 100
-			log(isVerbose, `Total Runs with Successful Tool Calls: ${totalRunsWithToolCalls}`)
-			log(isVerbose, `Total Runs with Successful Diff Edits: ${totalDiffEditSuccesses}`)
-			log(isVerbose, `Diff Edit Success Rate: ${diffSuccessRate.toFixed(1)}%`)
-		} else {
-			log(isVerbose, "No successful tool calls to analyze for diff edit success.")
-		}
-
-		log(isVerbose, "\n\n=== TOKEN & COST ANALYSIS ===")
-		if (runsWithUsageData > 0) {
-			log(isVerbose, `Total Input Tokens: ${totalInputTokens.toLocaleString()}`)
-			log(isVerbose, `Total Output Tokens: ${totalOutputTokens.toLocaleString()}`)
-			log(isVerbose, `Total Cost: $${totalCost.toFixed(6)}`)
-			log(isVerbose, "---")
-			log(
-				isVerbose,
-				`Avg Input Tokens / Run: ${(totalInputTokens / runsWithUsageData).toLocaleString(undefined, {
-					maximumFractionDigits: 0,
-				})}`,
-			)
-			log(
-				isVerbose,
-				`Avg Output Tokens / Run: ${(totalOutputTokens / runsWithUsageData).toLocaleString(undefined, {
-					maximumFractionDigits: 0,
-				})}`,
-			)
-			log(isVerbose, `Avg Cost / Run: $${(totalCost / runsWithUsageData).toFixed(6)}`)
-		} else {
-			log(isVerbose, "No usage data available to analyze.")
-		}
-	}
-}
-
-async function main() {
-	const program = new Command()
-
-	const defaultTestPath = path.join(__dirname, "test_cases")
-	const defaultOutputPath = path.join(__dirname, "test_outputs")
-
-	program
-		.name("TestRunner")
-		.description("Run evaluation tests for diff editing")
-		.version("1.0.0")
-		.option("--test-path <path>", "Path to the directory containing test case JSON files", defaultTestPath)
-		.option("--output-path <path>", "Path to the directory to save the test output JSON files", defaultOutputPath)
-		.option("--model-id <model_id>", "The model ID to use for the test")
-		.option("--system-prompt-name <name>", "The name of the system prompt to use", "basicSystemPrompt")
-		.option("-n, --number-of-runs <number>", "Number of times to run each test case", "1")
-		.option("--parsing-function <name>", "The parsing function to use", "parseAssistantMessageV2")
-		.option("--diff-edit-function <name>", "The diff editing function to use", "constructNewFileContentV2")
-		.option("--thinking-budget <tokens>", "Set the thinking tokens budget", "0")
-		.option("--parallel", "Run tests in parallel", false)
-		.option("--replay", "Run evaluation from a pre-recorded LLM output, skipping the API call", false)
-		.option("-v, --verbose", "Enable verbose logging", false)
-
-	program.parse(process.argv)
-
-	const options = program.opts()
-	const isVerbose = options.verbose
-	const testPath = options.testPath
-	const outputPath = options.outputPath
-
-	const testConfig: TestConfig = {
-		model_id: options.modelId,
-		system_prompt_name: options.systemPromptName,
-		number_of_runs: parseInt(options.numberOfRuns, 10),
-		parsing_function: options.parsingFunction,
-		diff_edit_function: options.diffEditFunction,
-		thinking_tokens_budget: parseInt(options.thinkingBudget, 10),
-		replay: options.replay,
-	}
-
-	try {
-		const startTime = Date.now()
-
-		const runner = new NodeTestRunner(testConfig.replay)
-		const testCases = runner.loadTestCases(testPath)
-
-		const processedTestCases: ProcessedTestCase[] = testCases.map((tc) => ({
-			...tc,
-			messages: runner.transformMessages(tc.messages),
-		}))
-
-		log(isVerbose, `-Loaded ${testCases.length} test cases.`)
-		log(isVerbose, `-Executing ${testConfig.number_of_runs} run(s) per test case.`)
-		if (testConfig.replay) {
-			log(isVerbose, `-Running in REPLAY mode. No API calls will be made.`)
-		}
-		log(isVerbose, "Starting tests...\n")
-
-		const results = options.parallel
-			? await runner.runAllTestsParallel(processedTestCases, testConfig, isVerbose)
-			: await runner.runAllTests(processedTestCases, testConfig, isVerbose)
-
-		runner.printSummary(results, isVerbose)
-
-		const endTime = Date.now()
-		const durationSeconds = ((endTime - startTime) / 1000).toFixed(2)
-		log(isVerbose, `\n-Total execution time: ${durationSeconds} seconds`)
-
-		runner.saveTestResults(results, outputPath)
-	} catch (error) {
-		console.error("\nError running tests:", error)
-		process.exit(1)
-	}
-}
-
-if (require.main === module) {
-	main()
-}

evals/package.json

@@ -0,0 +1,44 @@
+{
+  "name": "cline-evals",
+  "version": "0.1.0",
+  "description": "Evaluation scripts and tools for Cline",
+  "main": "cli/dist/index.js",
+  "scripts": {
+    "build:cli": "cd cli && tsc",
+    "start:cli": "cd cli && node dist/index.js",
+    "dev:cli": "cd cli && ts-node src/index.ts",
+    "diff-eval": "./diff-edits/run_and_open_dashboard.sh",
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "keywords": [
+    "cline",
+    "evaluation",
+    "benchmark",
+    "diff-edits"
+  ],
+  "author": "",
+  "license": "MIT",
+  "dependencies": {
+    "axios": "^1.8.2",
+    "better-sqlite3": "^11.10.0",
+    "chalk": "^4.1.2",
+    "dotenv": "^16.5.0",
+    "commander": "^9.4.1",
+    "execa": "^5.1.1",
+    "node-fetch": "^2.7.0",
+    "ora": "^5.4.1",
+    "sqlite": "^4.1.2",
+    "tiktoken": "^1.0.21",
+    "uuid": "^9.0.0",
+    "yargs": "^17.6.2"
+  },
+  "devDependencies": {
+    "@types/better-sqlite3": "^7.6.3",
+    "@types/node": "^18.11.18",
+    "@types/node-fetch": "^2.6.12",
+    "@types/uuid": "^9.0.0",
+    "@types/yargs": "^17.0.19",
+    "ts-node": "^10.9.1",
+    "typescript": "^4.9.4"
+  }
+}

evals/tsconfig.json

@@ -0,0 +1,6 @@
+{
+	"extends": "../tsconfig.json",
+	"compilerOptions": {
+		"baseUrl": ".."
+	}
+}

package.json

@@ -2,7 +2,7 @@
 	"name": "claude-dev",
 	"displayName": "Cline",
 	"description": "Autonomous coding agent right in your IDE, capable of creating/editing files, running commands, using the browser, and more with your permission every step of the way.",
-	"version": "3.17.12",
+	"version": "3.17.15",
 	"icon": "assets/icons/icon.png",
 	"engines": {
 		"vscode": "^1.84.0"
@@ -330,13 +330,13 @@
 		"watch:esbuild": "node esbuild.js --watch",
 		"watch:tsc": "tsc --noEmit --watch --project tsconfig.json",
 		"package": "npm run check-types && npm run build:webview && npm run lint && node esbuild.js --production",
-		"protos": "node proto/build-proto.js && node scripts/generate-server-setup.mjs",
-		"postprotos": "prettier src/shared/proto src/core/controller src/hosts/ webview-ui/src/services src/standalone/server-setup.ts --write --log-level silent",
+		"protos": "node proto/build-proto.js && node scripts/generate-server-setup.mjs && node scripts/generate-host-bridge-client.mjs",
+		"postprotos": "prettier src/shared/proto src/core/controller src/hosts/ webview-ui/src/services src/generated --write --log-level warn",
 		"compile-tests": "node ./scripts/build-tests.js",
 		"watch-tests": "tsc -p . -w --outDir out",
 		"pretest": "npm run compile-tests && npm run compile && npm run compile-standalone && npm run lint",
 		"check-types": "npm run protos && tsc --noEmit",
-		"lint": "eslint src --ext ts && eslint webview-ui/src --ext ts && cd webview-ui && npm run lint",
+		"lint": "eslint src --ext ts && eslint webview-ui/src --ext ts && buf lint && cd webview-ui && npm run lint",
 		"format": "prettier . --check",
 		"format:fix": "prettier . --write",
 		"test": "npm-run-all test:unit test:integration",
@@ -358,7 +358,13 @@
 		"docs:rename-file": "cd docs && mintlify rename",
 		"report-issue": "node scripts/report-issue.js"
 	},
+	"lint-staged": {
+		"*": [
+			"prettier --write --ignore-unknown --log-level=log"
+		]
+	},
 	"devDependencies": {
+		"@bufbuild/buf": "^1.54.0",
 		"@changesets/cli": "^2.27.12",
 		"@types/chai": "^5.0.1",
 		"@types/clone-deep": "^4.0.4",
@@ -384,6 +390,7 @@
 		"eslint-plugin-eslint-rules": "file:eslint-rules",
 		"grpc-tools": "^1.13.0",
 		"husky": "^9.1.7",
+		"lint-staged": "^16.1.0",
 		"mintlify": "^4.0.515",
 		"npm-run-all": "^4.1.5",
 		"prettier": "^3.3.3",
@@ -397,14 +404,14 @@
 		"typescript": "^5.4.5"
 	},
 	"dependencies": {
-		"@anthropic-ai/bedrock-sdk": "^0.12.4",
 		"@anthropic-ai/sdk": "^0.37.0",
 		"@anthropic-ai/vertex-sdk": "^0.6.4",
-		"@aws-sdk/client-bedrock-runtime": "^3.821.0",
+		"@aws-sdk/client-bedrock-runtime": "^3.826.0",
+		"@aws-sdk/credential-providers": "^3.826.0",
 		"@bufbuild/protobuf": "^2.2.5",
 		"@cerebras/cerebras_cloud_sdk": "^1.35.0",
 		"@google-cloud/vertexai": "^1.9.3",
-		"@google/genai": "^0.13.0",
+		"@google/genai": "1.0.0",
 		"@grpc/grpc-js": "^1.9.15",
 		"@grpc/reflection": "^1.0.4",
 		"@mistralai/mistralai": "^1.5.0",
@@ -441,6 +448,7 @@
 		"jschardet": "^3.1.4",
 		"mammoth": "^1.8.0",
 		"monaco-vscode-textmate-theme-converter": "^0.1.7",
+		"nice-grpc": "^2.1.12",
 		"ollama": "^0.5.13",
 		"open": "^10.1.2",
 		"open-graph-scraper": "^6.9.0",

proto/account.proto

@@ -19,4 +19,23 @@ service AccountService {
 
     // Subscribe to auth callback events (when authentication tokens are received)
     rpc subscribeToAuthCallback(EmptyRequest) returns (stream String);
+    
+    // Handles authentication state changes from the Firebase context.
+    // Updates the user info in global state and returns the updated value.
+    rpc authStateChanged(AuthStateChangedRequest) returns (AuthStateChanged);
+}
+
+message AuthStateChangedRequest {
+    Metadata metadata = 1;
+    UserInfo user = 2;
+}
+
+message AuthStateChanged {
+    optional UserInfo user = 1;
+}
+
+message UserInfo {
+    optional string display_name = 1;
+    optional string email = 2;
+    optional string photo_url = 3;
 }

proto/build-proto.js

@@ -10,24 +10,26 @@ import os from "os"
 
 import { createRequire } from "module"
 const require = createRequire(import.meta.url)
-const protoc = path.join(require.resolve("grpc-tools"), "../bin/protoc")
+const PROTOC = path.join(require.resolve("grpc-tools"), "../bin/protoc")
 
-const __filename = fileURLToPath(import.meta.url)
-const SCRIPT_DIR = path.dirname(__filename)
+const SCRIPT_DIR = path.dirname(fileURLToPath(import.meta.url))
 const ROOT_DIR = path.resolve(SCRIPT_DIR, "..")
 
+const TS_OUT_DIR = path.join(ROOT_DIR, "src/shared/proto")
+const GRPC_JS_OUT_DIR = path.join(ROOT_DIR, "src/generated/grpc-js")
+const NICE_JS_OUT_DIR = path.join(ROOT_DIR, "src/generated/nice-grpc")
+const DESCRIPTOR_OUT_DIR = path.join(ROOT_DIR, "dist-standalone/proto")
+
 const isWindows = process.platform === "win32"
-const tsProtoPlugin = isWindows
+const TS_PROTO_PLUGIN = isWindows
 	? path.join(ROOT_DIR, "node_modules", ".bin", "protoc-gen-ts_proto.cmd") // Use the .bin directory path for Windows
 	: require.resolve("ts-proto/protoc-gen-ts_proto")
 
 const TS_PROTO_OPTIONS = [
 	"env=node",
 	"esModuleInterop=true",
-
+	"outputServices=generic-definitions", // output generic ServiceDefinitions
 	"outputIndex=true", // output an index file for each package which exports all protos in the package.
-	"outputServices=generic-definitions",
-
 	"useOptionals=messages", // Message fields are optional, scalars are not.
 	"useDate=false", // Timestamp fields will not be automatically converted to Date.
 ]
@@ -49,7 +51,7 @@ const serviceNameMap = {
 	ui: "cline.UiService",
 	// Add new services here - no other code changes needed!
 }
-const serviceDirs = Object.keys(serviceNameMap).map((serviceKey) => path.join(ROOT_DIR, "src", "core", "controller", serviceKey))
+const serviceDirs = Object.keys(serviceNameMap).map((serviceKey) => path.join(ROOT_DIR, "src/core/controller", serviceKey))
 
 // List of host gRPC services (IDE API bridge)
 // These services are implemented in the IDE extension and called by the standalone Cline Core
@@ -58,9 +60,7 @@ const hostServiceNameMap = {
 	watch: "host.WatchService",
 	// Add new host services here
 }
-const hostServiceDirs = Object.keys(hostServiceNameMap).map((serviceKey) =>
-	path.join(ROOT_DIR, "src", "hosts", "vscode", serviceKey),
-)
+const hostServiceDirs = Object.keys(hostServiceNameMap).map((serviceKey) => path.join(ROOT_DIR, "src/hosts/vscode", serviceKey))
 
 async function main() {
 	console.log(chalk.bold.blue("Starting Protocol Buffer code generation..."))
@@ -68,70 +68,72 @@ async function main() {
 	// Check for Apple Silicon compatibility before proceeding
 	checkAppleSiliconCompatibility()
 
-	// Define output directories
-	const TS_OUT_DIR = path.join(ROOT_DIR, "src", "shared", "proto")
-
 	// Create output directories if they don't exist
-	await fs.mkdir(TS_OUT_DIR, { recursive: true })
-
-	// Clean up existing generated files
-	console.log(chalk.cyan("Cleaning up existing generated TypeScript files..."))
-	const existingFiles = await globby("**/*.ts", { cwd: TS_OUT_DIR })
-	for (const file of existingFiles) {
-		await fs.unlink(path.join(TS_OUT_DIR, file))
+	for (const dir of [TS_OUT_DIR, GRPC_JS_OUT_DIR, NICE_JS_OUT_DIR, DESCRIPTOR_OUT_DIR]) {
+		await fs.mkdir(dir, { recursive: true })
 	}
 
+	await cleanup()
+
 	// Check for missing proto files for services in serviceNameMap
 	await ensureProtoFilesExist()
 
 	// Process all proto files
-	console.log(chalk.cyan("Processing proto files from"), SCRIPT_DIR)
 	const protoFiles = await globby("**/*.proto", { cwd: SCRIPT_DIR, realpath: true })
+	console.log(chalk.cyan(`Processing ${protoFiles.length} proto files from`), SCRIPT_DIR)
 
-	// Build the protoc command with proper path handling for cross-platform
-	const tsProtocCommand = [
-		protoc,
-		`--proto_path="${SCRIPT_DIR}"`,
-		`--plugin=protoc-gen-ts_proto="${tsProtoPlugin}"`,
-		`--ts_proto_out="${TS_OUT_DIR}"`,
-		`--ts_proto_opt=${TS_PROTO_OPTIONS.join(",")} `,
-		...protoFiles,
-	].join(" ")
-	try {
-		console.log(chalk.cyan(`Generating TypeScript code for:\n${protoFiles.join("\n")}...`))
-		execSync(tsProtocCommand, { stdio: "inherit" })
-	} catch (error) {
-		console.error(chalk.red("Error generating TypeScript for proto files:"), error)
-		process.exit(1)
-	}
+	tsProtoc(TS_OUT_DIR, protoFiles, TS_PROTO_OPTIONS)
+	// grpc-js is used to generate service impls for the ProtoBus service.
+	tsProtoc(GRPC_JS_OUT_DIR, protoFiles, ["outputServices=grpc-js,outputClientImpl=false", ...TS_PROTO_OPTIONS])
+	// nice-js is used for the Host Bridge client impls because it uses promises.
+	tsProtoc(NICE_JS_OUT_DIR, protoFiles, ["outputServices=nice-grpc,useExactTypes=false", ...TS_PROTO_OPTIONS])
 
-	const descriptorOutDir = path.join(ROOT_DIR, "dist-standalone", "proto")
-	await fs.mkdir(descriptorOutDir, { recursive: true })
-	const descriptorFile = path.join(descriptorOutDir, "descriptor_set.pb")
+	const descriptorFile = path.join(DESCRIPTOR_OUT_DIR, "descriptor_set.pb")
 	const descriptorProtocCommand = [
-		protoc,
+		PROTOC,
 		`--proto_path="${SCRIPT_DIR}"`,
 		`--descriptor_set_out="${descriptorFile}"`,
 		"--include_imports",
 		...protoFiles,
 	].join(" ")
 	try {
-		console.log(chalk.cyan("Generating descriptor set..."))
+		log_verbose(chalk.cyan("Generating descriptor set..."))
 		execSync(descriptorProtocCommand, { stdio: "inherit" })
 	} catch (error) {
 		console.error(chalk.red("Error generating descriptor set for proto file:"), error)
 		process.exit(1)
 	}
 
-	console.log(chalk.green("Protocol Buffer code generation completed successfully."))
-	console.log(chalk.green(`TypeScript files generated in: ${TS_OUT_DIR}`))
+	log_verbose(chalk.green("Protocol Buffer code generation completed successfully."))
+	log_verbose(chalk.green(`TypeScript files generated in: ${TS_OUT_DIR}`))
 
 	await generateMethodRegistrations()
 	await generateHostMethodRegistrations()
 	await generateServiceConfig()
 	await generateHostServiceConfig()
 	await generateGrpcClientConfig()
-	await generateHostGrpcClientConfig()
+
+	console.log(chalk.bold.blue("Finished Protocol Buffer code generation."))
+}
+
+async function tsProtoc(outDir, protoFiles, protoOptions) {
+	// Build the protoc command with proper path handling for cross-platform
+	const command = [
+		PROTOC,
+		`--proto_path="${SCRIPT_DIR}"`,
+		`--plugin=protoc-gen-ts_proto="${TS_PROTO_PLUGIN}"`,
+		`--ts_proto_out="${outDir}"`,
+		`--ts_proto_opt=${protoOptions.join(",")} `,
+		...protoFiles.map((s) => `"${s}"`),
+	].join(" ")
+	try {
+		log_verbose(chalk.cyan(`Generating TypeScript code in ${outDir} for:\n${protoFiles.join("\n")}...`))
+		log_verbose(command)
+		execSync(command, { stdio: "inherit" })
+	} catch (error) {
+		console.error(chalk.red("Error generating TypeScript for proto files:"), error)
+		process.exit(1)
+	}
 }
 
 /**
@@ -139,14 +141,14 @@ async function main() {
  * This eliminates the need for manual imports and client creation in grpc-client.ts
  */
 async function generateGrpcClientConfig() {
-	console.log(chalk.cyan("Generating gRPC client configuration..."))
+	log_verbose(chalk.cyan("Generating gRPC client configuration..."))
 
 	const serviceImports = []
 	const serviceClientCreations = []
 	const serviceExports = []
 
 	// Process each service in the serviceNameMap
-	for (const [dirName, fullServiceName] of Object.entries(serviceNameMap)) {
+	for (const [dirName, _fullServiceName] of Object.entries(serviceNameMap)) {
 		const capitalizedName = dirName.charAt(0).toUpperCase() + dirName.slice(1)
 
 		// Add import statement
@@ -174,9 +176,9 @@ export {
 	${serviceExports.join(",\n\t")}
 }`
 
-	const configPath = path.join(ROOT_DIR, "webview-ui", "src", "services", "grpc-client.ts")
-	await fs.writeFile(configPath, content)
-	console.log(chalk.green(`Generated gRPC client at ${configPath}`))
+	const filePath = path.join(ROOT_DIR, "webview-ui/src/services/grpc-client.ts")
+	await writeFileWithMkdirs(filePath, content)
+	log_verbose(chalk.green(`Generated gRPC client at ${filePath}`))
 }
 
 /**
@@ -186,7 +188,7 @@ export {
  * @returns Map of service names to their streaming methods
  */
 async function parseProtoForStreamingMethods(protoFiles, scriptDir) {
-	console.log(chalk.cyan("Parsing proto files for streaming methods..."))
+	log_verbose(chalk.cyan("Parsing proto files for streaming methods..."))
 
 	// Map of service name to array of streaming method names
 	const streamingMethodsMap = new Map()
@@ -238,28 +240,18 @@ async function parseProtoForStreamingMethods(protoFiles, scriptDir) {
 }
 
 async function generateMethodRegistrations() {
-	console.log(chalk.cyan("Generating method registration files..."))
+	log_verbose(chalk.cyan("Generating method registration files..."))
 
 	// Parse proto files for streaming methods
 	const protoFiles = await globby("*.proto", { cwd: SCRIPT_DIR })
 	const streamingMethodsMap = await parseProtoForStreamingMethods(protoFiles, SCRIPT_DIR)
 
 	for (const serviceDir of serviceDirs) {
-		try {
-			await fs.access(serviceDir)
-		} catch (error) {
-			console.log(chalk.cyan(`Creating directory ${serviceDir} for new service`))
-			await fs.mkdir(serviceDir, { recursive: true })
-		}
-
 		const serviceName = path.basename(serviceDir)
-		const registryFile = path.join(serviceDir, "methods.ts")
-		const indexFile = path.join(serviceDir, "index.ts")
-
 		const fullServiceName = serviceNameMap[serviceName]
 		const streamingMethods = streamingMethodsMap.get(fullServiceName) || []
 
-		console.log(chalk.cyan(`Generating method registrations for ${serviceName}...`))
+		log_verbose(chalk.cyan(`Generating method registrations for ${serviceName}...`))
 
 		// Get all TypeScript files in the service directory
 		const files = await globby("*.ts", { cwd: serviceDir })
@@ -311,8 +303,9 @@ export function registerAllMethods(): void {
 		methodsContent += `}`
 
 		// Write the methods.ts file
-		await fs.writeFile(registryFile, methodsContent)
-		console.log(chalk.green(`Generated ${registryFile}`))
+		const registryFile = path.join(serviceDir, "methods.ts")
+		await writeFileWithMkdirs(registryFile, methodsContent)
+		log_verbose(chalk.green(`Generated ${registryFile}`))
 
 		// Generate index.ts file
 		const capitalizedServiceName = serviceName.charAt(0).toUpperCase() + serviceName.slice(1)
@@ -340,11 +333,12 @@ export const isStreamingMethod = ${serviceName}Service.isStreamingMethod
 registerAllMethods()`
 
 		// Write the index.ts file
-		await fs.writeFile(indexFile, indexContent)
-		console.log(chalk.green(`Generated ${indexFile}`))
+		const indexFile = path.join(serviceDir, "index.ts")
+		await writeFileWithMkdirs(indexFile, indexContent)
+		log_verbose(chalk.green(`Generated ${indexFile}`))
 	}
 
-	console.log(chalk.green("Method registration files generated successfully."))
+	log_verbose(chalk.green("Method registration files generated successfully."))
 }
 
 /**
@@ -352,7 +346,7 @@ registerAllMethods()`
  * This eliminates the need for manual switch/case statements in grpc-handler.ts
  */
 async function generateServiceConfig() {
-	console.log(chalk.cyan("Generating service configuration file..."))
+	log_verbose(chalk.cyan("Generating service configuration file..."))
 
 	const serviceImports = []
 	const serviceConfigs = []
@@ -391,9 +385,9 @@ export interface ServiceHandlerConfig {
 export const serviceHandlers: Record<string, ServiceHandlerConfig> = {${serviceConfigs.join(",")}
 };`
 
-	const configPath = path.join(ROOT_DIR, "src", "core", "controller", "grpc-service-config.ts")
-	await fs.writeFile(configPath, content)
-	console.log(chalk.green(`Generated service configuration at ${configPath}`))
+	const configPath = path.join(ROOT_DIR, "src/core/controller/grpc-service-config.ts")
+	await writeFileWithMkdirs(configPath, content)
+	log_verbose(chalk.green(`Generated service configuration at ${configPath}`))
 }
 
 /**
@@ -401,7 +395,7 @@ export const serviceHandlers: Record<string, ServiceHandlerConfig> = {${serviceC
  * If a .proto file doesn't exist, create a template file
  */
 async function ensureProtoFilesExist() {
-	console.log(chalk.cyan("Checking for missing proto files..."))
+	log_verbose(chalk.cyan("Checking for missing proto files..."))
 
 	// Get existing proto files
 	const existingProtoFiles = await globby("*.proto", { cwd: SCRIPT_DIR })
@@ -410,7 +404,7 @@ async function ensureProtoFilesExist() {
 	// Check each service in serviceNameMap
 	for (const [serviceName, fullServiceName] of Object.entries(serviceNameMap)) {
 		if (!existingProtoServices.includes(serviceName)) {
-			console.log(chalk.yellow(`Creating template proto file for ${serviceName}...`))
+			log_verbose(chalk.yellow(`Creating template proto file for ${serviceName}...`))
 
 			// Extract service class name from full name (e.g., "cline.ModelsService" -> "ModelsService")
 			const serviceClassName = fullServiceName.split(".").pop()
@@ -443,7 +437,7 @@ service ${serviceClassName} {
 			// Write the template proto file
 			const protoFilePath = path.join(SCRIPT_DIR, `${serviceName}.proto`)
 			await fs.writeFile(protoFilePath, protoContent)
-			console.log(chalk.green(`Created template proto file at ${protoFilePath}`))
+			log_verbose(chalk.green(`Created template proto file at ${protoFilePath}`))
 		}
 	}
 }
@@ -452,28 +446,18 @@ service ${serviceClassName} {
  * Generate method registration files for host services
  */
 async function generateHostMethodRegistrations() {
-	console.log(chalk.cyan("Generating host method registration files..."))
+	log_verbose(chalk.cyan("Generating host method registration files..."))
 
 	// Parse proto files for streaming methods
 	const hostProtoFiles = await globby("*.proto", { cwd: path.join(SCRIPT_DIR, "host") })
 	const streamingMethodsMap = await parseProtoForStreamingMethods(hostProtoFiles, path.join(SCRIPT_DIR, "host"))
 
 	for (const serviceDir of hostServiceDirs) {
-		try {
-			await fs.access(serviceDir)
-		} catch (error) {
-			console.log(chalk.cyan(`Creating directory ${serviceDir} for new host service`))
-			await fs.mkdir(serviceDir, { recursive: true })
-		}
-
 		const serviceName = path.basename(serviceDir)
-		const registryFile = path.join(serviceDir, "methods.ts")
-		const indexFile = path.join(serviceDir, "index.ts")
-
 		const fullServiceName = hostServiceNameMap[serviceName]
 		const streamingMethods = streamingMethodsMap.get(fullServiceName) || []
 
-		console.log(chalk.cyan(`Generating method registrations for host ${serviceName}...`))
+		log_verbose(chalk.cyan(`Generating method registrations for host ${serviceName}...`))
 
 		// Get all TypeScript files in the service directory
 		const files = await globby("*.ts", { cwd: serviceDir })
@@ -525,8 +509,9 @@ export function registerAllMethods(): void {
 		methodsContent += `}`
 
 		// Write the methods.ts file
-		await fs.writeFile(registryFile, methodsContent)
-		console.log(chalk.green(`Generated ${registryFile}`))
+		const registryFile = path.join(serviceDir, "methods.ts")
+		await writeFileWithMkdirs(registryFile, methodsContent)
+		log_verbose(chalk.green(`Generated ${registryFile}`))
 
 		// Generate index.ts file
 		const capitalizedServiceName = serviceName.charAt(0).toUpperCase() + serviceName.slice(1)
@@ -554,18 +539,19 @@ export const isStreamingMethod = ${serviceName}Service.isStreamingMethod
 registerAllMethods()`
 
 		// Write the index.ts file
-		await fs.writeFile(indexFile, indexContent)
-		console.log(chalk.green(`Generated ${indexFile}`))
+		const indexFile = path.join(serviceDir, "index.ts")
+		await writeFileWithMkdirs(indexFile, indexContent)
+		log_verbose(chalk.green(`Generated ${indexFile}`))
 	}
 
-	console.log(chalk.green("Host method registration files generated successfully."))
+	log_verbose(chalk.green("Host method registration files generated successfully."))
 }
 
 /**
  * Generate a service configuration file for host services
  */
 async function generateHostServiceConfig() {
-	console.log(chalk.cyan("Generating host service configuration file..."))
+	log_verbose(chalk.cyan("Generating host service configuration file..."))
 
 	const serviceImports = []
 	const serviceConfigs = []
@@ -603,55 +589,57 @@ export interface HostServiceHandlerConfig {
 export const hostServiceHandlers: Record<string, HostServiceHandlerConfig> = {${serviceConfigs.join(",")}
 };`
 
-	const configPath = path.join(ROOT_DIR, "src", "hosts", "vscode", "host-grpc-service-config.ts")
-	await fs.mkdir(path.dirname(configPath), { recursive: true })
-	await fs.writeFile(configPath, content)
-	console.log(chalk.green(`Generated host service configuration at ${configPath}`))
+	const filePath = path.join(ROOT_DIR, "src/hosts/vscode/host-grpc-service-config.ts")
+	await writeFileWithMkdirs(filePath, content)
+	log_verbose(chalk.green(`Generated host service configuration at ${filePath}`))
+}
+
+async function cleanup() {
+	// Clean up existing generated files
+	log_verbose(chalk.cyan("Cleaning up existing generated TypeScript files..."))
+	const existingFiles = await globby("**/*.ts", { cwd: TS_OUT_DIR })
+	for (const file of existingFiles) {
+		await fs.unlink(path.join(TS_OUT_DIR, file))
+	}
+	await rmdir(path.join(ROOT_DIR, "src/generated"))
+
+	// Clean up generated files that were moved.
+	await fs.rm(path.join(ROOT_DIR, "src/standalone/services/host-grpc-client.ts"), { force: true })
+	await rmdir(path.join(ROOT_DIR, "src/standalone/services"))
+
+	await fs.rm(path.join(ROOT_DIR, "hosts/vscode"), { force: true, recursive: true })
+	await rmdir(path.join(ROOT_DIR, "hosts"))
+
+	await fs.rm(path.join(ROOT_DIR, "src/standalone/server-setup.ts"), { force: true })
 }
 
 /**
- * Generate a gRPC client configuration file for host services
+ * Write `contents` to `filePath`, creating any necessary directories in `filePath`.
  */
-async function generateHostGrpcClientConfig() {
-	console.log(chalk.cyan("Generating host gRPC client configuration..."))
+async function writeFileWithMkdirs(filePath, content) {
+	await fs.mkdir(path.dirname(filePath), { recursive: true })
+	await fs.writeFile(filePath, content)
+}
 
-	const serviceImports = []
-	const serviceClientCreations = []
-	const serviceExports = []
-
-	// Process each service in the hostServiceNameMap
-	for (const [dirName, fullServiceName] of Object.entries(hostServiceNameMap)) {
-		const capitalizedName = dirName.charAt(0).toUpperCase() + dirName.slice(1)
-
-		// Add import statement
-		serviceImports.push(`import { ${capitalizedName}ServiceDefinition } from "@shared/proto/host/${dirName}"`)
-
-		// Add client creation
-		serviceClientCreations.push(
-			`const ${capitalizedName}ServiceClient = createGrpcClient(${capitalizedName}ServiceDefinition)`,
-		)
-
-		// Add to exports
-		serviceExports.push(`${capitalizedName}ServiceClient`)
+/**
+ * Remove an empty dir, do nothing if the directory doesn't exist or is not empty.
+ */
+async function rmdir(path) {
+	try {
+		await fs.rmdir(path)
+	} catch (error) {
+		if (error.code !== "ENOTEMPTY" && error.code !== "ENOENT") {
+			// Only re-throw if it's not "not empty" or "doesn't exist"
+			throw error
+		}
 	}
+}
 
-	// Generate the file content
-	const content = `// AUTO-GENERATED FILE - DO NOT MODIFY DIRECTLY
-// Generated by proto/build-proto.js
-
-import { createGrpcClient } from "./host-grpc-client-base"
-${serviceImports.join("\n")}
-
-${serviceClientCreations.join("\n")}
-
-export {
-	${serviceExports.join(",\n\t")}
-}`
-
-	const configPath = path.join(ROOT_DIR, "src", "hosts", "vscode", "client", "host-grpc-client.ts")
-	await fs.mkdir(path.dirname(configPath), { recursive: true })
-	await fs.writeFile(configPath, content)
-	console.log(chalk.green(`Generated host gRPC client at ${configPath}`))
+function serviceNameWithoutPackage(fullServiceName) {
+	return fullServiceName.replace(/.*\./, "")
+}
+function lowercaseFirstChar(str) {
+	return str.charAt(0).toLowerCase() + str.slice(1)
 }
 
 // Check for Apple Silicon compatibility
@@ -684,6 +672,12 @@ function checkAppleSiliconCompatibility() {
 	}
 }
 
+function log_verbose(s) {
+	if (process.argv.includes("-v") || process.argv.includes("--verbose")) {
+		console.log(s)
+	}
+}
+
 // Run the main function
 main().catch((error) => {
 	console.error(chalk.red("Error:"), error)

proto/host/uri.proto

@@ -25,12 +25,12 @@ message Uri {
   string path = 3;
   string query = 4;
   string fragment = 5;
-  string fsPath = 6;
+  string fs_path = 6;
 }
 
 // Request for joining path segments to a URI
 message JoinPathRequest {
   cline.Metadata metadata = 1;
   Uri base = 2;
-  repeated string pathSegments = 3;
+  repeated string path_segments = 3;
 }

proto/models.proto

@@ -22,19 +22,21 @@ service ModelsService {
   rpc refreshRequestyModels(EmptyRequest) returns (OpenRouterCompatibleModelInfo);
   // Subscribe to OpenRouter models updates
   rpc subscribeToOpenRouterModels(EmptyRequest) returns (stream OpenRouterCompatibleModelInfo);
+  // Updates API configuration
+  rpc updateApiConfigurationProto(UpdateApiConfigurationRequest) returns (Empty);
 }
 
 // List of VS Code LM models
 message VsCodeLmModelsArray {
-  repeated VsCodeLmModel models = 1;
+  repeated LanguageModelChatSelector models = 1;
 }
 
-// Structure representing a VS Code LM model
-message VsCodeLmModel {
-  string vendor = 1;
-  string family = 2;
-  string version = 3;
-  string id = 4;
+// Structure representing a language model chat selector
+message LanguageModelChatSelector {
+  optional string vendor = 1;
+  optional string family = 2;
+  optional string version = 3;
+  optional string id = 4;
 }
 
 // Price tier for tiered pricing models
@@ -83,6 +85,155 @@ message OpenRouterCompatibleModelInfo {
 // Request for fetching OpenAI models
 message OpenAiModelsRequest {
   Metadata metadata = 1;
-  string baseUrl = 2;
-  string apiKey = 3;
+  string base_url = 2;
+  string api_key = 3;
 }
+
+// Request for updating API configuration
+message UpdateApiConfigurationRequest {
+  Metadata metadata = 1;
+  ModelsApiConfiguration api_configuration = 2;
+}
+
+// API Provider enumeration
+enum ApiProvider {
+  ANTHROPIC = 0;
+  OPENROUTER = 1;
+  BEDROCK = 2;
+  VERTEX = 3;
+  OPENAI = 4;
+  OLLAMA = 5;
+  LMSTUDIO = 6;
+  GEMINI = 7;
+  OPENAI_NATIVE = 8;
+  REQUESTY = 9;
+  TOGETHER = 10;
+  DEEPSEEK = 11;
+  QWEN = 12;
+  DOUBAO = 13;
+  MISTRAL = 14;
+  VSCODE_LM = 15;
+  CLINE = 16;
+  LITELLM = 17;
+  NEBIUS = 18;
+  FIREWORKS = 19;
+  ASKSAGE = 20;
+  XAI = 21;
+  SAMBANOVA = 22;
+  CEREBRAS = 23;
+  SAPAICORE = 24;
+  CLAUDE_CODE = 25;
+}
+
+// Model info for OpenAI-compatible models
+message OpenAiCompatibleModelInfo {
+  optional int32 max_tokens = 1;
+  optional int32 context_window = 2;
+  optional bool supports_images = 3;
+  bool supports_prompt_cache = 4;
+  optional double input_price = 5;
+  optional double output_price = 6;
+  optional ThinkingConfig thinking_config = 7;
+  optional bool supports_global_endpoint = 8;
+  optional double cache_writes_price = 9;
+  optional double cache_reads_price = 10;
+  optional string description = 11;
+  repeated ModelTier tiers = 12;
+  optional double temperature = 13;
+  optional bool is_r1_format_required = 14;
+}
+
+// Model info for LiteLLM models
+message LiteLLMModelInfo {
+  optional int32 max_tokens = 1;
+  optional int32 context_window = 2;
+  optional bool supports_images = 3;
+  bool supports_prompt_cache = 4;
+  optional double input_price = 5;
+  optional double output_price = 6;
+  optional ThinkingConfig thinking_config = 7;
+  optional bool supports_global_endpoint = 8;
+  optional double cache_writes_price = 9;
+  optional double cache_reads_price = 10;
+  optional string description = 11;
+  repeated ModelTier tiers = 12;
+  optional double temperature = 13;
+}
+
+// Main ApiConfiguration message
+message ModelsApiConfiguration {
+  // From ApiHandlerOptions (excluding onRetryAttempt function)
+  optional string api_model_id = 1;
+  optional string api_key = 2;
+  optional string cline_api_key = 3;
+  optional string task_id = 4;
+  optional string lite_llm_base_url = 5;
+  optional string lite_llm_model_id = 6;
+  optional string lite_llm_api_key = 7;
+  optional bool lite_llm_use_prompt_cache = 8;
+  map<string, string> open_ai_headers = 9;
+  optional LiteLLMModelInfo lite_llm_model_info = 10;
+  optional string anthropic_base_url = 11;
+  optional string open_router_api_key = 12;
+  optional string open_router_model_id = 13;
+  optional OpenRouterModelInfo open_router_model_info = 14;
+  optional string open_router_provider_sorting = 15;
+  optional string aws_access_key = 16;
+  optional string aws_secret_key = 17;
+  optional string aws_session_token = 18;
+  optional string aws_region = 19;
+  optional bool aws_use_cross_region_inference = 20;
+  optional bool aws_bedrock_use_prompt_cache = 21;
+  optional bool aws_use_profile = 22;
+  optional string aws_profile = 23;
+  optional string aws_bedrock_endpoint = 24;
+  optional bool aws_bedrock_custom_selected = 25;
+  optional string aws_bedrock_custom_model_base_id = 26;
+  optional string vertex_project_id = 27;
+  optional string vertex_region = 28;
+  optional string open_ai_base_url = 29;
+  optional string open_ai_api_key = 30;
+  optional string open_ai_model_id = 31;
+  optional OpenAiCompatibleModelInfo open_ai_model_info = 32;
+  optional string ollama_model_id = 33;
+  optional string ollama_base_url = 34;
+  optional string ollama_api_options_ctx_num = 35;
+  optional string lm_studio_model_id = 36;
+  optional string lm_studio_base_url = 37;
+  optional string gemini_api_key = 38;
+  optional string gemini_base_url = 39;
+  optional string open_ai_native_api_key = 40;
+  optional string deep_seek_api_key = 41;
+  optional string requesty_api_key = 42;
+  optional string requesty_model_id = 43;
+  optional OpenRouterModelInfo requesty_model_info = 44;
+  optional string together_api_key = 45;
+  optional string together_model_id = 46;
+  optional string fireworks_api_key = 47;
+  optional string fireworks_model_id = 48;
+  optional int32 fireworks_model_max_completion_tokens = 49;
+  optional int32 fireworks_model_max_tokens = 50;
+  optional string qwen_api_key = 51;
+  optional string doubao_api_key = 52;
+  optional string mistral_api_key = 53;
+  optional string azure_api_version = 54;
+  optional LanguageModelChatSelector vs_code_lm_model_selector = 55;
+  optional string qwen_api_line = 56;
+  optional string nebius_api_key = 57;
+  optional string asksage_api_url = 58;
+  optional string asksage_api_key = 59;
+  optional string xai_api_key = 60;
+  optional int32 thinking_budget_tokens = 61;
+  optional string reasoning_effort = 62;
+  optional string sambanova_api_key = 63;
+  optional string cerebras_api_key = 64;
+  optional int32 request_timeout_ms = 65;
+  optional ApiProvider api_provider = 66;
+  repeated string favorited_model_ids = 67;
+  optional string sap_ai_core_client_id = 68;
+  optional string sap_ai_core_client_secret = 69;
+  optional string sap_ai_resource_group = 70;
+  optional string sap_ai_core_token_url = 71;
+  optional string sap_ai_core_base_url = 72;
+  optional string claude_code_path = 73;
+}

proto/state.proto

@@ -7,11 +7,14 @@ import "common.proto";
 
 service StateService {
   rpc getLatestState(EmptyRequest) returns (State);
+  rpc updateTerminalConnectionTimeout(Int64Request) returns (Int64);
+  rpc updateTerminalReuseEnabled(BooleanRequest) returns (Empty);
+  rpc updateDefaultTerminalProfile(StringRequest) returns (TerminalProfileUpdateResponse);
+  rpc getAvailableTerminalProfiles(EmptyRequest) returns (TerminalProfiles);
   rpc subscribeToState(EmptyRequest) returns (stream State);
   rpc toggleFavoriteModel(StringRequest) returns (Empty);
-  rpc resetState(EmptyRequest) returns (Empty);
-  rpc togglePlanActMode(TogglePlanActModeRequest) returns (Empty);
-  rpc updateTerminalConnectionTimeout(Int64Request) returns (Int64);
+  rpc resetState(ResetStateRequest) returns (Empty);
+  rpc togglePlanActMode(TogglePlanActModeRequest) returns (Boolean);
   rpc updateAutoApprovalSettings(AutoApprovalSettingsRequest) returns (Empty);
   rpc updateSettings(UpdateSettingsRequest) returns (Empty);
 }
@@ -20,6 +23,23 @@ message State {
   string state_json = 1;
 }
 
+message TerminalProfiles {
+  repeated TerminalProfile profiles = 1;
+}
+
+message TerminalProfile {
+  string id = 1;
+  string name = 2;
+  optional string path = 3;
+  optional string description = 4;
+}
+
+message TerminalProfileUpdateResponse {
+  int32 closed_count = 1;
+  int32 busy_terminals_count = 2;
+  bool has_busy_terminals = 3;
+}
+
 message TogglePlanActModeRequest {
   Metadata metadata = 1;
   ChatSettings chat_settings = 2;
@@ -43,10 +63,13 @@ message ChatContent {
   repeated string files = 3;
 }
 
-// Message for auto approval settings
+message ResetStateRequest {
+  Metadata metadata = 1;
+  optional bool global = 2;
+}
+
 message AutoApprovalSettingsRequest {
   Metadata metadata = 1;
-
   message Actions {
     bool read_files = 1;
     bool read_files_externally = 2;
@@ -57,7 +80,6 @@ message AutoApprovalSettingsRequest {
     bool use_browser = 7;
     bool use_mcp = 8;
   }
-
   int32 version = 2;
   bool enabled = 3;
   Actions actions = 4;
@@ -70,15 +92,16 @@ message AutoApprovalSettingsRequest {
 message UpdateSettingsRequest {
   Metadata metadata = 1;
   optional ApiConfiguration api_configuration = 2;
-  optional string custom_instructions_setting = 3;
-  optional string telemetry_setting = 4;
-  optional bool plan_act_separate_models_setting = 5;
-  optional bool enable_checkpoints_setting = 6;
-  optional bool mcp_marketplace_enabled = 7;
-  optional ChatSettings chat_settings = 8;
-  optional int64 shell_integration_timeout = 9;
-  optional bool terminal_reuse_enabled = 10;
-  optional bool mcp_responses_collapsed = 11;
+  optional string telemetry_setting = 3;
+  optional bool plan_act_separate_models_setting = 4;
+  optional bool enable_checkpoints_setting = 5;
+  optional bool mcp_marketplace_enabled = 6;
+  optional ChatSettings chat_settings = 7;
+  optional int64 shell_integration_timeout = 8;
+  optional bool terminal_reuse_enabled = 9;
+  optional bool mcp_responses_collapsed = 10;
+  optional bool mcp_rich_display_enabled = 11;
+  optional int64 terminal_output_line_limit = 12;
 }
 
 // Complete API Configuration message
@@ -187,4 +210,14 @@ message ApiConfiguration {
   
   // Favorited model IDs
   repeated string favorited_model_ids = 71;
+
+  // SAP AI Core specific
+  optional string sap_ai_core_client_id = 72;
+  optional string sap_ai_core_client_secret = 73;
+  optional string sap_ai_core_base_url = 74;
+  optional string sap_ai_core_token_url = 75;
+  optional string sap_ai_resource_group = 76;
+
+  // Claude Code specific
+  optional string claude_code_path = 77;
 }

proto/ui.proto

@@ -15,7 +15,7 @@ enum WebviewProviderType {
 // Define a new message type for webview provider info
 message WebviewProviderTypeRequest {
   Metadata metadata = 1;
-  WebviewProviderType providerType = 2;
+  WebviewProviderType provider_type = 2;
 }
 
 // Enum for ClineMessage type
@@ -65,13 +65,14 @@ enum ClineSay {
   BROWSER_ACTION_RESULT = 16;
   MCP_SERVER_REQUEST_STARTED = 17;
   MCP_SERVER_RESPONSE = 18;
-  USE_MCP_SERVER_SAY = 19;
-  DIFF_ERROR = 20;
-  DELETED_API_REQS = 21;
-  CLINEIGNORE_ERROR = 22;
-  CHECKPOINT_CREATED = 23;
-  LOAD_MCP_DOCUMENTATION = 24;
-  INFO = 25;
+  MCP_NOTIFICATION = 19;
+  USE_MCP_SERVER_SAY = 20;
+  DIFF_ERROR = 21;
+  DELETED_API_REQS = 22;
+  CLINEIGNORE_ERROR = 23;
+  CHECKPOINT_CREATED = 24;
+  LOAD_MCP_DOCUMENTATION = 25;
+  INFO = 26;
 }
 
 // Enum for ClineSayTool tool types
@@ -255,4 +256,13 @@ service UiService {
   
   // Subscribe to relinquish control events
   rpc subscribeToRelinquishControl(EmptyRequest) returns (stream Empty);
+  
+  // Subscribe to focus chat input events with client ID
+  rpc subscribeToFocusChatInput(StringRequest) returns (stream Empty);
+  
+  // Subscribe to webview visibility change events
+  rpc subscribeToDidBecomeVisible(EmptyRequest) returns (stream Empty);
+
+  // Returns the HTML for the webview index page. This is only used by external clients, not by the vscode webview.
+  rpc getWebviewHtml(EmptyRequest) returns (String);
 }

scripts/build-tests.js

@@ -34,7 +34,6 @@ const srcConfig = {
 	format: "cjs",
 	platform: "node",
 	define: {
-		"process.env.IS_DEV": "true",
 		"process.env.IS_TEST": "true",
 	},
 	external: ["vscode"],

scripts/generate-host-bridge-client.mjs

@@ -0,0 +1,192 @@
+#!/usr/bin/env node
+
+import * as fs from "fs/promises"
+import * as path from "path"
+import * as grpc from "@grpc/grpc-js"
+import * as protoLoader from "@grpc/proto-loader"
+import chalk from "chalk"
+
+const IMPL_FILE = path.resolve("src/generated/standalone/host-bridge-clients.ts")
+const INTERFACE_FILE = path.resolve("src/generated/hosts/host-bridge-client-types.ts")
+const DESCRIPTOR_SET = path.resolve("dist-standalone/proto/descriptor_set.pb")
+
+const typeNameToFQN = new Map()
+
+function addTypeNameToFqn(name, fqn) {
+	if (typeNameToFQN.has(name)) {
+		throw new Error(`Proto type ${name} redefined (${fqn}).`)
+	}
+	typeNameToFQN.set(name, fqn)
+}
+function getFqn(name) {
+	if (!typeNameToFQN.has(name)) {
+		throw Error(`No FQN for ${name}`)
+	}
+	return typeNameToFQN.get(name)
+}
+/**
+ * Main function to generate the host bridge client
+ */
+async function main() {
+	// Load service definitions from descriptor set
+	const descriptorBuffer = await fs.readFile(DESCRIPTOR_SET)
+	const packageDefinition = protoLoader.loadFileDescriptorSetFromBuffer(descriptorBuffer)
+	const proto = grpc.loadPackageDefinition(packageDefinition)
+
+	// Extract host services and proto messages from the proto definition
+	const hostServices = {}
+	for (const [name, def] of Object.entries(proto.host)) {
+		if (def && "service" in def) {
+			hostServices[name] = def
+		} else {
+			addTypeNameToFqn(name, `proto.host.${name}`)
+		}
+	}
+	for (const [name, def] of Object.entries(proto.cline)) {
+		if (def && !("service" in def)) {
+			addTypeNameToFqn(name, `proto.cline.${name}`)
+		}
+	}
+
+	// Generate interfaces file
+	await generateInterfacesFile(hostServices)
+
+	// // Generate implementation file
+	await generateImplementationFile(hostServices)
+
+	console.log(`Generated host bridge client files at:`)
+	console.log(`- ${INTERFACE_FILE}`)
+	console.log(`- ${IMPL_FILE}`)
+}
+
+/**
+ * Generate the client interfaces file.
+ */
+async function generateInterfacesFile(hostServices) {
+	const clientInterfaces = []
+	for (const [name, def] of Object.entries(hostServices)) {
+		const clientInterface = generateClientInterface(name, def)
+		clientInterfaces.push(clientInterface)
+	}
+	const content = `// GENERATED CODE -- DO NOT EDIT!
+// Generated by scripts/generate-host-bridge-client.mjs
+import * as proto from "@shared/proto/index"
+import { StreamingCallbacks } from "@hosts/host-provider-types"
+
+${clientInterfaces.join("\n\n")}
+`
+	// Write output file
+	await fs.mkdir(path.dirname(INTERFACE_FILE), { recursive: true })
+	await fs.writeFile(INTERFACE_FILE, content)
+}
+
+/**
+ * Generate a client interface for a service.
+ */
+function generateClientInterface(serviceName, serviceDefinition) {
+	// Get the methods from the service definition
+	const methods = Object.entries(serviceDefinition.service)
+		.map(([methodName, methodDef]) => {
+			const requestType = getFqn(methodDef.requestType.type.name)
+			const responseType = getFqn(methodDef.responseType.type.name)
+
+			if (!methodDef.responseStream) {
+				// Generate unary method signature.
+				return `	${methodName}(request: ${requestType}): Promise<${responseType}>;`
+			}
+			// Generate streaming method signature.
+			return `	${methodName}(request: ${requestType}, callbacks: StreamingCallbacks<${responseType}>): () => void;`
+		})
+		.join("\n\n")
+
+	// Generate the interface
+	return `/**
+ * Interface for ${serviceName} client.
+ */
+export interface ${serviceName}ClientInterface {
+
+${methods}
+}`
+}
+
+/**
+ * Generate the client implementations file.
+ */
+async function generateImplementationFile(hostServices) {
+	// Generate imports
+	const imports = []
+	// Add imports for the interfaces
+	for (const [name, _def] of Object.entries(hostServices)) {
+		imports.push(`import { ${name}ClientInterface } from "@generated/hosts/host-bridge-client-types"`)
+	}
+	const clientImplementations = []
+	for (const [name, def] of Object.entries(hostServices)) {
+		clientImplementations.push(generateClientImplementation(name, def))
+	}
+
+	const content = `// GENERATED CODE -- DO NOT EDIT!
+// Generated by scripts/generate-host-bridge-client.mjs
+import { asyncIteratorToCallbacks } from "@/standalone/utils"
+import * as niceGrpc from "@generated/nice-grpc/index"
+import { StreamingCallbacks } from "@hosts/host-provider-types"
+import * as proto from "@shared/proto/index"
+import { Channel, createClient } from "nice-grpc"
+
+${imports.join("\n")}
+
+${clientImplementations.join("\n\n")}
+`
+
+	// Write output file
+	await fs.mkdir(path.dirname(IMPL_FILE), { recursive: true })
+	await fs.writeFile(IMPL_FILE, content)
+}
+
+/**
+ * Generate a client implementation class for a service
+ */
+function generateClientImplementation(serviceName, serviceDefinition) {
+	// Get the methods from the service definition
+	const methods = Object.entries(serviceDefinition.service)
+		.map(([methodName, methodDef]) => {
+			// Get fully qualified type names
+			const requestType = getFqn(methodDef.requestType.type.name)
+			const responseType = getFqn(methodDef.responseType.type.name)
+			const isStreamingResponse = methodDef.responseStream
+
+			if (!isStreamingResponse) {
+				return `  ${methodName}(request: ${requestType}): Promise<${responseType}> {
+    return this.client.${methodName}(request)
+  }`
+			} else {
+				// Generate streaming method
+				return `  ${methodName}(request: ${requestType}, callbacks: StreamingCallbacks<${responseType}>): () => void {
+	const abortController = new AbortController()
+	const stream: AsyncIterable<${responseType}> = this.client.${methodName}(request, {signal: abortController.signal})
+    asyncIteratorToCallbacks(stream, callbacks)
+	return () => {abortController.abort()}
+  }`
+			}
+		})
+		.join("\n\n")
+
+	// Generate the class
+	return `/**
+ * Type-safe client implementation for ${serviceName}.
+ */
+export class ${serviceName}ClientImpl implements ${serviceName}ClientInterface {
+  private client: niceGrpc.host.${serviceName}Client 
+
+  constructor(channel: Channel) {
+    this.client = createClient(niceGrpc.host.${serviceName}Definition, channel)
+  }
+
+${methods}
+}`
+}
+
+// Run the main function
+main().catch((error) => {
+	console.error(chalk.red("Error:"), error)
+	process.exit(1)
+})

scripts/generate-server-setup.mjs

@@ -1,11 +1,11 @@
-import * as fs from "fs"
 import * as grpc from "@grpc/grpc-js"
 import * as protoLoader from "@grpc/proto-loader"
+import * as fs from "fs"
 import * as health from "grpc-health-check"
+import path, { basename, dirname } from "path"
 import { fileURLToPath } from "url"
-import path from "path"
 
-const OUT_FILE = path.resolve("src/standalone/server-setup.ts")
+const OUT_FILE = path.resolve("src/generated/standalone/server-setup.ts")
 const DESCRIPTOR_SET = path.resolve("dist-standalone/proto/descriptor_set.pb")
 
 // Load service definitions.
@@ -29,17 +29,17 @@ function generateHandlersAndExports() {
 		const dir = domain.charAt(0).toLowerCase() + domain.slice(1)
 		imports.push(`// ${domain} Service`)
 		handlerSetup.push(`    // ${domain} Service`)
-		handlerSetup.push(`    server.addService(proto.cline.${name}.service, {`)
+		handlerSetup.push(`    server.addService(cline.${name}Service, {`)
 		for (const [rpcName, rpc] of Object.entries(def.service)) {
-			imports.push(`import { ${rpcName} } from "../core/controller/${dir}/${rpcName}"`)
-			const requestType = "proto.cline." + rpc.requestType.type.name
+			imports.push(`import { ${rpcName} } from "@core/controller/${dir}/${rpcName}"`)
+			const requestType = "cline." + rpc.requestType.type.name
 			if (rpc.requestStream) {
 				throw new Error("Request streaming is not supported")
 			}
 			if (rpc.responseStream) {
 				handlerSetup.push(`        ${rpcName}: wrapStreamingResponse<${requestType},void>(${rpcName}, controller),`)
 			} else {
-				const responseType = "proto.cline." + rpc.responseType.type.name
+				const responseType = "cline." + rpc.responseType.type.name
 				handlerSetup.push(`         ${rpcName}: wrapper<${requestType},${responseType}>(${rpcName}, controller),`)
 			}
 		}
@@ -60,14 +60,13 @@ const scriptName = path.basename(fileURLToPath(import.meta.url))
 let output = `// GENERATED CODE -- DO NOT EDIT!
 // Generated by ${scriptName}
 import * as grpc from "@grpc/grpc-js"
-import * as proto from "@/shared/proto"
-import { Controller } from "../core/controller"
-import { GrpcHandlerWrapper, GrpcStreamingResponseHandlerWrapper } from "./grpc-types"
+import { cline } from "@generated/grpc-js"
+import { Controller } from "@core/controller"
+import { GrpcHandlerWrapper, GrpcStreamingResponseHandlerWrapper } from "@/standalone/grpc-types"
 
 ${imports}
-export function addServices(
+export function addProtobusServices(
 	server: grpc.Server,
-	proto: any,
 	controller: Controller,
 	wrapper: GrpcHandlerWrapper,
 	wrapStreamingResponse: GrpcStreamingResponseHandlerWrapper,
@@ -76,6 +75,7 @@ ${handlerSetup}
 }
 `
 // Write output file
+fs.mkdirSync(dirname(OUT_FILE), { recursive: true })
 fs.writeFileSync(OUT_FILE, output)
 
 console.log(`Generated service handlers in ${OUT_FILE}.`)

scripts/get-vscode-usages.sh

@@ -12,6 +12,7 @@ git grep -h 'vscode\.' $DIR |
 grep -Ev '//.*vscode' | # remove commented out code
 sed 's|.*vscode\.|vscode.|'| # remove everything before vscode.
 sed 's/[^a-zA-Z0-9_.].*$//' | # remove everything after last identifier
+grep -E '\.[a-z][^.]+$' | # remove types (last part of identifier should be lowercase)
 sort | uniq > $SDK_DEST
 }
 echo Wrote uses of the vscode SDK to $(realpath $SDK_DEST)

scripts/package-standalone.mjs

@@ -56,13 +56,24 @@ archive.glob("**/*", {
 
 // Add the whole cline directory under "extension"
 archive.directory(process.cwd(), "extension", (entry) => {
-	// Skip certain directories
-	if (
-		entry.name.startsWith(BUILD_DIR + "/") ||
-		entry.name.startsWith("node_modules/") || // node_modules nearly 1GB.
-		entry.name.startsWith("webview-ui/node_modules/") || // node_modules nearly 1GB.
-		entry.name.match(/(^|\/)\./) // exclude dot directories
-	) {
+	// Skip certain directories.
+	const exclude = [
+		BUILD_DIR + "/",
+		"node_modules/", // node_modules nearly 1GB.
+		"webview-ui/node_modules/", // node_modules nearly 1GB.
+	]
+	// These node modules are used at runtime as assets, they need to be included.
+	const include = ["node_modules/@vscode/", "webview-ui/node_modules/katex"]
+	const name = entry.name
+
+	if (include.some((prefix) => name.startsWith(prefix))) {
+		return entry
+	}
+	if (exclude.some((prefix) => name.startsWith(prefix))) {
+		return false
+	}
+	if (name.match(/(^|\/)\./)) {
+		// exclude dot directories
 		return false
 	}
 	return entry

src/api/index.ts

@@ -25,6 +25,8 @@ import { AskSageHandler } from "./providers/asksage"
 import { XAIHandler } from "./providers/xai"
 import { SambanovaHandler } from "./providers/sambanova"
 import { CerebrasHandler } from "./providers/cerebras"
+import { SapAiCoreHandler } from "./providers/sapaicore"
+import { ClaudeCodeHandler } from "./providers/claude-code"
 
 export interface ApiHandler {
 	createMessage(systemPrompt: string, messages: Anthropic.Messages.MessageParam[]): ApiStream
@@ -87,6 +89,10 @@ export function buildApiHandler(configuration: ApiConfiguration): ApiHandler {
 			return new SambanovaHandler(options)
 		case "cerebras":
 			return new CerebrasHandler(options)
+		case "sapaicore":
+			return new SapAiCoreHandler(options)
+		case "claude-code":
+			return new ClaudeCodeHandler(options)
 		default:
 			return new AnthropicHandler(options)
 	}

src/api/providers/__tests__/bedrock.test.ts

@@ -184,34 +184,35 @@ describe("AwsBedrockHandler", () => {
 		})
 	})
 
+	const mockOptions: ApiHandlerOptions = {
+		apiModelId: "anthropic.claude-3-7-sonnet-20250219-v1:0",
+		awsRegion: "us-east-1",
+		awsAccessKey: "test-key",
+		awsSecretKey: "test-secret",
+		awsSessionToken: "",
+		awsUseProfile: false,
+		awsProfile: "",
+		awsBedrockUsePromptCache: false,
+		awsUseCrossRegionInference: false,
+		awsBedrockEndpoint: "",
+		awsBedrockCustomSelected: false,
+		awsBedrockCustomModelBaseId: undefined,
+		thinkingBudgetTokens: 1600,
+	}
+
+	const mockModelInfo = {
+		maxTokens: 8192,
+		contextWindow: 200000,
+		supportsPromptCache: true,
+		supportsImages: true,
+		inputPrice: 3.0,
+		outputPrice: 15.0,
+		cacheWritesPrice: 3.75,
+		cacheReadsPrice: 0.3,
+	}
+
 	describe("executeConverseStream", () => {
 		let handler: AwsBedrockHandler
-		const mockOptions: ApiHandlerOptions = {
-			apiModelId: "anthropic.claude-3-7-sonnet-20250219-v1:0",
-			awsRegion: "us-east-1",
-			awsAccessKey: "test-key",
-			awsSecretKey: "test-secret",
-			awsSessionToken: "",
-			awsUseProfile: false,
-			awsProfile: "",
-			awsBedrockUsePromptCache: false,
-			awsUseCrossRegionInference: false,
-			awsBedrockEndpoint: "",
-			awsBedrockCustomSelected: false,
-			awsBedrockCustomModelBaseId: undefined,
-			thinkingBudgetTokens: 1600,
-		}
-
-		const mockModelInfo = {
-			maxTokens: 8192,
-			contextWindow: 200000,
-			supportsPromptCache: true,
-			supportsImages: true,
-			inputPrice: 3.0,
-			outputPrice: 15.0,
-			cacheWritesPrice: 3.75,
-			cacheReadsPrice: 0.3,
-		}
 
 		beforeEach(() => {
 			handler = new AwsBedrockHandler(mockOptions)
@@ -591,4 +592,102 @@ describe("AwsBedrockHandler", () => {
 			})
 		})
 	})
+
+	describe("getModelId", () => {
+		it("should return raw model ID for custom models", async () => {
+			const customOptions: ApiHandlerOptions = {
+				...mockOptions,
+				awsBedrockCustomSelected: true,
+				apiModelId:
+					"arn:aws:bedrock:us-west-2:123456789012:custom-model/anthropic.claude-3-5-sonnet-20241022-v2:0/Qk8MMyLmRd",
+			}
+			const customHandler = new AwsBedrockHandler(customOptions)
+
+			const modelId = await customHandler.getModelId()
+			modelId.should.equal(
+				"arn:aws:bedrock:us-west-2:123456789012:custom-model/anthropic.claude-3-5-sonnet-20241022-v2:0/Qk8MMyLmRd",
+			)
+		})
+
+		it("should not encode custom model IDs with slashes", async () => {
+			const customOptions: ApiHandlerOptions = {
+				...mockOptions,
+				awsBedrockCustomSelected: true,
+				apiModelId: "my-namespace/my-custom-model",
+			}
+			const customHandler = new AwsBedrockHandler(customOptions)
+
+			const modelId = await customHandler.getModelId()
+			modelId.should.equal("my-namespace/my-custom-model")
+			modelId.should.not.match(/%2F/)
+		})
+
+		it("should apply cross-region prefix for non-custom models when enabled", async () => {
+			const crossRegionOptions: ApiHandlerOptions = {
+				...mockOptions,
+				awsUseCrossRegionInference: true,
+				awsRegion: "us-west-2",
+			}
+			const crossRegionHandler = new AwsBedrockHandler(crossRegionOptions)
+
+			const modelId = await crossRegionHandler.getModelId()
+			modelId.should.equal("us.anthropic.claude-3-7-sonnet-20250219-v1:0")
+		})
+
+		it("should apply EU cross-region prefix", async () => {
+			const euOptions: ApiHandlerOptions = {
+				...mockOptions,
+				awsUseCrossRegionInference: true,
+				awsRegion: "eu-central-1",
+			}
+			const euHandler = new AwsBedrockHandler(euOptions)
+
+			const modelId = await euHandler.getModelId()
+			modelId.should.equal("eu.anthropic.claude-3-7-sonnet-20250219-v1:0")
+		})
+
+		it("should apply APAC cross-region prefix", async () => {
+			const apacOptions: ApiHandlerOptions = {
+				...mockOptions,
+				awsUseCrossRegionInference: true,
+				awsRegion: "ap-northeast-1",
+			}
+			const apacHandler = new AwsBedrockHandler(apacOptions)
+
+			const modelId = await apacHandler.getModelId()
+			modelId.should.equal("apac.anthropic.claude-3-7-sonnet-20250219-v1:0")
+		})
+
+		it("should not apply cross-region prefix for custom models even when enabled", async () => {
+			const customCrossRegionOptions: ApiHandlerOptions = {
+				...mockOptions,
+				awsBedrockCustomSelected: true,
+				apiModelId: "arn:aws:bedrock:us-west-2:123456789012:custom-model/my-model",
+				awsUseCrossRegionInference: true,
+			}
+			const customCrossRegionHandler = new AwsBedrockHandler(customCrossRegionOptions)
+
+			const modelId = await customCrossRegionHandler.getModelId()
+			modelId.should.equal("arn:aws:bedrock:us-west-2:123456789012:custom-model/my-model")
+		})
+
+		it("should handle UltraThink model ARN correctly", async () => {
+			const ultraThinkOptions: ApiHandlerOptions = {
+				...mockOptions,
+				awsBedrockCustomSelected: true,
+				apiModelId:
+					"arn:aws:bedrock:us-west-2:123456789012:custom-model/anthropic.claude-3-5-sonnet-20241022-v2:0/Qk8MMyLmRd",
+				awsBedrockCustomModelBaseId: "anthropic.claude-3-5-sonnet-20241022-v2:0",
+			}
+			const ultraThinkHandler = new AwsBedrockHandler(ultraThinkOptions)
+
+			const modelId = await ultraThinkHandler.getModelId()
+			// Should return the raw ARN without any encoding
+			modelId.should.equal(
+				"arn:aws:bedrock:us-west-2:123456789012:custom-model/anthropic.claude-3-5-sonnet-20241022-v2:0/Qk8MMyLmRd",
+			)
+			modelId.should.not.match(/%2F/)
+			modelId.should.not.match(/%3A/)
+		})
+	})
 })

src/api/providers/bedrock.ts

@@ -35,6 +35,59 @@ interface ExtendedMetadata {
 	}
 }
 
+// Define types for stream response content blocks
+interface ContentBlockStart {
+	contentBlockIndex?: number
+	start?: {
+		type?: string
+		thinking?: string
+	}
+	contentBlock?: {
+		type?: string
+		thinking?: string
+	}
+	type?: string
+	thinking?: string
+}
+
+// Define types for stream response deltas
+interface ContentBlockDelta {
+	contentBlockIndex?: number
+	delta?: {
+		type?: string
+		thinking?: string
+		text?: string
+		reasoningContent?: {
+			text?: string
+		}
+	}
+}
+
+// Define types for supported content types
+type SupportedContentType = "text" | "image" | "thinking"
+
+interface ContentItem {
+	type: SupportedContentType
+	text?: string
+	source?: {
+		data: string | Buffer | Uint8Array
+		media_type?: string
+	}
+}
+
+// Define cache point type for AWS Bedrock
+interface CachePointContentBlock {
+	cachePoint: {
+		type: "default"
+	}
+}
+
+// Define provider options type based on AWS SDK patterns
+interface ProviderChainOptions {
+	ignoreCache?: boolean
+	profile?: string
+}
+
 // https://docs.anthropic.com/en/api/claude-on-amazon-bedrock
 export class AwsBedrockHandler implements ApiHandler {
 	private options: ApiHandlerOptions
@@ -43,7 +96,7 @@ export class AwsBedrockHandler implements ApiHandler {
 		this.options = options
 	}
 
-	@withRetry()
+	@withRetry({ maxRetries: 4 })
 	async *createMessage(systemPrompt: string, messages: Anthropic.Messages.MessageParam[]): ApiStream {
 		// cross region inference requires prefixing the model id with the region
 		const modelId = await this.getModelId()
@@ -80,11 +133,20 @@ export class AwsBedrockHandler implements ApiHandler {
 
 		const customSelected = this.options.awsBedrockCustomSelected
 		const baseModel = this.options.awsBedrockCustomModelBaseId
-		if (customSelected && modelId && baseModel && baseModel in bedrockModels) {
-			// Use the user-input model ID but inherit capabilities from the base model
+
+		// Handle custom models
+		if (customSelected && modelId) {
+			// If base model is provided and valid, use its capabilities
+			if (baseModel && baseModel in bedrockModels) {
+				return {
+					id: modelId,
+					info: bedrockModels[baseModel],
+				}
+			}
+			// For custom models without valid base model in bedrock model list, use default model's capabilities
 			return {
 				id: modelId,
-				info: bedrockModels[baseModel],
+				info: bedrockModels[bedrockDefaultModelId],
 			}
 		}
 
@@ -107,7 +169,7 @@ export class AwsBedrockHandler implements ApiHandler {
 		sessionToken?: string
 	}> {
 		// Configure provider options
-		const providerOptions: any = {}
+		const providerOptions: ProviderChainOptions = {}
 		if (this.options.awsUseProfile) {
 			// For profile-based auth, always use ignoreCache to detect credential file changes
 			// This solves the AWS Identity Manager issue where credential files change externally
@@ -161,12 +223,9 @@ export class AwsBedrockHandler implements ApiHandler {
 
 	/**
 	 * Gets the appropriate model ID, accounting for cross-region inference if enabled.
-	 * If the model ID is an ARN that contains a slash, you will get the URL encoded ARN.
+	 * For custom models, returns the raw model ID without any encoding.
 	 */
 	async getModelId(): Promise<string> {
-		if (this.options.awsBedrockCustomSelected && this.getModel().id.includes("/")) {
-			return encodeURIComponent(this.getModel().id)
-		}
 		if (!this.options.awsBedrockCustomSelected && this.options.awsUseCrossRegionInference) {
 			const regionPrefix = this.getRegion().slice(0, 3)
 			switch (regionPrefix) {
@@ -461,7 +520,7 @@ export class AwsBedrockHandler implements ApiHandler {
 
 					// Handle content block start - check if Bedrock uses Anthropic SDK format
 					if (chunk.contentBlockStart) {
-						const blockStart = chunk.contentBlockStart as any
+						const blockStart = chunk.contentBlockStart as ContentBlockStart
 						const blockIndex = chunk.contentBlockStart.contentBlockIndex
 
 						// Check for thinking block in various possible formats
@@ -497,7 +556,7 @@ export class AwsBedrockHandler implements ApiHandler {
 
 							// Check if this is a thinking block
 							const blockType = blockTypes.get(blockIndex)
-							const delta = chunk.contentBlockDelta.delta as any
+							const delta = chunk.contentBlockDelta.delta as ContentBlockDelta["delta"]
 
 							// Handle thinking delta (Anthropic SDK format)
 							if (delta?.type === "thinking_delta" || delta?.thinking) {
@@ -727,7 +786,7 @@ export class AwsBedrockHandler implements ApiHandler {
 						}
 
 						// Log unsupported content types for debugging
-						console.warn(`Unsupported content type: ${(item as any).type}`)
+						console.warn(`Unsupported content type: ${(item as ContentItem).type}`)
 						return null
 					})
 					.filter((item): item is ContentBlock => item !== null)
@@ -771,7 +830,7 @@ export class AwsBedrockHandler implements ApiHandler {
 				imageData = new Uint8Array(Buffer.from(base64Data, "base64"))
 			} else if (item.source.data && typeof item.source.data === "object") {
 				// Try to convert to Uint8Array
-				imageData = new Uint8Array(Buffer.from(item.source.data as any))
+				imageData = new Uint8Array(Buffer.from(item.source.data as Buffer | Uint8Array))
 			} else {
 				throw new Error("Unsupported image data format")
 			}
@@ -817,7 +876,7 @@ export class AwsBedrockHandler implements ApiHandler {
 							cachePoint: {
 								type: "default",
 							},
-						} as any, // Type assertion needed for AWS SDK compatibility
+						} as CachePointContentBlock, // Properly typed cache point for AWS SDK
 					]
 				}
 

src/api/providers/claude-code.ts

@@ -0,0 +1,165 @@
+import type { Anthropic } from "@anthropic-ai/sdk"
+import { claudeCodeDefaultModelId, ClaudeCodeModelId, claudeCodeModels, type ApiHandlerOptions } from "@/shared/api"
+import { type ApiHandler } from ".."
+import { ApiStreamUsageChunk, type ApiStream } from "../transform/stream"
+import { withRetry } from "../retry"
+import { runClaudeCode } from "@/integrations/claude-code/run"
+import { ClaudeCodeMessage } from "@/integrations/claude-code/types"
+
+export class ClaudeCodeHandler implements ApiHandler {
+	private options: ApiHandlerOptions
+
+	constructor(options: ApiHandlerOptions) {
+		this.options = options
+	}
+
+	@withRetry({
+		maxRetries: 4,
+		baseDelay: 2000,
+		maxDelay: 15000,
+	})
+	async *createMessage(systemPrompt: string, messages: Anthropic.Messages.MessageParam[]): ApiStream {
+		const claudeProcess = runClaudeCode({
+			systemPrompt,
+			messages,
+			path: this.options.claudeCodePath,
+			modelId: this.getModel().id,
+		})
+
+		const dataQueue: string[] = []
+		let processError = null
+		let errorOutput = ""
+		let exitCode: number | null = null
+
+		claudeProcess.stdout.on("data", (data) => {
+			const output = data.toString()
+			const lines = output.split("\n").filter((line: string) => line.trim() !== "")
+
+			for (const line of lines) {
+				dataQueue.push(line)
+			}
+		})
+
+		claudeProcess.stderr.on("data", (data) => {
+			errorOutput += data.toString()
+		})
+
+		claudeProcess.on("close", (code) => {
+			exitCode = code
+		})
+
+		claudeProcess.on("error", (error) => {
+			processError = error
+		})
+
+		// Usage is included with assistant messages,
+		// but cost is included in the result chunk
+		let usage: ApiStreamUsageChunk = {
+			type: "usage",
+			inputTokens: 0,
+			outputTokens: 0,
+			cacheReadTokens: 0,
+			cacheWriteTokens: 0,
+		}
+
+		while (exitCode !== 0 || dataQueue.length > 0) {
+			if (dataQueue.length === 0) {
+				await new Promise((resolve) => setImmediate(resolve))
+			}
+
+			if (exitCode !== null && exitCode !== 0) {
+				throw new Error(
+					`Claude Code process exited with code ${exitCode}.${errorOutput ? ` Error output: ${errorOutput.trim()}` : ""}`,
+				)
+			}
+
+			const data = dataQueue.shift()
+			if (!data) {
+				continue
+			}
+
+			const chunk = this.attemptParseChunk(data)
+
+			if (!chunk) {
+				yield {
+					type: "text",
+					text: data || "",
+				}
+
+				continue
+			}
+
+			if (chunk.type === "system" && chunk.subtype === "init") {
+				continue
+			}
+
+			if (chunk.type === "assistant" && "message" in chunk) {
+				const message = chunk.message
+
+				if (message.stop_reason !== null && message.stop_reason !== "tool_use") {
+					const errorMessage = message.content[0]?.text || `Claude Code stopped with reason: ${message.stop_reason}`
+
+					if (errorMessage.includes("Invalid model name")) {
+						throw new Error(
+							errorMessage +
+								`\n\nAPI keys and subscription plans allow different models. Make sure the selected model is included in your plan.`,
+						)
+					}
+
+					throw new Error(errorMessage)
+				}
+
+				for (const content of message.content) {
+					if (content.type === "text") {
+						yield {
+							type: "text",
+							text: content.text,
+						}
+					} else {
+						console.warn("Unsupported content type:", content.type)
+					}
+				}
+
+				usage.inputTokens += message.usage.input_tokens
+				usage.outputTokens += message.usage.output_tokens
+				usage.cacheReadTokens = (usage.cacheReadTokens || 0) + (message.usage.cache_read_input_tokens || 0)
+				usage.cacheWriteTokens = (usage.cacheWriteTokens || 0) + (message.usage.cache_creation_input_tokens || 0)
+
+				continue
+			}
+
+			if (chunk.type === "result" && "result" in chunk) {
+				usage.totalCost = chunk.cost_usd || 0
+
+				yield usage
+			}
+
+			if (processError) {
+				throw processError
+			}
+		}
+	}
+
+	getModel() {
+		const modelId = this.options.apiModelId
+		if (modelId && modelId in claudeCodeModels) {
+			const id = modelId as ClaudeCodeModelId
+			return { id, info: claudeCodeModels[id] }
+		}
+
+		return {
+			id: claudeCodeDefaultModelId,
+			info: claudeCodeModels[claudeCodeDefaultModelId],
+		}
+	}
+
+	// TOOD: Validate instead of parsing
+	private attemptParseChunk(data: string): ClaudeCodeMessage | null {
+		try {
+			return JSON.parse(data)
+		} catch (error) {
+			console.error("Error parsing chunk:", error)
+			return null
+		}
+	}
+}

src/api/providers/gemini.ts

@@ -2,6 +2,7 @@ import type { Anthropic } from "@anthropic-ai/sdk"
 // Restore GenerateContentConfig import and add GenerateContentResponseUsageMetadata
 import { GoogleGenAI, type GenerateContentConfig, type GenerateContentResponseUsageMetadata } from "@google/genai"
 import { withRetry } from "../retry"
+import { Part } from "@google/genai"
 import { ApiHandler } from "../"
 import { ApiHandlerOptions, geminiDefaultModelId, GeminiModelId, geminiModels, ModelInfo } from "@shared/api"
 import { convertAnthropicMessageToGemini } from "../transform/gemini-format"
@@ -96,9 +97,10 @@ export class GeminiHandler implements ApiHandler {
 		}
 
 		// Add thinking config if the model supports it
-		if (info.thinkingConfig?.outputPrice !== undefined && maxBudget > 0) {
+		if (thinkingBudget > 0) {
 			requestConfig.thinkingConfig = {
 				thinkingBudget: thinkingBudget,
+				includeThoughts: true,
 			}
 		}
 
@@ -111,6 +113,7 @@ export class GeminiHandler implements ApiHandler {
 		let promptTokens = 0
 		let outputTokens = 0
 		let cacheReadTokens = 0
+		let thoughtsTokenCount = 0 // Initialize thought token counts
 		let lastUsageMetadata: GenerateContentResponseUsageMetadata | undefined
 
 		try {
@@ -130,6 +133,31 @@ export class GeminiHandler implements ApiHandler {
 					isFirstSdkChunk = false
 				}
 
+				// Handle thinking content from Gemini's response
+				const candidateForThoughts = chunk?.candidates?.[0]
+				const partsForThoughts = candidateForThoughts?.content?.parts
+				let thoughts = "" // Initialize as empty string
+
+				if (partsForThoughts) {
+					// This ensures partsForThoughts is a Part[] array
+					for (const part of partsForThoughts) {
+						const { thought, text } = part as Part
+						if (thought && text) {
+							// Ensure part.text exists
+							// Handle the thought part
+							thoughts += text + "\n" // Append thought and a newline
+						}
+					}
+				}
+
+				if (thoughts.trim() !== "") {
+					yield {
+						type: "reasoning",
+						reasoning: thoughts.trim(),
+					}
+					thoughts = "" // Reset thoughts after yielding
+				}
+
 				if (chunk.text) {
 					yield {
 						type: "text",
@@ -141,6 +169,7 @@ export class GeminiHandler implements ApiHandler {
 					lastUsageMetadata = chunk.usageMetadata
 					promptTokens = lastUsageMetadata.promptTokenCount ?? promptTokens
 					outputTokens = lastUsageMetadata.candidatesTokenCount ?? outputTokens
+					thoughtsTokenCount = lastUsageMetadata.thoughtsTokenCount ?? thoughtsTokenCount
 					cacheReadTokens = lastUsageMetadata.cachedContentTokenCount ?? cacheReadTokens
 				}
 			}
@@ -151,12 +180,14 @@ export class GeminiHandler implements ApiHandler {
 					info,
 					inputTokens: promptTokens,
 					outputTokens,
+					thoughtsTokenCount,
 					cacheReadTokens,
 				})
 				yield {
 					type: "usage",
-					inputTokens: promptTokens,
+					inputTokens: promptTokens - cacheReadTokens,
 					outputTokens,
+					thoughtsTokenCount,
 					cacheReadTokens,
 					cacheWriteTokens: 0,
 					totalCost,
@@ -239,11 +270,13 @@ export class GeminiHandler implements ApiHandler {
 		info,
 		inputTokens,
 		outputTokens,
+		thoughtsTokenCount = 0,
 		cacheReadTokens = 0,
 	}: {
 		info: ModelInfo
 		inputTokens: number
 		outputTokens: number
+		thoughtsTokenCount: number
 		cacheReadTokens?: number
 	}) {
 		// Exit early if any required pricing information is missing
@@ -275,18 +308,18 @@ export class GeminiHandler implements ApiHandler {
 		const inputTokensCost = inputPrice * (uncachedInputTokens / 1_000_000)
 
 		// 2. Output token costs
-		const outputTokensCost = outputPrice * (outputTokens / 1_000_000)
+		const responseTokensCost = outputPrice * ((outputTokens + thoughtsTokenCount) / 1_000_000)
 
 		// 3. Cache read costs (immediate)
 		const cacheReadCost = (cacheReadTokens ?? 0) > 0 ? cacheReadsPrice * ((cacheReadTokens ?? 0) / 1_000_000) : 0
 
 		// Calculate total immediate cost (excluding cache write/storage costs)
-		const totalCost = inputTokensCost + outputTokensCost + cacheReadCost
+		const totalCost = inputTokensCost + responseTokensCost + cacheReadCost
 
 		// Create the trace object for debugging
 		const trace: Record<string, { price: number; tokens: number; cost: number }> = {
 			input: { price: inputPrice, tokens: uncachedInputTokens, cost: inputTokensCost },
-			output: { price: outputPrice, tokens: outputTokens, cost: outputTokensCost },
+			output: { price: outputPrice, tokens: outputTokens, cost: responseTokensCost },
 		}
 
 		// Only include cache read costs in the trace (cache write costs are tracked separately)

src/api/providers/sapaicore.ts

@@ -0,0 +1,540 @@
+import { Anthropic } from "@anthropic-ai/sdk"
+import axios from "axios"
+import OpenAI from "openai"
+import { ApiHandler } from "../"
+import { ApiHandlerOptions, ModelInfo, sapAiCoreDefaultModelId, SapAiCoreModelId, sapAiCoreModels } from "../../shared/api"
+import { convertToOpenAiMessages } from "../transform/openai-format"
+import { ApiStream } from "../transform/stream"
+
+interface Deployment {
+	id: string
+	name: string
+}
+interface Token {
+	access_token: string
+	expires_in: number
+	scope: string
+	jti: string
+	token_type: string
+	expires_at: number
+}
+export class SapAiCoreHandler implements ApiHandler {
+	private options: ApiHandlerOptions
+	private token?: Token
+	private deployments?: Deployment[]
+
+	constructor(options: ApiHandlerOptions) {
+		this.options = options
+	}
+
+	private async authenticate(): Promise<Token> {
+		const payload = {
+			grant_type: "client_credentials",
+			client_id: this.options.sapAiCoreClientId || "",
+			client_secret: this.options.sapAiCoreClientSecret || "",
+		}
+
+		const tokenUrl = (this.options.sapAiCoreTokenUrl || "").replace(/\/+$/, "") + "/oauth/token"
+		const response = await axios.post(tokenUrl, payload, {
+			headers: { "Content-Type": "application/x-www-form-urlencoded" },
+		})
+		const token = response.data as Token
+		token.expires_at = Date.now() + token.expires_in * 1000
+		return token
+	}
+
+	private async getToken(): Promise<string> {
+		if (!this.token || this.token.expires_at < Date.now()) {
+			this.token = await this.authenticate()
+		}
+		return this.token.access_token
+	}
+
+	private async getAiCoreDeployments(): Promise<Deployment[]> {
+		if (this.options.sapAiCoreClientSecret === "") {
+			return [{ id: "notconfigured", name: "ai-core-not-configured" }]
+		}
+
+		const token = await this.getToken()
+		const headers = {
+			Authorization: `Bearer ${token}`,
+			"AI-Resource-Group": this.options.sapAiResourceGroup || "default",
+			"Content-Type": "application/json",
+		}
+
+		const url = `${this.options.sapAiCoreBaseUrl}/v2/lm/deployments?$top=10000&$skip=0`
+
+		try {
+			const response = await axios.get(url, { headers })
+			const deployments = response.data.resources
+
+			return deployments
+				.filter((deployment: any) => deployment.targetStatus === "RUNNING")
+				.map((deployment: any) => {
+					const model = deployment.details?.resources?.backend_details?.model
+					if (!model?.name || !model?.version) {
+						return null // Skip this row
+					}
+					return {
+						id: deployment.id,
+						name: `${model.name}:${model.version}`,
+					}
+				})
+				.filter((deployment: any) => deployment !== null)
+		} catch (error) {
+			console.error("Error fetching deployments:", error)
+			throw new Error("Failed to fetch deployments")
+		}
+	}
+
+	private async getDeploymentForModel(modelId: string): Promise<string> {
+		// If deployments are not fetched yet or the model is not found in the fetched deployments, fetch deployments
+		if (!this.deployments || !this.hasDeploymentForModel(modelId)) {
+			this.deployments = await this.getAiCoreDeployments()
+		}
+
+		const deployment = this.deployments.find((d) => {
+			const deploymentBaseName = d.name.split(":")[0].toLowerCase()
+			const modelBaseName = modelId.split(":")[0].toLowerCase()
+			return deploymentBaseName === modelBaseName
+		})
+
+		if (!deployment) {
+			throw new Error(`No running deployment found for model ${modelId}`)
+		}
+
+		return deployment.id
+	}
+
+	private hasDeploymentForModel(modelId: string): boolean {
+		return this.deployments?.some((d) => d.name.split(":")[0].toLowerCase() === modelId.split(":")[0].toLowerCase()) ?? false
+	}
+
+	async *createMessage(systemPrompt: string, messages: Anthropic.Messages.MessageParam[]): ApiStream {
+		const token = await this.getToken()
+		const headers = {
+			Authorization: `Bearer ${token}`,
+			"AI-Resource-Group": this.options.sapAiResourceGroup || "default",
+			"Content-Type": "application/json",
+		}
+
+		const model = this.getModel()
+		const deploymentId = await this.getDeploymentForModel(model.id)
+
+		const anthropicModels = [
+			"anthropic--claude-3.7-sonnet",
+			"anthropic--claude-3.5-sonnet",
+			"anthropic--claude-3-sonnet",
+			"anthropic--claude-3-haiku",
+			"anthropic--claude-3-opus",
+		]
+
+		const openAIModels = ["gpt-4o", "gpt-4", "gpt-4o-mini", "o1", "gpt-4.1", "gpt-4.1-nano", "o3-mini", "o3", "o4-mini"]
+
+		let url: string
+		let payload: any
+		if (anthropicModels.includes(model.id)) {
+			url = `${this.options.sapAiCoreBaseUrl}/v2/inference/deployments/${deploymentId}/invoke-with-response-stream`
+
+			if (model.id === "anthropic--claude-3.7-sonnet") {
+				url = `${this.options.sapAiCoreBaseUrl}/v2/inference/deployments/${deploymentId}/converse-stream`
+				payload = {
+					inferenceConfig: {
+						maxTokens: model.info.maxTokens,
+						temperature: 0.0,
+					},
+					system: systemPrompt ? [{ text: systemPrompt }] : undefined,
+					messages: this.formatAnthropicMessages(messages),
+				}
+			} else {
+				payload = {
+					max_tokens: model.info.maxTokens,
+					system: systemPrompt,
+					messages,
+					anthropic_version: "bedrock-2023-05-31",
+				}
+			}
+		} else if (openAIModels.includes(model.id)) {
+			let openAiMessages: OpenAI.Chat.ChatCompletionMessageParam[] = [
+				{ role: "system", content: systemPrompt },
+				...convertToOpenAiMessages(messages),
+			]
+
+			url = `${this.options.sapAiCoreBaseUrl}/v2/inference/deployments/${deploymentId}/chat/completions?api-version=2024-12-01-preview`
+			payload = {
+				stream: true,
+				messages: openAiMessages,
+				max_tokens: model.info.maxTokens,
+				temperature: 0.0,
+				frequency_penalty: 0,
+				presence_penalty: 0,
+				stop: null,
+				stream_options: { include_usage: true },
+			}
+
+			if (["o1", "o3-mini", "o3", "o4-mini"].includes(model.id)) {
+				delete payload.max_tokens
+				delete payload.temperature
+			}
+
+			if (model.id === "o3-mini") {
+				delete payload.stream
+				delete payload.stream_options
+			}
+		} else {
+			throw new Error(`Unsupported model: ${model.id}`)
+		}
+
+		try {
+			const response = await axios.post(url, JSON.stringify(payload, null, 2), {
+				headers,
+				responseType: "stream",
+			})
+
+			if (model.id === "o3-mini") {
+				const response = await axios.post(url, JSON.stringify(payload, null, 2), { headers })
+
+				// Yield the usage information
+				if (response.data.usage) {
+					yield {
+						type: "usage",
+						inputTokens: response.data.usage.prompt_tokens,
+						outputTokens: response.data.usage.completion_tokens,
+					}
+				}
+
+				// Yield the content
+				if (response.data.choices && response.data.choices.length > 0) {
+					yield {
+						type: "text",
+						text: response.data.choices[0].message.content,
+					}
+				}
+
+				// Final usage yield
+				if (response.data.usage) {
+					yield {
+						type: "usage",
+						inputTokens: response.data.usage.prompt_tokens,
+						outputTokens: response.data.usage.completion_tokens,
+					}
+				}
+			} else if (openAIModels.includes(model.id)) {
+				yield* this.streamCompletionGPT(response.data, model)
+			} else if (model.id === "anthropic--claude-3.7-sonnet") {
+				yield* this.streamCompletionSonnet37(response.data, model)
+			} else {
+				yield* this.streamCompletion(response.data, model)
+			}
+		} catch (error) {
+			if (error.response) {
+				// The request was made and the server responded with a status code
+				// that falls out of the range of 2xx
+				console.error("Error status:", error.response.status)
+				console.error("Error data:", error.response.data)
+				console.error("Error headers:", error.response.headers)
+
+				if (error.response.status === 404) {
+					console.error("404 Error reason:", error.response.data)
+					throw new Error(`404 Not Found: ${error.response.data}`)
+				}
+			} else if (error.request) {
+				// The request was made but no response was received
+				console.error("Error request:", error.request)
+				throw new Error("No response received from server")
+			} else {
+				// Something happened in setting up the request that triggered an Error
+				console.error("Error message:", error.message)
+				throw new Error(`Error setting up request: ${error.message}`)
+			}
+
+			throw new Error("Failed to create message")
+		}
+	}
+
+	private async *streamCompletion(
+		stream: any,
+		model: { id: SapAiCoreModelId; info: ModelInfo },
+	): AsyncGenerator<any, void, unknown> {
+		let usage = { input_tokens: 0, output_tokens: 0 }
+
+		try {
+			for await (const chunk of stream) {
+				const lines = chunk.toString().split("\n").filter(Boolean)
+				for (const line of lines) {
+					if (line.startsWith("data: ")) {
+						const jsonData = line.slice(6)
+						try {
+							const data = JSON.parse(jsonData)
+							console.log("Received data:", data)
+							if (data.type === "message_start") {
+								usage.input_tokens = data.message.usage.input_tokens
+								yield {
+									type: "usage",
+									inputTokens: usage.input_tokens,
+									outputTokens: usage.output_tokens,
+								}
+							} else if (data.type === "content_block_start" || data.type === "content_block_delta") {
+								const contentBlock = data.type === "content_block_start" ? data.content_block : data.delta
+
+								if (contentBlock.type === "text" || contentBlock.type === "text_delta") {
+									yield {
+										type: "text",
+										text: contentBlock.text || "",
+									}
+								}
+							} else if (data.type === "message_delta") {
+								if (data.usage) {
+									usage.output_tokens = data.usage.output_tokens
+									yield {
+										type: "usage",
+										inputTokens: 0,
+										outputTokens: data.usage.output_tokens,
+									}
+								}
+							}
+						} catch (error) {
+							console.error("Failed to parse JSON data:", error)
+						}
+					}
+				}
+			}
+		} catch (error) {
+			console.error("Error streaming completion:", error)
+			throw error
+		}
+	}
+
+	private async *streamCompletionSonnet37(
+		stream: any,
+		model: { id: SapAiCoreModelId; info: ModelInfo },
+	): AsyncGenerator<any, void, unknown> {
+		function toStrictJson(str: string): string {
+			// Wrap it in parentheses so JS will treat it as an expression
+			const obj = new Function("return " + str)()
+			return JSON.stringify(obj)
+		}
+
+		let usage = { input_tokens: 0, output_tokens: 0 }
+
+		try {
+			// Iterate over the stream and process each chunk
+			for await (const chunk of stream) {
+				const lines = chunk.toString().split("\n").filter(Boolean)
+
+				for (const line of lines) {
+					if (line.startsWith("data: ")) {
+						const jsonData = line.slice(6)
+
+						try {
+							// Parse the incoming JSON data from the stream
+							const data = JSON.parse(toStrictJson(jsonData))
+							console.log("Received data:", data)
+
+							// Handle metadata (token usage)
+							if (data.metadata?.usage) {
+								const inputTokens = data.metadata.usage.inputTokens || 0
+								const outputTokens = data.metadata.usage.outputTokens || 0
+
+								yield {
+									type: "usage",
+									inputTokens,
+									outputTokens,
+								}
+							}
+
+							// Handle content block delta (text generation)
+							if (data.contentBlockDelta) {
+								if (data.contentBlockDelta?.delta?.text) {
+									yield {
+										type: "text",
+										text: data.contentBlockDelta.delta.text,
+									}
+								}
+
+								// Handle reasoning content if present
+								if (data.contentBlockDelta?.delta?.reasoningContent?.text) {
+									yield {
+										type: "reasoning",
+										reasoning: data.contentBlockDelta.delta.reasoningContent.text,
+									}
+								}
+							}
+						} catch (error) {
+							console.error("Failed to parse JSON data:", error)
+							yield {
+								type: "text",
+								text: `[ERROR] Failed to parse response data: ${error instanceof Error ? error.message : String(error)}`,
+							}
+						}
+					}
+				}
+			}
+		} catch (error) {
+			console.error("Error streaming completion:", error)
+			yield {
+				type: "text",
+				text: `[ERROR] Failed to process stream: ${error instanceof Error ? error.message : String(error)}`,
+			}
+		}
+	}
+
+	private async *streamCompletionGPT(
+		stream: any,
+		model: { id: SapAiCoreModelId; info: ModelInfo },
+	): AsyncGenerator<any, void, unknown> {
+		let currentContent = ""
+		let inputTokens = 0
+		let outputTokens = 0
+
+		try {
+			for await (const chunk of stream) {
+				const lines = chunk.toString().split("\n").filter(Boolean)
+				for (const line of lines) {
+					if (line.trim() === "data: [DONE]") {
+						// End of stream, yield final usage
+						yield {
+							type: "usage",
+							inputTokens,
+							outputTokens,
+						}
+						return
+					}
+
+					if (line.startsWith("data: ")) {
+						const jsonData = line.slice(6)
+						try {
+							const data = JSON.parse(jsonData)
+							console.log("Received GPT data:", data)
+
+							if (data.choices && data.choices.length > 0) {
+								const choice = data.choices[0]
+								if (choice.delta && choice.delta.content) {
+									yield {
+										type: "text",
+										text: choice.delta.content,
+									}
+									currentContent += choice.delta.content
+								}
+							}
+
+							// Handle usage information
+							if (data.usage) {
+								inputTokens = data.usage.prompt_tokens || inputTokens
+								outputTokens = data.usage.completion_tokens || outputTokens
+								yield {
+									type: "usage",
+									inputTokens,
+									outputTokens,
+								}
+							}
+
+							if (data.choices && data.choices[0].finish_reason === "stop") {
+								// Final usage yield, if not already provided
+								if (!data.usage) {
+									yield {
+										type: "usage",
+										inputTokens,
+										outputTokens,
+									}
+								}
+							}
+						} catch (error) {
+							console.error("Failed to parse GPT JSON data:", error)
+						}
+					}
+				}
+			}
+		} catch (error) {
+			console.error("Error streaming GPT completion:", error)
+			throw error
+		}
+	}
+
+	createUserReadableRequest(
+		userContent: Array<
+			Anthropic.TextBlockParam | Anthropic.ImageBlockParam | Anthropic.ToolUseBlockParam | Anthropic.ToolResultBlockParam
+		>,
+	): any {
+		return {
+			model: this.getModel().id,
+			max_tokens: this.getModel().info.maxTokens,
+			system: "(see SYSTEM_PROMPT in src/ClaudeDev.ts)",
+			messages: [{ conversation_history: "..." }, { role: "user", content: userContent }],
+			tools: "(see tools in src/ClaudeDev.ts)",
+			tool_choice: { type: "auto" },
+		}
+	}
+
+	getModel(): { id: SapAiCoreModelId; info: ModelInfo } {
+		const modelId = this.options.apiModelId
+		if (modelId && modelId in sapAiCoreModels) {
+			const id = modelId as SapAiCoreModelId
+			return { id, info: sapAiCoreModels[id] }
+		}
+		return { id: sapAiCoreDefaultModelId, info: sapAiCoreModels[sapAiCoreDefaultModelId] }
+	}
+
+	private getValidImageFormat(mediaType: string): string {
+		const format = mediaType.split("/")[1]?.toLowerCase()
+		const validFormats = ["png", "jpeg", "gif", "webp"]
+
+		if (validFormats.includes(format)) {
+			return format
+		}
+		throw new Error(`Unsupported image format: ${format}`)
+	}
+
+	private formatAnthropicMessages(messages: Anthropic.Messages.MessageParam[]): any[] {
+		return messages.map((m) => {
+			const contentBlocks: any[] = []
+
+			if (typeof m.content === "string") {
+				contentBlocks.push({ text: m.content })
+			} else if (Array.isArray(m.content)) {
+				for (const block of m.content) {
+					if (block.type === "text") {
+						if (!block.text) {
+							throw new Error('Text block is missing the "text" field.')
+						}
+						contentBlocks.push({ text: block.text })
+					} else if (block.type === "image") {
+						if (!block.source) {
+							throw new Error('Image block is missing the "source" field.')
+						}
+
+						const { type, media_type, data } = block.source
+
+						if (!type || !media_type || !data) {
+							throw new Error('Image source must have "type", "media_type", and "data" fields.')
+						}
+
+						if (type !== "base64") {
+							throw new Error(`Unsupported image source type: ${type}. Only "base64" is supported.`)
+						}
+
+						const format = this.getValidImageFormat(media_type)
+
+						contentBlocks.push({
+							image: {
+								format,
+								source: {
+									bytes: data,
+								},
+							},
+						})
+					} else {
+						throw new Error(`Unsupported content block type: ${block.type}`)
+					}
+				}
+			} else {
+				throw new Error("Unsupported content format.")
+			}
+
+			return {
+				role: m.role,
+				content: contentBlocks,
+			}
+		})
+	}
+}

src/api/retry.ts

@@ -57,7 +57,7 @@ export function withRetry(options: RetryOptions = {}) {
 					const handlerInstance = this as any
 					if (handlerInstance.options?.onRetryAttempt) {
 						try {
-							handlerInstance.options.onRetryAttempt(attempt + 1, maxRetries, delay, error)
+							await handlerInstance.options.onRetryAttempt(attempt + 1, maxRetries, delay, error)
 						} catch (e) {
 							console.error("Error in onRetryAttempt callback:", e)
 						}

src/api/transform/stream.ts

@@ -17,5 +17,6 @@ export interface ApiStreamUsageChunk {
 	outputTokens: number
 	cacheWriteTokens?: number
 	cacheReadTokens?: number
+	thoughtsTokenCount?: number // openrouter
 	totalCost?: number // openrouter
 }

src/core/assistant-message/diff.test.ts

@@ -19,14 +19,34 @@ new content
 			isFinal: true,
 		},
 		{
-			name: "full file replacement",
-			original: "old content",
-			diff: `------- SEARCH
+			name: "malformed search - mixed symbols",
+			original: "line1\nline2\nline3",
+			diff: `<<-- SEARCH
+line2
 =======
-new content
+replaced
 +++++++ REPLACE`,
-			expected: "new content\n",
-			isFinal: true,
+			shouldThrow: true,
+		},
+		{
+			name: "malformed search - insufficient dashes",
+			original: "line1\nline2\nline3",
+			diff: `-- SEARCH
+line2
+=======
+replaced
++++++++ REPLACE`,
+			shouldThrow: true,
+		},
+		{
+			name: "malformed search - missing space",
+			original: "line1\nline2\nline3",
+			diff: `-------SEARCH
+line2
+=======
+replaced
++++++++ REPLACE`,
+			shouldThrow: true,
 		},
 		{
 			name: "exact match replacement",
@@ -139,17 +159,33 @@ replaced
 	]
 	//.filter(({name}) => name === "multiple ordered replacements")
 	//.filter(({name}) => name === "delete then replace")
-	testCases.forEach(({ name, original, diff, expected, isFinal }) => {
+	testCases.forEach(({ name, original, diff, expected, isFinal, shouldThrow }) => {
 		it(`should handle ${name} case correctly`, async () => {
-			const result1 = await cnfc(diff, original, isFinal)
-			const result2 = await cnfc2(diff, original, isFinal)
-			const equal = result1 === result2
-			const equal2 = result1 === expected
-			// Verify both implementations produce same result
-			expect(result1).to.equal(result2)
+			if (shouldThrow) {
+				try {
+					await cnfc(diff, original, isFinal ?? true)
+					expect.fail("Expected an error to be thrown")
+				} catch (err) {
+					expect(err).to.be.an("error")
+				}
 
-			// Verify result matches expected
-			expect(result1).to.equal(expected)
+				try {
+					await cnfc2(diff, original, isFinal ?? true)
+					expect.fail("Expected an error to be thrown")
+				} catch (err) {
+					expect(err).to.be.an("error")
+				}
+			} else {
+				const result1 = await cnfc(diff, original, isFinal ?? true)
+				const result2 = await cnfc2(diff, original, isFinal ?? true)
+				const equal = result1 === result2
+				const equal2 = result1 === expected
+				// Verify both implementations produce same result
+				expect(result1).to.equal(result2)
+
+				// Verify result matches expected
+				expect(result1).to.equal(expected)
+			}
 		})
 	})
 

src/core/assistant-message/diff.ts

@@ -4,15 +4,19 @@ const REPLACE_BLOCK_END = "+++++++ REPLACE"
 
 const SEARCH_BLOCK_CHAR = "-"
 const REPLACE_BLOCK_CHAR = "+"
+const LEGACY_SEARCH_BLOCK_CHAR = "<"
+const LEGACY_REPLACE_BLOCK_CHAR = ">"
 
 // Replace the exact string constants with flexible regex patterns
 const SEARCH_BLOCK_START_REGEX = /^[-]{3,} SEARCH$/
 const SEARCH_BLOCK_END_REGEX = /^[=]{3,}$/
 const REPLACE_BLOCK_END_REGEX = /^[+]{3,} REPLACE$/
+const LEGACY_SEARCH_BLOCK_START_REGEX = /^[<]{3,} SEARCH$/
+const LEGACY_REPLACE_BLOCK_END_REGEX = /^[>]{3,} REPLACE$/
 
 // Helper functions to check if a line matches the flexible patterns
 function isSearchBlockStart(line: string): boolean {
-	return SEARCH_BLOCK_START_REGEX.test(line)
+	return SEARCH_BLOCK_START_REGEX.test(line) || LEGACY_SEARCH_BLOCK_START_REGEX.test(line)
 }
 
 function isSearchBlockEnd(line: string): boolean {
@@ -20,7 +24,7 @@ function isSearchBlockEnd(line: string): boolean {
 }
 
 function isReplaceBlockEnd(line: string): boolean {
-	return REPLACE_BLOCK_END_REGEX.test(line)
+	return REPLACE_BLOCK_END_REGEX.test(line) || LEGACY_REPLACE_BLOCK_END_REGEX.test(line)
 }
 
 /**
@@ -269,7 +273,11 @@ async function constructNewFileContentV1(diffContent: string, originalContent: s
 	const lastLine = lines[lines.length - 1]
 	if (
 		lines.length > 0 &&
-		(lastLine.startsWith(SEARCH_BLOCK_CHAR) || lastLine.startsWith("=") || lastLine.startsWith(REPLACE_BLOCK_CHAR)) &&
+		(lastLine.startsWith(SEARCH_BLOCK_CHAR) ||
+			lastLine.startsWith(LEGACY_SEARCH_BLOCK_CHAR) ||
+			lastLine.startsWith("=") ||
+			lastLine.startsWith(REPLACE_BLOCK_CHAR) ||
+			lastLine.startsWith(LEGACY_REPLACE_BLOCK_CHAR)) &&
 		!isSearchBlockStart(lastLine) &&
 		!isSearchBlockEnd(lastLine) &&
 		!isReplaceBlockEnd(lastLine)
@@ -303,9 +311,12 @@ async function constructNewFileContentV1(diffContent: string, originalContent: s
 					searchMatchIndex = 0
 					searchEndIndex = 0
 				} else {
-					// Complete file replacement scenario: treat the entire file as matched
-					searchMatchIndex = 0
-					searchEndIndex = originalContent.length
+					// ERROR: Empty search block with non-empty file indicates malformed SEARCH marker
+					throw new Error(
+						"Empty SEARCH block detected with non-empty file. This usually indicates a malformed SEARCH marker.\n" +
+							"Please ensure your SEARCH marker follows the correct format:\n" +
+							"- Use '------- SEARCH' (7+ dashes + space + SEARCH)\n",
+					)
 				}
 			} else {
 				// Add check for inefficient full-file search
@@ -570,7 +581,7 @@ class NewFileContentConstructor {
 		pendingNonStandardLineLimit: number,
 	): number {
 		let removeLineCount = 0
-		if (line === SEARCH_BLOCK_START) {
+		if (isSearchBlockStart(line)) {
 			removeLineCount = this.trimPendingNonStandardTrailingEmptyLines(pendingNonStandardLineLimit)
 			if (removeLineCount > 0) {
 				pendingNonStandardLineLimit = pendingNonStandardLineLimit - removeLineCount
@@ -580,7 +591,7 @@ class NewFileContentConstructor {
 				canWritependingNonStandardLines && (this.pendingNonStandardLines.length = 0)
 			}
 			this.activateSearchState()
-		} else if (line === SEARCH_BLOCK_END) {
+		} else if (isSearchBlockEnd(line)) {
 			// 校验非标内容
 			if (!this.isSearchingActive()) {
 				this.tryFixSearchBlock(pendingNonStandardLineLimit)
@@ -588,7 +599,7 @@ class NewFileContentConstructor {
 			}
 			this.activateReplaceState()
 			this.beforeReplace()
-		} else if (line === REPLACE_BLOCK_END) {
+		} else if (isReplaceBlockEnd(line)) {
 			if (!this.isReplacingActive()) {
 				this.tryFixReplaceBlock(pendingNonStandardLineLimit)
 				canWritependingNonStandardLines && (this.pendingNonStandardLines.length = 0)
@@ -695,7 +706,7 @@ class NewFileContentConstructor {
 		if (!lineLimit) {
 			throw new Error("Invalid SEARCH/REPLACE block structure - no lines available to process")
 		}
-		let searchTagRegexp = /^[-]{3,} SEARCH$/
+		let searchTagRegexp = /^([-]{3,}|[<]{3,}) SEARCH$/
 		const searchTagIndex = this.findLastMatchingLineIndex(searchTagRegexp, lineLimit)
 		if (searchTagIndex !== -1) {
 			let fixLines = this.pendingNonStandardLines.slice(searchTagIndex, lineLimit)
@@ -746,7 +757,7 @@ class NewFileContentConstructor {
 			throw new Error()
 		}
 
-		let replaceEndTagRegexp = /^[+]{3,} REPLACE$/
+		let replaceEndTagRegexp = /^([+]{3,}|[>]{3,}) REPLACE$/
 		const replaceEndTagIndex = this.findLastMatchingLineIndex(replaceEndTagRegexp, lineLimit)
 		const likeReplaceEndTag = replaceEndTagIndex === lineLimit - 1
 		if (likeReplaceEndTag) {
@@ -795,7 +806,11 @@ export async function constructNewFileContentV2(diffContent: string, originalCon
 	const lastLine = lines[lines.length - 1]
 	if (
 		lines.length > 0 &&
-		(lastLine.startsWith(SEARCH_BLOCK_CHAR) || lastLine.startsWith("=") || lastLine.startsWith(REPLACE_BLOCK_CHAR)) &&
+		(lastLine.startsWith(SEARCH_BLOCK_CHAR) ||
+			lastLine.startsWith(LEGACY_SEARCH_BLOCK_CHAR) ||
+			lastLine.startsWith("=") ||
+			lastLine.startsWith(REPLACE_BLOCK_CHAR) ||
+			lastLine.startsWith(LEGACY_REPLACE_BLOCK_CHAR)) &&
 		lastLine !== SEARCH_BLOCK_START &&
 		lastLine !== SEARCH_BLOCK_END &&
 		lastLine !== REPLACE_BLOCK_END

src/core/context/context-management/context-error-handling.ts

@@ -8,10 +8,7 @@ export function checkIsOpenRouterContextWindowError(error: any): boolean {
 
 export function checkIsAnthropicContextWindowError(response: any): boolean {
 	try {
-		return (
-			response?.error?.error?.type === "invalid_request_error" &&
-			response?.error?.error?.message?.includes("prompt is too long")
-		)
+		return response?.error?.error?.type === "invalid_request_error"
 	} catch (e: unknown) {
 		return false
 	}

src/core/context/context-tracking/FileContextTracker.ts

@@ -1,7 +1,10 @@
 import * as path from "path"
 import * as vscode from "vscode"
 import { getTaskMetadata, saveTaskMetadata } from "@core/storage/disk"
+import { getWorkspaceState, updateWorkspaceState } from "@core/storage/state"
+import { getGlobalState } from "@core/storage/state"
 import type { FileMetadataEntry } from "./ContextTrackerTypes"
+import type { ClineMessage } from "@shared/ExtensionMessage"
 
 // This class is responsible for tracking file operations that may result in stale context.
 // If a user modifies a file outside of Cline, the context may become stale and need to be updated.
@@ -11,10 +14,12 @@ import type { FileMetadataEntry } from "./ContextTrackerTypes"
 // a diff edit because the file was modified since Cline last read it.
 
 // FileContextTracker
-//
-// This class is responsible for tracking file operations.
-// If the full contents of a file are pass to Cline via a tool, mention, or edit, the file is marked as active.
-// If a file is modified outside of Cline, we detect and track this change to prevent stale context.
+/**
+This class is responsible for tracking file operations.
+If the full contents of a file are passed to Cline via a tool, mention, or edit, the file is marked as active.
+If a file is modified outside of Cline, we detect and track this change to prevent stale context.
+This is used when restoring a task (non-git "checkpoint" restore), and mid-task.
+*/
 export class FileContextTracker {
 	private context: vscode.ExtensionContext
 	readonly taskId: string
@@ -29,7 +34,9 @@ export class FileContextTracker {
 		this.taskId = taskId
 	}
 
-	// Gets the current working directory or returns undefined if it cannot be determined
+	/**
+	 * Gets the current working directory or returns undefined if it cannot be determined
+	 */
 	private getCwd(): string | undefined {
 		const cwd = vscode.workspace.workspaceFolders?.map((folder) => folder.uri.fsPath).at(0)
 		if (!cwd) {
@@ -38,7 +45,9 @@ export class FileContextTracker {
 		return cwd
 	}
 
-	// File watchers are set up for each file that is tracked in the task metadata.
+	/**
+	 * File watchers are set up for each file that is tracked in the task metadata.
+	 */
 	async setupFileWatcher(filePath: string) {
 		// Only setup watcher if it doesn't already exist for this file
 		if (this.fileWatchers.has(filePath)) {
@@ -70,8 +79,10 @@ export class FileContextTracker {
 		this.fileWatchers.set(filePath, watcher)
 	}
 
-	// Tracks a file operation in metadata and sets up a watcher for the file
-	// This is the main entry point for FileContextTracker and is called when a file is passed to Cline via a tool, mention, or edit.
+	/**
+	 * Tracks a file operation in metadata and sets up a watcher for the file
+	 * This is the main entry point for FileContextTracker and is called when a file is passed to Cline via a tool, mention, or edit.
+	 */
 	async trackFileContext(filePath: string, operation: "read_tool" | "user_edited" | "cline_edited" | "file_mentioned") {
 		try {
 			const cwd = this.getCwd()
@@ -89,9 +100,11 @@ export class FileContextTracker {
 		}
 	}
 
-	// Adds a file to the metadata tracker
-	// This handles the business logic of determining if the file is new, stale, or active.
-	// It also updates the metadata with the latest read/edit dates.
+	/**
+	 * Adds a file to the metadata tracker
+	 * This handles the business logic of determining if the file is new, stale, or active.
+	 * It also updates the metadata with the latest read/edit dates.
+	 */
 	async addFileToFileContextTracker(
 		context: vscode.ExtensionContext,
 		taskId: string,
@@ -154,23 +167,149 @@ export class FileContextTracker {
 		}
 	}
 
-	// Returns (and then clears) the set of recently modified files
+	/**
+	 * Returns (and then clears) the set of recently modified files
+	 */
 	getAndClearRecentlyModifiedFiles(): string[] {
 		const files = Array.from(this.recentlyModifiedFiles)
 		this.recentlyModifiedFiles.clear()
 		return files
 	}
 
-	// Marks a file as edited by Cline to prevent false positives in file watchers
+	/**
+	 * Marks a file as edited by Cline to prevent false positives in file watchers
+	 */
 	markFileAsEditedByCline(filePath: string): void {
 		this.recentlyEditedByCline.add(filePath)
 	}
 
-	// Disposes all file watchers
+	/**
+	 * Disposes all file watchers
+	 */
 	dispose(): void {
 		for (const watcher of this.fileWatchers.values()) {
 			watcher.dispose()
 		}
 		this.fileWatchers.clear()
 	}
+
+	/**
+	 * Detects files that were edited by Cline or users after a specific message timestamp
+	 * This is used when restoring checkpoints to warn about potential file content mismatches
+	 */
+	async detectFilesEditedAfterMessage(messageTs: number, deletedMessages: ClineMessage[]): Promise<string[]> {
+		const editedFiles: string[] = []
+
+		try {
+			// Check task metadata for files that were edited by Cline or users after the message timestamp
+			const taskMetadata = await getTaskMetadata(this.context, this.taskId)
+
+			if (taskMetadata?.files_in_context) {
+				for (const fileEntry of taskMetadata.files_in_context) {
+					const clineEditedAfter = fileEntry.cline_edit_date && fileEntry.cline_edit_date > messageTs
+					const userEditedAfter = fileEntry.user_edit_date && fileEntry.user_edit_date > messageTs
+
+					if (clineEditedAfter || userEditedAfter) {
+						editedFiles.push(fileEntry.path)
+					}
+				}
+			}
+		} catch (error) {
+			console.error("Error checking file context metadata:", error)
+		}
+
+		// Also check deleted task messages for file operations
+		for (const message of deletedMessages) {
+			if (message.say === "tool" && message.text) {
+				try {
+					const toolData = JSON.parse(message.text)
+					if ((toolData.tool === "editedExistingFile" || toolData.tool === "newFileCreated") && toolData.path) {
+						if (!editedFiles.includes(toolData.path)) {
+							editedFiles.push(toolData.path)
+						}
+					}
+				} catch (error) {
+					console.error("Error checking task messages:", error)
+				}
+			}
+		}
+		return [...new Set(editedFiles)]
+	}
+
+	/**
+	 * Stores pending file context warning in workspace state so it persists across task reinitialization
+	 */
+	async storePendingFileContextWarning(files: string[]): Promise<void> {
+		try {
+			const key = `pendingFileContextWarning_${this.taskId}`
+			await updateWorkspaceState(this.context, key, files)
+		} catch (error) {
+			console.error("Error storing pending file context warning:", error)
+		}
+	}
+
+	/**
+	 * Retrieves pending file context warning from workspace state (without clearing it)
+	 */
+	async retrievePendingFileContextWarning(): Promise<string[] | undefined> {
+		try {
+			const key = `pendingFileContextWarning_${this.taskId}`
+			const files = (await getWorkspaceState(this.context, key)) as string[]
+			return files
+		} catch (error) {
+			console.error("Error retrieving pending file context warning:", error)
+		}
+		return undefined
+	}
+
+	/**
+	 * Retrieves and clears pending file context warning from workspace state
+	 */
+	async retrieveAndClearPendingFileContextWarning(): Promise<string[] | undefined> {
+		try {
+			const files = await this.retrievePendingFileContextWarning()
+			if (files) {
+				await updateWorkspaceState(this.context, `pendingFileContextWarning_${this.taskId}`, undefined)
+				return files
+			}
+		} catch (error) {
+			console.error("Error retrieving pending file context warning:", error)
+		}
+		return undefined
+	}
+
+	/**
+	 * Static method to clean up orphaned pending file context warnings at startup
+	 * This removes warnings for tasks that may no longer exist
+	 */
+	static async cleanupOrphanedWarnings(context: vscode.ExtensionContext): Promise<void> {
+		const startTime = Date.now()
+		try {
+			const taskHistory = ((await getGlobalState(context, "taskHistory")) as Array<{ id: string }>) || []
+			const existingTaskIds = new Set(taskHistory.map((task) => task.id))
+			const allStateKeys = context.workspaceState.keys()
+			const pendingWarningKeys = allStateKeys.filter((key) => key.startsWith("pendingFileContextWarning_"))
+
+			const orphanedPendingContextTasks: string[] = []
+			for (const key of pendingWarningKeys) {
+				const taskId = key.replace("pendingFileContextWarning_", "")
+				if (!existingTaskIds.has(taskId)) {
+					orphanedPendingContextTasks.push(key)
+				}
+			}
+
+			if (orphanedPendingContextTasks.length > 0) {
+				for (const key of orphanedPendingContextTasks) {
+					await updateWorkspaceState(context, key, undefined)
+				}
+			}
+
+			const duration = Date.now() - startTime
+			console.log(
+				`FileContextTracker: Processed ${existingTaskIds.size} tasks, found ${pendingWarningKeys.length} pending warnings, ${orphanedPendingContextTasks.length} orphaned, deleted ${orphanedPendingContextTasks.length}, took ${duration}ms`,
+			)
+		} catch (error) {
+			console.error("Error cleaning up orphaned file context warnings:", error)
+		}
+	}
 }

src/core/controller/account/authStateChanged.ts

@@ -0,0 +1,23 @@
+import { AuthStateChangedRequest, AuthStateChanged } from "@shared/proto/account"
+import type { Controller } from "../index"
+import { updateGlobalState } from "../../storage/state"
+
+/**
+ * Handles authentication state changes from the Firebase context.
+ * Updates the user info in global state and returns the updated value.
+ * @param controller The controller instance
+ * @param request The auth state change request
+ * @returns The updated user info
+ */
+export async function authStateChanged(controller: Controller, request: AuthStateChangedRequest): Promise<AuthStateChanged> {
+	try {
+		// Store the user info directly in global state
+		await updateGlobalState(controller.context, "userInfo", request.user)
+
+		// Return the same user info
+		return AuthStateChanged.create({ user: request.user })
+	} catch (error) {
+		console.error(`Failed to update auth state: ${error}`)
+		throw error
+	}
+}

src/core/controller/checkpoints/checkpointRestore.ts

@@ -9,7 +9,7 @@ export async function checkpointRestore(controller: Controller, request: Checkpo
 
 	if (request.number) {
 		// wait for messages to be loaded
-		await pWaitFor(() => controller.task?.isInitialized === true, {
+		await pWaitFor(() => controller.task?.taskState.isInitialized === true, {
 			timeout: 3_000,
 		}).catch(() => {
 			console.error("Failed to init new cline instance")

src/core/controller/file/getRelativePaths.ts

@@ -3,8 +3,8 @@ import { RelativePathsRequest, RelativePaths } from "@shared/proto/file"
 import { FileMethodHandler } from "./index"
 import * as vscode from "vscode"
 import * as path from "path"
-import { UriServiceClient } from "@hosts/host-bridge-client"
 import { Metadata, StringRequest } from "@shared/proto/common"
+import { getHostBridgeProvider } from "@hosts/host-providers"
 
 /**
  * Converts a list of URIs to workspace-relative paths
@@ -20,7 +20,7 @@ export const getRelativePaths: FileMethodHandler = async (
 		request.uris.map(async (uriString) => {
 			try {
 				// Use the host URI service client instead of directly using vscode.Uri.parse
-				const parseResponse = await UriServiceClient.parse(
+				const parseResponse = await getHostBridgeProvider().uriServiceClient.parse(
 					StringRequest.create({
 						metadata: Metadata.create({}),
 						value: uriString,

src/core/controller/grpc-request-registry.ts

@@ -64,17 +64,17 @@ export class GrpcRequestRegistry {
 	 */
 	public cancelRequest(requestId: string): boolean {
 		const requestInfo = this.activeRequests.get(requestId)
-		if (requestInfo) {
-			try {
-				requestInfo.cleanup()
-				console.log(`[DEBUG] Cleaned up request: ${requestId}`)
-			} catch (error) {
-				console.error(`Error cleaning up request ${requestId}:`, error)
-			}
-			this.activeRequests.delete(requestId)
-			return true
+		if (!requestInfo) {
+			return false
 		}
-		return false
+		try {
+			requestInfo.cleanup()
+			console.log(`[DEBUG] Cleaned up request: ${requestId}`)
+		} catch (error) {
+			console.error(`Error cleaning up request ${requestId}:`, error)
+		}
+		this.activeRequests.delete(requestId)
+		return true
 	}
 
 	/**

src/core/controller/index.ts

@@ -20,6 +20,7 @@ import { ChatSettings } from "@shared/ChatSettings"
 import { ExtensionMessage, ExtensionState, Platform } from "@shared/ExtensionMessage"
 import { HistoryItem } from "@shared/HistoryItem"
 import { McpMarketplaceCatalog } from "@shared/mcp"
+import { UserInfo } from "@shared/UserInfo"
 import { TelemetrySetting } from "@shared/TelemetrySetting"
 import { WebviewMessage } from "@shared/WebviewMessage"
 import { fileExistsAtPath } from "@utils/fs"
@@ -51,7 +52,7 @@ https://github.com/KumarVariable/vscode-extension-sidebar-html/blob/master/src/c
 */
 
 export class Controller {
-	readonly id: string = uuidv4()
+	readonly id: string
 	private postMessage: (message: ExtensionMessage) => Thenable<boolean> | undefined
 
 	private disposables: vscode.Disposable[] = []
@@ -65,7 +66,9 @@ export class Controller {
 		readonly context: vscode.ExtensionContext,
 		private readonly outputChannel: vscode.OutputChannel,
 		postMessage: (message: ExtensionMessage) => Thenable<boolean> | undefined,
+		id: string,
 	) {
+		this.id = id
 		this.outputChannel.appendLine("ClineProvider instantiated")
 		this.postMessage = postMessage
 
@@ -122,7 +125,7 @@ export class Controller {
 		}
 	}
 
-	async setUserInfo(info?: { displayName: string | null; email: string | null; photoURL: string | null }) {
+	async setUserInfo(info?: UserInfo) {
 		await updateGlobalState(this.context, "userInfo", info)
 	}
 
@@ -130,12 +133,13 @@ export class Controller {
 		await this.clearTask() // ensures that an existing task doesn't exist before starting a new one, although this shouldn't be possible since user must clear task before starting a new one
 		const {
 			apiConfiguration,
-			customInstructions,
 			autoApprovalSettings,
 			browserSettings,
 			chatSettings,
 			shellIntegrationTimeout,
 			terminalReuseEnabled,
+			terminalOutputLineLimit,
+			defaultTerminalProfile,
 			enableCheckpointsSetting,
 			isNewUser,
 			taskHistory,
@@ -171,8 +175,9 @@ export class Controller {
 			chatSettings,
 			shellIntegrationTimeout,
 			terminalReuseEnabled ?? true,
+			terminalOutputLineLimit ?? 500,
+			defaultTerminalProfile ?? "default",
 			enableCheckpointsSetting ?? true,
-			customInstructions,
 			task,
 			images,
 			files,
@@ -200,19 +205,6 @@ export class Controller {
 	 */
 	async handleWebviewMessage(message: WebviewMessage) {
 		switch (message.type) {
-			case "authStateChanged":
-				await this.setUserInfo(message.user || undefined)
-				await this.postStateToWebview()
-				break
-			case "apiConfiguration":
-				if (message.apiConfiguration) {
-					await updateApiConfiguration(this.context, message.apiConfiguration)
-					if (this.task) {
-						this.task.api = buildApiHandler(message.apiConfiguration)
-					}
-				}
-				await this.postStateToWebview()
-				break
 			case "fetchUserCreditsData": {
 				await this.fetchUserCreditsData()
 				break
@@ -230,6 +222,7 @@ export class Controller {
 				await this.postStateToWebview()
 				break
 			}
+
 			case "clearAllTaskHistory": {
 				const answer = await vscode.window.showWarningMessage(
 					"What would you like to delete?",
@@ -273,7 +266,7 @@ export class Controller {
 		telemetryService.updateTelemetryState(isOptedIn)
 	}
 
-	async togglePlanActModeWithChatSettings(chatSettings: ChatSettings, chatContent?: ChatContent) {
+	async togglePlanActModeWithChatSettings(chatSettings: ChatSettings, chatContent?: ChatContent): Promise<boolean> {
 		const didSwitchToActMode = chatSettings.mode === "act"
 
 		// Capture mode switch telemetry | Capture regardless of if we know the taskId
@@ -290,6 +283,12 @@ export class Controller {
 			previousModeReasoningEffort: newReasoningEffort,
 			previousModeAwsBedrockCustomSelected: newAwsBedrockCustomSelected,
 			previousModeAwsBedrockCustomModelBaseId: newAwsBedrockCustomModelBaseId,
+			previousModeSapAiCoreClientId: newSapAiCoreClientId,
+			previousModeSapAiCoreClientSecret: newSapAiCoreClientSecret,
+			previousModeSapAiCoreBaseUrl: newSapAiCoreBaseUrl,
+			previousModeSapAiCoreTokenUrl: newSapAiCoreTokenUrl,
+			previousModeSapAiCoreResourceGroup: newSapAiResourceGroup,
+			previousModeSapAiCoreModelId: newSapAiCoreModelId,
 			planActSeparateModelsSetting,
 		} = await getAllExtensionState(this.context)
 
@@ -355,6 +354,23 @@ export class Controller {
 					await updateWorkspaceState(this.context, "previousModeModelId", apiConfiguration.requestyModelId)
 					await updateWorkspaceState(this.context, "previousModeModelInfo", apiConfiguration.requestyModelInfo)
 					break
+				case "sapaicore":
+					await updateWorkspaceState(this.context, "previousModeModelId", apiConfiguration.apiModelId)
+					await updateWorkspaceState(this.context, "previousModeSapAiCoreClientId", apiConfiguration.sapAiCoreClientId)
+					await updateWorkspaceState(
+						this.context,
+						"previousModeSapAiCoreClientSecret",
+						apiConfiguration.sapAiCoreClientSecret,
+					)
+					await updateWorkspaceState(this.context, "previousModeSapAiCoreBaseUrl", apiConfiguration.sapAiCoreBaseUrl)
+					await updateWorkspaceState(this.context, "previousModeSapAiCoreTokenUrl", apiConfiguration.sapAiCoreTokenUrl)
+					await updateWorkspaceState(
+						this.context,
+						"previousModeSapAiCoreResourceGroup",
+						apiConfiguration.sapAiResourceGroup,
+					)
+					await updateWorkspaceState(this.context, "previousModeSapAiCoreModelId", apiConfiguration.sapAiCoreModelId)
+					break
 			}
 
 			// Restore the model used in previous mode
@@ -403,13 +419,16 @@ export class Controller {
 						await updateWorkspaceState(this.context, "lmStudioModelId", newModelId)
 						break
 					case "litellm":
-						await updateWorkspaceState(this.context, "previousModeModelId", apiConfiguration.liteLlmModelId)
-						await updateWorkspaceState(this.context, "previousModeModelInfo", apiConfiguration.liteLlmModelInfo)
+						await updateWorkspaceState(this.context, "liteLlmModelId", newModelId)
+						await updateWorkspaceState(this.context, "liteLlmModelInfo", newModelInfo)
 						break
 					case "requesty":
 						await updateWorkspaceState(this.context, "requestyModelId", newModelId)
 						await updateWorkspaceState(this.context, "requestyModelInfo", newModelInfo)
 						break
+					case "sapaicore":
+						await updateWorkspaceState(this.context, "apiModelId", newModelId)
+						break
 				}
 
 				if (this.task) {
@@ -424,8 +443,8 @@ export class Controller {
 
 		if (this.task) {
 			this.task.chatSettings = chatSettings
-			if (this.task.isAwaitingPlanResponse && didSwitchToActMode) {
-				this.task.didRespondToPlanAskBySwitchingMode = true
+			if (this.task.taskState.isAwaitingPlanResponse && didSwitchToActMode) {
+				this.task.taskState.didRespondToPlanAskBySwitchingMode = true
 				// Use chatContent if provided, otherwise use default message
 				await this.task.handleWebviewAskResponse(
 					"messageResponse",
@@ -433,10 +452,15 @@ export class Controller {
 					chatContent?.images || [],
 					chatContent?.files || [],
 				)
+
+				return true
 			} else {
 				this.cancelTask()
+				return false
 			}
 		}
+
+		return false
 	}
 
 	async cancelTask() {
@@ -450,9 +474,9 @@ export class Controller {
 			await pWaitFor(
 				() =>
 					this.task === undefined ||
-					this.task.isStreaming === false ||
-					this.task.didFinishAbortingStream ||
-					this.task.isWaitingForFirstChunk, // if only first chunk is processed, then there's no need to wait for graceful abort (closes edits, browser, etc)
+					this.task.taskState.isStreaming === false ||
+					this.task.taskState.didFinishAbortingStream ||
+					this.task.taskState.isWaitingForFirstChunk, // if only first chunk is processed, then there's no need to wait for graceful abort (closes edits, browser, etc)
 				{
 					timeout: 3_000,
 				},
@@ -461,21 +485,13 @@ export class Controller {
 			})
 			if (this.task) {
 				// 'abandoned' will prevent this cline instance from affecting future cline instance gui. this may happen if its hanging on a streaming request
-				this.task.abandoned = true
+				this.task.taskState.abandoned = true
 			}
 			await this.initTask(undefined, undefined, undefined, historyItem) // clears task again, so we need to abortTask manually above
 			// await this.postStateToWebview() // new Cline instance will post state when it's ready. having this here sent an empty messages array to webview leading to virtuoso having to reload the entire list
 		}
 	}
 
-	async updateCustomInstructions(instructions?: string) {
-		// User may be clearing the field
-		await updateGlobalState(this.context, "customInstructions", instructions || undefined)
-		if (this.task) {
-			this.task.customInstructions = instructions || undefined
-		}
-	}
-
 	// Account
 
 	async fetchUserCreditsData() {
@@ -957,13 +973,13 @@ export class Controller {
 		const {
 			apiConfiguration,
 			lastShownAnnouncementId,
-			customInstructions,
 			taskHistory,
 			autoApprovalSettings,
 			browserSettings,
 			chatSettings,
 			userInfo,
 			mcpMarketplaceEnabled,
+			mcpRichDisplayEnabled,
 			telemetrySetting,
 			planActSeparateModelsSetting,
 			enableCheckpointsSetting,
@@ -971,8 +987,10 @@ export class Controller {
 			globalWorkflowToggles,
 			shellIntegrationTimeout,
 			terminalReuseEnabled,
+			defaultTerminalProfile,
 			isNewUser,
 			mcpResponsesCollapsed,
+			terminalOutputLineLimit,
 		} = await getAllExtensionState(this.context)
 
 		const localClineRulesToggles =
@@ -989,11 +1007,10 @@ export class Controller {
 		return {
 			version: this.context.extension?.packageJSON?.version ?? "",
 			apiConfiguration,
-			customInstructions,
 			uriScheme: vscode.env.uriScheme,
 			currentTaskItem: this.task?.taskId ? (taskHistory || []).find((item) => item.id === this.task?.taskId) : undefined,
-			checkpointTrackerErrorMessage: this.task?.checkpointTrackerErrorMessage,
-			clineMessages: this.task?.clineMessages || [],
+			checkpointTrackerErrorMessage: this.task?.taskState.checkpointTrackerErrorMessage,
+			clineMessages: this.task?.messageStateHandler.getClineMessages() || [],
 			taskHistory: (taskHistory || [])
 				.filter((item) => item.ts && item.task)
 				.sort((a, b) => b.ts - a.ts)
@@ -1005,6 +1022,7 @@ export class Controller {
 			chatSettings,
 			userInfo,
 			mcpMarketplaceEnabled,
+			mcpRichDisplayEnabled,
 			telemetrySetting,
 			planActSeparateModelsSetting,
 			enableCheckpointsSetting: enableCheckpointsSetting ?? true,
@@ -1017,8 +1035,10 @@ export class Controller {
 			globalWorkflowToggles: globalWorkflowToggles || {},
 			shellIntegrationTimeout,
 			terminalReuseEnabled,
+			defaultTerminalProfile,
 			isNewUser,
 			mcpResponsesCollapsed,
+			terminalOutputLineLimit,
 		}
 	}
 

src/core/controller/models/refreshRequestyModels.ts

@@ -41,11 +41,6 @@ export async function refreshRequestyModels(controller: Controller, _: EmptyRequ
 				models[model.id] = modelInfo
 			}
 			console.log("Requesty models fetched", models)
-
-			controller.postMessageToWebview({
-				type: "requestyModels",
-				requestyModels: models,
-			})
 		} else {
 			console.error("Invalid response from Requesty API")
 		}

src/core/controller/models/updateApiConfigurationProto.ts

@@ -0,0 +1,43 @@
+import type { Controller } from "../index"
+import { Empty } from "@shared/proto/common"
+import { UpdateApiConfigurationRequest } from "@shared/proto/models"
+import { updateApiConfiguration } from "../../storage/state"
+import { buildApiHandler } from "@api/index"
+import { convertProtoToApiConfiguration } from "@shared/proto-conversions/models/api-configuration-conversion"
+
+/**
+ * Updates API configuration
+ * @param controller The controller instance
+ * @param request The update API configuration request
+ * @returns Empty response
+ */
+export async function updateApiConfigurationProto(
+	controller: Controller,
+	request: UpdateApiConfigurationRequest,
+): Promise<Empty> {
+	try {
+		if (!request.apiConfiguration) {
+			console.log("[APICONFIG: updateApiConfigurationProto] API configuration is required")
+			throw new Error("API configuration is required")
+		}
+
+		// Convert proto ApiConfiguration to application ApiConfiguration
+		const appApiConfiguration = convertProtoToApiConfiguration(request.apiConfiguration)
+
+		// Update the API configuration in storage
+		await updateApiConfiguration(controller.context, appApiConfiguration)
+
+		// Update the task's API handler if there's an active task
+		if (controller.task) {
+			controller.task.api = buildApiHandler(appApiConfiguration)
+		}
+
+		// Post updated state to webview
+		await controller.postStateToWebview()
+
+		return Empty.create()
+	} catch (error) {
+		console.error(`Failed to update API configuration: ${error}`)
+		throw error
+	}
+}

src/core/controller/state/getAvailableTerminalProfiles.ts

@@ -0,0 +1,19 @@
+import { Controller } from "../index"
+import * as proto from "@/shared/proto"
+import { getAvailableTerminalProfiles as getTerminalProfilesFromShell } from "../../../utils/shell"
+
+export async function getAvailableTerminalProfiles(
+	controller: Controller,
+	request: proto.cline.EmptyRequest,
+): Promise<proto.cline.TerminalProfiles> {
+	const profiles = getTerminalProfilesFromShell()
+
+	return proto.cline.TerminalProfiles.create({
+		profiles: profiles.map((profile) => ({
+			id: profile.id,
+			name: profile.name,
+			path: profile.path || "",
+			description: profile.description || "",
+		})),
+	})
+}

src/core/controller/state/resetState.ts

@@ -1,19 +1,25 @@
 import { Controller } from ".."
-import { Empty, EmptyRequest } from "../../../shared/proto/common"
-import { resetExtensionState } from "../../../core/storage/state"
+import { Empty } from "../../../shared/proto/common"
+import { ResetStateRequest } from "../../../shared/proto/state"
+import { resetGlobalState, resetWorkspaceState } from "../../../core/storage/state"
 import * as vscode from "vscode"
 import { sendChatButtonClickedEvent } from "../ui/subscribeToChatButtonClicked"
 
 /**
  * Resets the extension state to its defaults
  * @param controller The controller instance
- * @param request An empty request (no parameters needed)
+ * @param request The reset state request containing the global flag
  * @returns An empty response
  */
-export async function resetState(controller: Controller, request: EmptyRequest): Promise<Empty> {
+export async function resetState(controller: Controller, request: ResetStateRequest): Promise<Empty> {
 	try {
-		vscode.window.showInformationMessage("Resetting state...")
-		await resetExtensionState(controller.context)
+		if (request.global) {
+			vscode.window.showInformationMessage("Resetting global state...")
+			await resetGlobalState(controller.context)
+		} else {
+			vscode.window.showInformationMessage("Resetting workspace state...")
+			await resetWorkspaceState(controller.context)
+		}
 
 		if (controller.task) {
 			controller.task.abortTask()

src/core/controller/state/togglePlanActMode.ts

@@ -1,5 +1,5 @@
 import { Controller } from ".."
-import { Empty } from "../../../shared/proto/common"
+import { Boolean } from "../../../shared/proto/common"
 import { TogglePlanActModeRequest } from "../../../shared/proto/state"
 import {
 	convertProtoChatContentToChatContent,
@@ -12,7 +12,7 @@ import {
  * @param request The request containing the chat settings and optional chat content
  * @returns An empty response
  */
-export async function togglePlanActMode(controller: Controller, request: TogglePlanActModeRequest): Promise<Empty> {
+export async function togglePlanActMode(controller: Controller, request: TogglePlanActModeRequest): Promise<Boolean> {
 	try {
 		if (!request.chatSettings) {
 			throw new Error("Chat settings are required")
@@ -22,9 +22,11 @@ export async function togglePlanActMode(controller: Controller, request: ToggleP
 		const chatContent = request.chatContent ? convertProtoChatContentToChatContent(request.chatContent) : undefined
 
 		// Call the existing controller implementation
-		await controller.togglePlanActModeWithChatSettings(chatSettings, chatContent)
+		const sentMessage = await controller.togglePlanActModeWithChatSettings(chatSettings, chatContent)
 
-		return Empty.create()
+		return Boolean.create({
+			value: sentMessage,
+		})
 	} catch (error) {
 		console.error("Failed to toggle Plan/Act mode:", error)
 		throw error

src/core/controller/state/updateDefaultTerminalProfile.ts

@@ -0,0 +1,50 @@
+import * as vscode from "vscode"
+import { Controller } from "../index"
+import * as proto from "@/shared/proto"
+import { updateGlobalState } from "../../storage/state"
+import { TerminalInfo } from "@/integrations/terminal/TerminalRegistry"
+
+export async function updateDefaultTerminalProfile(
+	controller: Controller,
+	request: proto.cline.StringRequest,
+): Promise<proto.cline.TerminalProfileUpdateResponse> {
+	const profileId = request.value
+
+	// Update the terminal profile in the state
+	await updateGlobalState(controller.context, "defaultTerminalProfile", profileId)
+
+	let closedCount = 0
+	let busyTerminals: TerminalInfo[] = []
+
+	// Update the terminal manager of the current task if it exists
+	if (controller.task) {
+		// Call the updated setDefaultTerminalProfile method that returns closed terminal info
+		const result = controller.task.terminalManager.setDefaultTerminalProfile(profileId)
+		closedCount = result.closedCount
+		busyTerminals = result.busyTerminals
+
+		// Show information message if terminals were closed
+		if (closedCount > 0) {
+			vscode.window.showInformationMessage(
+				`Closed ${closedCount} ${closedCount === 1 ? "terminal" : "terminals"} with different profile.`,
+			)
+		}
+
+		// Show warning if there are busy terminals that couldn't be closed
+		if (busyTerminals.length > 0) {
+			vscode.window.showWarningMessage(
+				`${busyTerminals.length} busy ${busyTerminals.length === 1 ? "terminal has" : "terminals have"} a different profile. ` +
+					`Close ${busyTerminals.length === 1 ? "it" : "them"} to use the new profile for all commands.`,
+			)
+		}
+	}
+
+	// Broadcast state update to all webviews
+	await controller.postStateToWebview()
+
+	return proto.cline.TerminalProfileUpdateResponse.create({
+		closedCount,
+		busyTerminalsCount: busyTerminals.length,
+		hasBusyTerminals: busyTerminals.length > 0,
+	})
+}

src/core/controller/state/updateSettings.ts

@@ -25,11 +25,6 @@ export async function updateSettings(controller: Controller, request: UpdateSett
 			}
 		}
 
-		// Update custom instructions
-		if (request.customInstructionsSetting !== undefined) {
-			await controller.updateCustomInstructions(request.customInstructionsSetting)
-		}
-
 		// Update telemetry setting
 		if (request.telemetrySetting) {
 			await controller.updateTelemetrySetting(request.telemetrySetting as TelemetrySetting)
@@ -55,10 +50,15 @@ export async function updateSettings(controller: Controller, request: UpdateSett
 			await controller.context.globalState.update("mcpResponsesCollapsed", request.mcpResponsesCollapsed)
 		}
 
+		// Update MCP responses collapsed setting
+		if (request.mcpRichDisplayEnabled !== undefined) {
+			await controller.context.globalState.update("mcpRichDisplayEnabled", request.mcpRichDisplayEnabled)
+		}
+
 		// Update chat settings
 		if (request.chatSettings) {
 			const chatSettings = convertProtoChatSettingsToChatSettings(request.chatSettings)
-			await controller.context.globalState.update("chatSettings", chatSettings)
+			await controller.context.workspaceState.update("chatSettings", chatSettings)
 			if (controller.task) {
 				controller.task.chatSettings = chatSettings
 			}
@@ -74,6 +74,11 @@ export async function updateSettings(controller: Controller, request: UpdateSett
 			await controller.context.globalState.update("terminalReuseEnabled", request.terminalReuseEnabled)
 		}
 
+		// Update terminal output line limit
+		if (request.terminalOutputLineLimit !== undefined) {
+			await controller.context.globalState.update("terminalOutputLineLimit", Number(request.terminalOutputLineLimit))
+		}
+
 		// Post updated state to webview
 		await controller.postStateToWebview()
 

src/core/controller/state/updateTerminalConnectionTimeout.ts

@@ -1,27 +1,18 @@
-import { Controller } from ".."
-import { Int64, Int64Request } from "../../../shared/proto/common"
+import { Controller } from "../index"
+import * as proto from "@/shared/proto"
 import { updateGlobalState } from "../../storage/state"
 
-/**
- * Updates the terminal connection timeout setting
- * @param controller The controller instance
- * @param request The request containing the timeout value in milliseconds
- * @returns The updated timeout value
- */
-export async function updateTerminalConnectionTimeout(controller: Controller, request: Int64Request): Promise<Int64> {
-	try {
-		const timeout = request.value
+export async function updateTerminalConnectionTimeout(
+	controller: Controller,
+	request: proto.cline.Int64Request,
+): Promise<proto.cline.Int64> {
+	const timeoutValue = request.value
 
-		if (typeof timeout === "number" && !isNaN(timeout) && timeout > 0) {
-			// Update the global state directly
-			await updateGlobalState(controller.context, "shellIntegrationTimeout", timeout)
-			return Int64.create({ value: timeout })
-		} else {
-			console.warn(`Invalid shell integration timeout value received: ${timeout}. Expected a positive number.`)
-			throw new Error("Invalid timeout value. Expected a positive number.")
-		}
-	} catch (error) {
-		console.error(`Failed to update terminal connection timeout: ${error}`)
-		throw error
-	}
+	// Update the terminal connection timeout setting in the state
+	await updateGlobalState(controller.context, "shellIntegrationTimeout", timeoutValue)
+
+	// Broadcast state update to all webviews
+	await controller.postStateToWebview()
+
+	return proto.cline.Int64.create({ value: timeoutValue })
 }

src/core/controller/state/updateTerminalReuseEnabled.ts

@@ -0,0 +1,18 @@
+import { Controller } from "../index"
+import * as proto from "@/shared/proto"
+import { updateGlobalState } from "../../storage/state"
+
+export async function updateTerminalReuseEnabled(
+	controller: Controller,
+	request: proto.cline.BooleanRequest,
+): Promise<proto.cline.Empty> {
+	const enabled = request.value
+
+	// Update the terminal reuse setting in the state
+	await updateGlobalState(controller.context, "terminalReuseEnabled", enabled)
+
+	// Broadcast state update to all webviews
+	await controller.postStateToWebview()
+
+	return proto.cline.Empty.create({})
+}

src/core/controller/ui/getWebviewHtml.ts

@@ -0,0 +1,16 @@
+import type { Controller } from "../index"
+import { EmptyRequest, Empty, String } from "@shared/proto/common"
+import * as hostProviders from "@hosts/host-providers"
+import { WebviewProviderType } from "@/shared/webview/types"
+
+/**
+ * Initialize webview when it launches
+ * @param controller The controller instance
+ * @param request The empty request
+ * @returns Empty response
+ */
+export async function getWebviewHtml(_controller: Controller, _: EmptyRequest): Promise<String> {
+	const webviewProvider = hostProviders.createWebviewProvider(WebviewProviderType.SIDEBAR)
+
+	return Promise.resolve(String.create({ value: webviewProvider.getHtmlContent() }))
+}

src/core/controller/ui/subscribeToDidBecomeVisible.ts

@@ -0,0 +1,63 @@
+import { Controller } from "../index"
+import { Empty } from "@shared/proto/common"
+import { EmptyRequest } from "@shared/proto/common"
+import { StreamingResponseHandler, getRequestRegistry } from "../grpc-handler"
+
+// Keep track of active didBecomeVisible subscriptions by controller ID
+const activeDidBecomeVisibleSubscriptions = new Map<string, StreamingResponseHandler>()
+
+/**
+ * Subscribe to didBecomeVisible events
+ * @param controller The controller instance
+ * @param request The empty request
+ * @param responseStream The streaming response handler
+ * @param requestId The ID of the request (passed by the gRPC handler)
+ */
+export async function subscribeToDidBecomeVisible(
+	controller: Controller,
+	request: EmptyRequest,
+	responseStream: StreamingResponseHandler,
+	requestId?: string,
+): Promise<void> {
+	const controllerId = controller.id
+	console.log(`[DEBUG] set up didBecomeVisible subscription for controller ${controllerId}`)
+
+	// Add this subscription to the active subscriptions with the controller ID
+	activeDidBecomeVisibleSubscriptions.set(controllerId, responseStream)
+
+	// Register cleanup when the connection is closed
+	const cleanup = () => {
+		activeDidBecomeVisibleSubscriptions.delete(controllerId)
+	}
+
+	// Register the cleanup function with the request registry if we have a requestId
+	if (requestId) {
+		getRequestRegistry().registerRequest(requestId, cleanup, { type: "didBecomeVisible_subscription" }, responseStream)
+	}
+}
+
+/**
+ * Send a didBecomeVisible event to a specific controller's subscription
+ * @param controllerId The ID of the controller to send the event to
+ */
+export async function sendDidBecomeVisibleEvent(controllerId: string): Promise<void> {
+	// Get the subscription for this specific controller
+	const responseStream = activeDidBecomeVisibleSubscriptions.get(controllerId)
+
+	if (!responseStream) {
+		console.log(`[DEBUG] No active subscription for controller ${controllerId}`)
+		return
+	}
+
+	try {
+		const event: Empty = Empty.create({})
+		await responseStream(
+			event,
+			false, // Not the last message
+		)
+	} catch (error) {
+		console.error(`Error sending didBecomeVisible event to controller ${controllerId}:`, error)
+		// Remove the subscription if there was an error
+		activeDidBecomeVisibleSubscriptions.delete(controllerId)
+	}
+}

src/core/controller/ui/subscribeToFocusChatInput.ts

@@ -0,0 +1,62 @@
+import { StringRequest, Empty } from "@shared/proto/common"
+import { StreamingResponseHandler, getRequestRegistry } from "../grpc-handler"
+import type { Controller } from "../index"
+
+// Map client IDs to their subscription handlers
+const focusChatInputSubscriptions = new Map<string, StreamingResponseHandler>()
+
+/**
+ * Subscribe to focus chat input events
+ * @param controller The controller instance
+ * @param request The request containing the client ID
+ * @param responseStream The streaming response handler
+ * @param requestId The ID of the request
+ */
+export async function subscribeToFocusChatInput(
+	controller: Controller,
+	request: StringRequest,
+	responseStream: StreamingResponseHandler,
+	requestId?: string,
+): Promise<void> {
+	const clientId = request.value
+	if (!clientId) {
+		throw new Error("Client ID is required for focusChatInput subscription")
+	}
+
+	// Store this subscription with its client ID
+	focusChatInputSubscriptions.set(clientId, responseStream)
+
+	// Register cleanup when the connection is closed
+	const cleanup = () => {
+		focusChatInputSubscriptions.delete(clientId)
+	}
+
+	// Register the cleanup function with the request registry if we have a requestId
+	if (requestId) {
+		getRequestRegistry().registerRequest(requestId, cleanup, { type: "focus_chat_input_subscription" }, responseStream)
+	}
+}
+
+/**
+ * Send a focus chat input event to a specific webview by client ID
+ * @param clientId The ID of the client to send the event to
+ */
+export async function sendFocusChatInputEvent(clientId: string): Promise<void> {
+	const responseStream = focusChatInputSubscriptions.get(clientId)
+	if (!responseStream) {
+		console.warn(`No subscription found for client ID: ${clientId}`)
+		return
+	}
+
+	try {
+		const event = Empty.create({})
+		await responseStream(
+			event,
+			false, // Not the last message
+		)
+	} catch (error) {
+		console.error(`Error sending focus chat input event to client ${clientId}:`, error)
+		// Remove the subscription if there was an error
+		focusChatInputSubscriptions.delete(clientId)
+	}
+}

src/core/controller/ui/subscribeToMcpButtonClicked.ts

@@ -1,33 +1,33 @@
 import { Controller } from "../index"
 import { Empty } from "@shared/proto/common"
-import { WebviewProviderType, WebviewProviderTypeRequest } from "@shared/proto/ui"
+import { EmptyRequest } from "@shared/proto/common"
 import { StreamingResponseHandler, getRequestRegistry } from "../grpc-handler"
 
-// Track subscriptions with their provider type
-const mcpButtonClickedSubscriptions = new Map<StreamingResponseHandler, WebviewProviderType>()
+// Keep track of active mcpButtonClicked subscriptions by controller ID
+const activeMcpButtonClickedSubscriptions = new Map<string, StreamingResponseHandler>()
 
 /**
  * Subscribe to mcpButtonClicked events
  * @param controller The controller instance
- * @param request The webview provider type request
+ * @param request The empty request
  * @param responseStream The streaming response handler
  * @param requestId The ID of the request (passed by the gRPC handler)
  */
 export async function subscribeToMcpButtonClicked(
-	_controller: Controller,
-	request: WebviewProviderTypeRequest,
+	controller: Controller,
+	_request: EmptyRequest,
 	responseStream: StreamingResponseHandler,
 	requestId?: string,
 ): Promise<void> {
-	const providerType = request.providerType
-	console.log(`[DEBUG] set up mcpButtonClicked subscription for ${WebviewProviderType[providerType]} webview`)
+	const controllerId = controller.id
+	console.log(`[DEBUG] set up mcpButtonClicked subscription for controller ${controllerId}`)
 
-	// Store the subscription with its provider type
-	mcpButtonClickedSubscriptions.set(responseStream, providerType)
+	// Add this subscription to the active subscriptions with the controller ID
+	activeMcpButtonClickedSubscriptions.set(controllerId, responseStream)
 
 	// Register cleanup when the connection is closed
 	const cleanup = () => {
-		mcpButtonClickedSubscriptions.delete(responseStream)
+		activeMcpButtonClickedSubscriptions.delete(controllerId)
 	}
 
 	// Register the cleanup function with the request registry if we have a requestId
@@ -37,26 +37,27 @@ export async function subscribeToMcpButtonClicked(
 }
 
 /**
- * Send a mcpButtonClicked event to active subscribers based on webview type
- * @param webviewType The type of webview that triggered the event (SIDEBAR or TAB)
+ * Send a mcpButtonClicked event to a specific controller's subscription
+ * @param controllerId The ID of the controller to send the event to
  */
-export async function sendMcpButtonClickedEvent(webviewType?: WebviewProviderType): Promise<void> {
-	const event = Empty.create({})
+export async function sendMcpButtonClickedEvent(controllerId: string): Promise<void> {
+	// Get the subscription for this specific controller
+	const responseStream = activeMcpButtonClickedSubscriptions.get(controllerId)
 
-	// Process all subscriptions, filtering based on the source
-	const promises = Array.from(mcpButtonClickedSubscriptions.entries()).map(async ([responseStream, providerType]) => {
-		// Only send to subscribers of the same type as the event source
-		if (webviewType !== providerType) {
-			return // Skip subscribers of different types
-		}
+	if (!responseStream) {
+		console.error(`[DEBUG] No active subscription for controller ${controllerId}`)
+		return
+	}
 
-		try {
-			await responseStream(event, false)
-		} catch (error) {
-			console.error(`Error sending mcpButtonClicked event to ${WebviewProviderType[providerType]}:`, error)
-			mcpButtonClickedSubscriptions.delete(responseStream)
-		}
-	})
-
-	await Promise.all(promises)
+	try {
+		const event = Empty.create({})
+		await responseStream(
+			event,
+			false, // Not the last message
+		)
+	} catch (error) {
+		console.error(`Error sending mcpButtonClicked event to controller ${controllerId}:`, error)
+		// Remove the subscription if there was an error
+		activeMcpButtonClickedSubscriptions.delete(controllerId)
+	}
 }

src/core/prompts/model_prompts/claude4-experimental.ts

@@ -128,7 +128,10 @@ ${
 					const config = JSON.parse(server.config)
 
 					return (
-						`## ${server.name} (\`${config.command}${config.args && Array.isArray(config.args) ? ` ${config.args.join(" ")}` : ""}\`)` +
+						`## ${server.name}` +
+						(config.command
+							? ` (\`${config.command}${config.args && Array.isArray(config.args) ? ` ${config.args.join(" ")}` : ""}\`)`
+							: "") +
 						(tools ? `\n\n### Available Tools\n${tools}` : "") +
 						(templates ? `\n\n### Resource Templates\n${templates}` : "") +
 						(resources ? `\n\n### Direct Resources\n${resources}` : "")

src/core/prompts/model_prompts/claude4.ts

@@ -470,7 +470,10 @@ ${
                     const config = JSON.parse(server.config)
 
                     return (
-                        `## ${server.name} (\`${config.command}${config.args && Array.isArray(config.args) ? ` ${config.args.join(" ")}` : ""}\`)` +
+						`## ${server.name}` +
+						(config.command
+							? ` (\`${config.command}${config.args && Array.isArray(config.args) ? ` ${config.args.join(" ")}` : ""}\`)`
+							: "") +
                         (tools ? `\n\n### Available Tools\n${tools}` : "") +
                         (templates ? `\n\n### Resource Templates\n${templates}` : "") +
                         (resources ? `\n\n### Direct Resources\n${resources}` : "")
@@ -663,7 +666,6 @@ You accomplish a given task iteratively, breaking it down into clear steps and w
     }
 
 export function addUserInstructions(
-    settingsCustomInstructions?: string,
     globalClineRulesFileInstructions?: string,
     localClineRulesFileInstructions?: string,
     localCursorRulesFileInstructions?: string,
@@ -676,9 +678,6 @@ export function addUserInstructions(
     if (preferredLanguageInstructions) {
         customInstructions += preferredLanguageInstructions + "\n\n"
     }
-    if (settingsCustomInstructions) {
-        customInstructions += settingsCustomInstructions + "\n\n"
-    }
     if (globalClineRulesFileInstructions) {
         customInstructions += globalClineRulesFileInstructions + "\n\n"
     }