Fix Changeset

* Run pretest in CI to build all tests

* Alias paths when running tests

* Bundle ES modules with esbuild

* alias packages

.changeset/bright-knives-wait.md

@@ -0,0 +1,5 @@
+---
+"claude-dev": patch
+---
+
+updated drag and drop text to say "drop" instead of "drag"

.changeset/bright-zebras-sip.md

@@ -0,0 +1,5 @@
+---
+"claude-dev": patch
+---
+
+Add markdown copy to chat

.changeset/custom-timeout-setting.md

@@ -0,0 +1,5 @@
+---
+"claude-dev": minor
+---
+
+Add support for custom API request timeout. Previously, timeouts were hardcoded to 30 seconds for providers like Ollama or 15 seconds for OpenRouter and Cline. Now users can set a custom timeout value in milliseconds through the settings interface.

.changeset/fair-papayas-learn.md

@@ -0,0 +1,5 @@
+---
+"claude-dev": patch
+---
+
+Minor UX improvement to drag and drop ux

.changeset/khaki-cooks-walk.md

@@ -1,5 +0,0 @@
----
-"claude-dev": minor
----
-
-update ordering of messages during task restore

.changeset/kind-pears-invent.md

@@ -0,0 +1,5 @@
+---
+"claude-dev": patch
+---
+
+Add dependsOn to more blocks in the tasks.json

.changeset/long-cheetahs-love.md

@@ -0,0 +1,5 @@
+---
+"claude-dev": patch
+---
+
+Adding args to allow Cursor to open workspaces (for checkpoint testing/development)

.changeset/loud-spies-deny.md

@@ -0,0 +1,5 @@
+---
+"claude-dev": minor
+---
+
+add truncation notice when truncating manually

.changeset/nine-rings-shout.md

@@ -0,0 +1,5 @@
+---
+"claude-dev": minor
+---
+
+add checkpoints after more messages

.changeset/ninety-numbers-decide.md

@@ -0,0 +1,5 @@
+---
+"claude-dev": patch
+---
+
+Introduce UI library for future UI development

.changeset/rare-ladybugs-talk.md

@@ -2,4 +2,4 @@
 "claude-dev": minor
 ---
 
-added IS_TEST build flag
+add newrule slash command

.changeset/serious-baboons-train.md

@@ -0,0 +1,5 @@
+---
+"claude-dev": patch
+---
+
+protobus migration for openImage

.changeset/shy-buckets-itch.md

@@ -0,0 +1,5 @@
+---
+"claude-dev": minor
+---
+
+add cache ui for open router and cline provider

.changeset/six-hairs-rush.md

@@ -0,0 +1,5 @@
+---
+"claude-dev": minor
+---
+
+showing expanded task by default

.changeset/sweet-ligers-sing.md

@@ -1,5 +0,0 @@
----
-"claude-dev": patch
----
-
-Include model and apiProvider in metadata for cline to read

.changeset/sweet-toes-bake.md

@@ -0,0 +1,5 @@
+---
+"claude-dev": minor
+---
+
+Refactor to not pass a message for showing the MCP View from the servers modal

.changeset/tall-ghosts-help.md

@@ -0,0 +1,5 @@
+---
+"claude-dev": minor
+---
+
+Migrate the addRemoteServer to protobus

.changeset/thick-rice-rush.md

@@ -0,0 +1,5 @@
+---
+"claude-dev": patch
+---
+
+Lowering Gemini cache TTL time

.changeset/thirty-feet-rule.md

@@ -0,0 +1,5 @@
+---
+"claude-dev": patch
+---
+
+Adding UI to show openrouter balance next to provider

.changeset/twenty-comics-enjoy.md

@@ -0,0 +1,5 @@
+---
+"claude-dev": patch
+---
+
+fix cost calculation

.clinerules/cline-overview.md

@@ -8,8 +8,8 @@ Cline is a VSCode extension that provides AI assistance through a combination of
 
 ```mermaid
 graph TB
-    subgraph VSCode Extension Host
-        subgraph Core Extension
+    subgraph VSCodeExtensionHost[VSCode Extension Host]
+        subgraph CoreExtension[Core Extension]
             ExtensionEntry[Extension Entry<br/>src/extension.ts]
             WebviewProvider[WebviewProvider<br/>src/core/webview/index.ts]
             Controller[Controller<br/>src/core/controller/index.ts]
@@ -19,7 +19,7 @@ graph TB
             McpHub[McpHub<br/>src/services/mcp/McpHub.ts]
         end
 
-        subgraph Webview UI
+        subgraph WebviewUI[Webview UI]
             WebviewApp[React App<br/>webview-ui/src/App.tsx]
             ExtStateContext[ExtensionStateContext<br/>webview-ui/src/context/ExtensionStateContext.tsx]
             ReactComponents[React Components]
@@ -30,14 +30,14 @@ graph TB
             CheckpointSystem[Git-based Checkpoints]
         end
 
-        subgraph API Providers
+        subgraph apiProviders[API Providers]
             AnthropicAPI[Anthropic]
             OpenRouterAPI[OpenRouter]
             BedrockAPI[AWS Bedrock]
             OtherAPIs[Other Providers]
         end
 
-        subgraph MCP Servers
+        subgraph MCPServers[MCP Servers]
             ExternalMcpServers[External MCP Servers]
         end
     end
@@ -51,7 +51,7 @@ graph TB
     Task --> SecretsStorage
     Task --> TaskStorage
     Task --> CheckpointSystem
-    Task --> |API Requests| API Providers
+    Task --> |API Requests| apiProviders
     McpHub --> |Connects to| ExternalMcpServers
     Task --> |Uses| McpHub
 
@@ -67,7 +67,7 @@ graph TB
     style ExtStateContext fill:#bbf,stroke:#333,stroke-width:2px
     style WebviewProvider fill:#bfb,stroke:#333,stroke-width:2px
     style McpHub fill:#bfb,stroke:#333,stroke-width:2px
-    style API Providers fill:#fdb,stroke:#333,stroke-width:2px
+    style apiProviders fill:#fdb,stroke:#333,stroke-width:2px
 ```
 
 ## Definitions 

.codespellrc

@@ -0,0 +1,6 @@
+[codespell]
+# Ref: https://github.com/codespell-project/codespell#using-a-config-file
+skip = .git*,*.svg,package-lock.json,*.css,.codespellrc,locales
+check-hidden = true
+ignore-regex = (\b(optIn|isTaller)\b|https://\S+)
+# ignore-words-list =

.github/CODEOWNERS

@@ -1 +1 @@
-*       @saoudrizwan @ocasta181 @NightTrek @pashpashpash @dcbartlett @saito-sv
+*       @saoudrizwan @ocasta181 @NightTrek @pashpashpash @dcbartlett @saito-sv @Garoth

.github/workflows/codespell.yml

@@ -0,0 +1,27 @@
+# Codespell configuration is within .codespellrc
+---
+name: Codespell
+
+on:
+    push:
+        branches: [main]
+    pull_request:
+        branches: [main]
+
+permissions:
+    contents: read
+
+jobs:
+    codespell:
+        name: Check for spelling errors
+        runs-on: ubuntu-latest
+
+        steps:
+            - name: Checkout
+              uses: actions/checkout@v4
+            - name: Annotate locations with typos
+              uses: codespell-project/codespell-problem-matcher@v1
+            - name: Codespell
+              uses: codespell-project/actions-codespell@v2
+              with:
+                  only_warn: 1

.github/workflows/pull-request.yml

@@ -0,0 +1,177 @@
+name: Create Linear Issue on Pull Request
+
+on:
+    workflow_dispatch:
+    pull_request:
+        branches: [main]
+        types: [opened]
+
+permissions:
+    pull-requests: write
+
+jobs:
+    create-linear-issue-on-pull-request:
+        runs-on: ubuntu-latest
+        steps:
+            - name: Check for existing Linear link
+              id: check-linear
+              uses: actions/github-script@v6
+              with:
+                  result-encoding: string
+                  script: |
+                      const pr = context.payload.pull_request;
+                      // 1) PR body
+                      if (/https?:\/\/linear\.app/.test(pr.body||"")) {
+                        return "true";
+                      }
+                      // 2) Any linked GitHub issues?
+                      const res = await github.graphql(
+                        `query($owner:String!,$repo:String!,$prNumber:Int!){
+                           repository(owner:$owner,name:$repo){
+                             pullRequest(number:$prNumber){
+                               closingIssuesReferences(first:10){
+                                 nodes{number}
+                               }
+                             }
+                           }
+                         }`,
+                        {
+                          owner: context.repo.owner,
+                          repo: context.repo.repo,
+                          prNumber: pr.number
+                        }
+                      );
+                      for (const {number} of res.repository.pullRequest.closingIssuesReferences.nodes) {
+                        const comments = await github.rest.issues.listComments({
+                          owner: context.repo.owner,
+                          repo: context.repo.repo,
+                          issue_number: number
+                        });
+                        if (comments.data.some(c=>/https?:\/\/linear\.app/.test(c.body))) {
+                          return "true";
+                        }
+                      }
+                      return "false";
+
+            - name: Find or create Linear issue via GraphQL
+              if: steps.check-linear.outputs.result == 'false'
+              id: linear
+              uses: actions/github-script@v6
+              env:
+                  LINEAR_API_KEY: ${{ secrets.LINEAR_API_KEY }}
+              with:
+                  result-encoding: string
+                  script: |
+                      const API = 'https://api.linear.app/graphql';
+                      const apiKey = process.env.LINEAR_API_KEY;
+
+                      // Check if API key exists
+                      if (!apiKey) {
+                        core.setFailed('LINEAR_API_KEY is not set. Please add it to your repository secrets.');
+                        core.setOutput('error', 'true');
+                        core.setOutput('error-message', 'LINEAR_API_KEY is not set. Please add it to your repository secrets.');
+                        return;
+                      }
+
+                      // Helper to call Linear with error handling
+                      async function gql(q, v) {
+                        try {
+                          const r = await fetch(API, {
+                            method:'POST',
+                            headers:{
+                              'Content-Type':'application/json',
+                              'Authorization': apiKey
+                            },
+                            body: JSON.stringify({ query: q, variables: v })
+                          });
+                          
+                          if (!r.ok) {
+                            throw new Error(`Linear API responded with status ${r.status}: ${await r.text()}`);
+                          }
+                          
+                          const json = await r.json();
+                          
+                          // Check for GraphQL errors
+                          if (json.errors && json.errors.length > 0) {
+                            const errorMessages = json.errors.map(e => e.message).join(', ');
+                            throw new Error(`Linear GraphQL errors: ${errorMessages}`);
+                          }
+                          
+                          return json.data;
+                        } catch (error) {
+                          core.error(`Error calling Linear API: ${error.message}`);
+                          throw error;
+                        }
+                      }
+
+                      try {
+                        // 1) Set team ID
+                        const teamId = "19b9c1b2-5f58-498c-b1bf-23ee8f52a677"
+
+                        // 2) Look for existing issue by PR URL
+                        const pr = context.payload.pull_request;
+                        const searchData = await gql(
+                          `query($team:ID!,$q:String!){
+                              issues(filter: { team: { id: { eq: $team } } attachments: { some: { url: { eq: $q } } } }){nodes{id,url}}
+                            }`,
+                          { team: teamId, q: pr.html_url }
+                        );
+                        let issue = searchData.issues.nodes[0];
+
+                        // 3) Create if missing
+                        if (!issue) {
+                          const createData = await gql(
+                            `mutation($input:IssueCreateInput!){
+                                issueCreate(input:$input){issue{id,url}}
+                              }`,
+                            {
+                              input: {
+                                teamId,
+                                title: `[GITHUB] ${pr.title}`,
+                                description: `${pr.body||''}\n\n${pr.html_url}`,
+                                stateId: "4d9bcba2-6712-47e3-b577-6ec1ee023dc2",
+                                labelIds: ["504e7d60-5037-483f-a9b8-7e298bdf116f"]
+                              }
+                            }
+                          );
+                          issue = createData.issueCreate.issue;
+                        }
+
+                        // Set output for next steps
+                        core.setOutput('linear-issue-url', issue.url);
+                        core.setOutput('error', 'false');
+                      } catch (error) {
+                        core.setOutput('error', 'true');
+                        core.setOutput('error-message', error.message);
+                        core.setFailed(`Failed to create or find Linear issue: ${error.message}`);
+                      }
+
+            - name: Comment PR with Linear link
+              if: steps.check-linear.outputs.result == 'false'
+              uses: actions/github-script@v6
+              with:
+                  script: |
+                      const pr = context.payload.pull_request;
+                      const url = `${{ steps.linear.outputs.linear-issue-url }}`;
+                      const body = `🔗 Linear issue created: ${url}`;
+                      // Fetch existing comments
+                      const { data: comments } = await github.rest.issues.listComments({
+                        ...context.repo,
+                        issue_number: pr.number
+                      });
+                      const botComment = comments.find(c =>
+                        c.user.type === "Bot" && c.body.startsWith("🔗 Linear issue created:")
+                      );
+                      if (botComment) {
+                        await github.rest.issues.updateComment({
+                          ...context.repo,
+                          comment_id: botComment.id,
+                          body
+                        });
+                      } else {
+                        await github.rest.issues.createComment({
+                          ...context.repo,
+                          issue_number: pr.number,
+                          body
+                        });
+                      }

.github/workflows/test.yml

@@ -23,7 +23,7 @@ jobs:
             - name: Setup Node.js environment
               uses: actions/setup-node@v4
               with:
-                  node-version: 20.15.1
+                  node-version: 22
 
             # Setup Python for coverage script
             - name: Setup Python
@@ -70,19 +70,18 @@ jobs:
               run: npm run format
 
             # Build the extension before running tests
-            - name: Build Extension
-              run: npm run compile
+            - name: Build Tests and Extension
+              run: npm run pretest
 
-            # Disabling due to compatability with test framework and ESM modules
-            # - name: Unit Tests
-            #   run: npm run test:unit
+            - name: Unit Tests
+              run: npm run test:unit
 
             # Run extension tests with coverage
             - name: Extension Tests with Coverage
               id: extension_coverage
               continue-on-error: true
               run: |
-                  xvfb-run -a npm run test:coverage > extension_coverage.txt 2>&1 || true
+                  xvfb-run -a npm run test:coverage > extension_coverage.txt 2>&1
                   PYTHONPATH=.github/scripts python -m coverage_check extract-coverage extension_coverage.txt --type=extension --github-output --verbose
 
             # Run webview tests with coverage
@@ -107,6 +106,18 @@ jobs:
                       webview-ui/webview_coverage.txt
                   retention-period: workflow # Artifacts are automatically deleted when the workflow completes
 
+            # Set the check as failed if any of the tests failed
+            - name: Check for test failures
+              run: |
+                  # Check if any of the test steps failed
+                  # https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/accessing-contextual-information-about-workflow-runs#steps-context
+                  if [ "${{ steps.extension_coverage.outcome }}" != "success" ] || [ "${{ steps.webview_coverage.outcome }}" != "success" ]; then
+                      echo "Tests failed."
+                      cat extension_coverage.txt
+                      cat webview-ui/webview_coverage.txt
+                      exit 1
+                  fi
+
     coverage:
         needs: test
         runs-on: ubuntu-latest
@@ -132,7 +143,7 @@ jobs:
             - name: Setup Node.js environment
               uses: actions/setup-node@v4
               with:
-                  node-version: 20.15.1
+                  node-version: 22
 
             # Cache root dependencies - only reuse if package-lock.json exactly matches
             - name: Cache root dependencies

.gitignore

@@ -17,3 +17,5 @@ pnpm-lock.yaml
 coverage
 # But don't ignore the coverage scripts in .github/scripts/
 !.github/scripts/coverage/
+
+*evals.env

.mocharc.json

@@ -1,6 +1,6 @@
 {
 	"extension": ["ts"],
 	"spec": "src/**/__tests__/*.ts",
-	"require": ["ts-node/register", "source-map-support/register"],
+	"require": ["ts-node/register", "source-map-support/register", "./src/test/requires.ts"],
 	"recursive": true
 }

.vscode-test.mjs

@@ -6,6 +6,10 @@ export default defineConfig({
 	mocha: {
 		ui: "bdd",
 		timeout: 20000, // Maximum time (in ms) that a test can run before failing
+		/** Set up alias path resolution during tests
+		 * @See {@link file://./test-setup.js}
+		 */
+		require: ["./test-setup.js"],
 	},
 	workspaceFolder: "test-workspace",
 	version: "stable",

.vscode/launch.json

@@ -9,26 +9,13 @@
 			"name": "Run Extension",
 			"type": "extensionHost",
 			"request": "launch",
-			"args": ["--extensionDevelopmentPath=${workspaceFolder}"],
+			"args": ["--extensionDevelopmentPath=${workspaceFolder}", "--disable-workspace-trust", "${workspaceFolder}"],
 			"outFiles": ["${workspaceFolder}/dist/**/*.js"],
 			"preLaunchTask": "${defaultBuildTask}",
 			"env": {
 				"IS_DEV": "true",
 				"DEV_WORKSPACE_FOLDER": "${workspaceFolder}"
 			}
-		},
-		{
-			"name": "Run Extension (Test Mode)",
-			"type": "extensionHost",
-			"request": "launch",
-			"args": ["--extensionDevelopmentPath=${workspaceFolder}"],
-			"outFiles": ["${workspaceFolder}/dist/**/*.js"],
-			"preLaunchTask": "${defaultBuildTask}",
-			"env": {
-				"IS_DEV": "true",
-				"IS_TEST": "true",
-				"DEV_WORKSPACE_FOLDER": "${workspaceFolder}"
-			}
 		}
 	]
 }

.vscode/tasks.json

@@ -4,10 +4,25 @@
 	"version": "2.0.0",
 	"tasks": [
 		{
-			"label": "watch",
-			"dependsOn": ["npm: build:webview", "npm: dev:webview", "npm: watch:tsc", "npm: watch:esbuild"],
+			"label": "npm: protos",
+			"type": "npm",
+			"script": "protos",
+			"problemMatcher": [],
+			"isBackground": false,
 			"presentation": {
-				"reveal": "never"
+				"reveal": "always"
+			},
+			"options": {
+				"env": {
+					"IS_DEV": "true"
+				}
+			}
+		},
+		{
+			"label": "watch",
+			"dependsOn": ["npm: protos", "npm: build:webview", "npm: dev:webview", "npm: watch:tsc", "npm: watch:esbuild"],
+			"presentation": {
+				"reveal": "always"
 			},
 			"group": {
 				"kind": "build",
@@ -16,9 +31,15 @@
 		},
 		{
 			"label": "watch:test",
-			"dependsOn": ["npm: build:webview:test", "npm: dev:webview", "npm: watch:tsc", "npm: watch:esbuild:test"],
+			"dependsOn": [
+				"npm: protos",
+				"npm: build:webview:test",
+				"npm: dev:webview",
+				"npm: watch:tsc",
+				"npm: watch:esbuild:test"
+			],
 			"presentation": {
-				"reveal": "never"
+				"reveal": "always"
 			},
 			"group": "build"
 		},
@@ -29,10 +50,10 @@
 			"problemMatcher": [],
 			"isBackground": true,
 			"label": "npm: build:webview",
+			"dependsOn": ["npm: protos"],
 			"presentation": {
 				"group": "watch",
-				"reveal": "never",
-				"close": true
+				"reveal": "always"
 			},
 			"options": {
 				"env": {
@@ -47,10 +68,10 @@
 			"problemMatcher": [],
 			"isBackground": true,
 			"label": "npm: build:webview:test",
+			"dependsOn": ["npm: protos"],
 			"presentation": {
 				"group": "watch",
-				"reveal": "never",
-				"close": true
+				"reveal": "always"
 			},
 			"options": {
 				"env": {
@@ -82,10 +103,10 @@
 			],
 			"isBackground": true,
 			"label": "npm: dev:webview",
+			"dependsOn": ["npm: protos"],
 			"presentation": {
 				"group": "watch",
-				"reveal": "never",
-				"close": true
+				"reveal": "always"
 			},
 			"options": {
 				"env": {
@@ -100,10 +121,10 @@
 			"problemMatcher": "$esbuild-watch",
 			"isBackground": true,
 			"label": "npm: watch:esbuild",
+			"dependsOn": ["npm: protos"],
 			"presentation": {
 				"group": "watch",
-				"reveal": "never",
-				"close": true
+				"reveal": "always"
 			},
 			"options": {
 				"env": {
@@ -118,10 +139,10 @@
 			"problemMatcher": "$esbuild-watch",
 			"isBackground": true,
 			"label": "npm: watch:esbuild:test",
+			"dependsOn": ["npm: protos"],
 			"presentation": {
 				"group": "watch",
-				"reveal": "never",
-				"close": true
+				"reveal": "always"
 			},
 			"options": {
 				"env": {
@@ -137,10 +158,10 @@
 			"problemMatcher": "$tsc-watch",
 			"isBackground": true,
 			"label": "npm: watch:tsc",
+			"dependsOn": ["npm: protos"],
 			"presentation": {
 				"group": "watch",
-				"reveal": "never",
-				"close": true
+				"reveal": "always"
 			}
 		},
 		{
@@ -148,15 +169,16 @@
 			"script": "watch-tests",
 			"problemMatcher": "$tsc-watch",
 			"isBackground": true,
+			"dependsOn": ["npm: protos"],
 			"presentation": {
-				"reveal": "never",
+				"reveal": "always",
 				"group": "watchers"
 			},
 			"group": "build"
 		},
 		{
 			"label": "tasks: watch-tests",
-			"dependsOn": ["npm: watch", "npm: watch-tests"],
+			"dependsOn": ["npm: protos", "npm: watch", "npm: watch-tests"],
 			"problemMatcher": []
 		},
 		{

CHANGELOG.md

@@ -1,10 +1,92 @@
 # Changelog
 
+## [3.13.3]
+
+-   Add download counts to MCP marketplace items
+-   Add `/compact` command
+-   Add prompt caching to gemini models in cline / openrouter providers
+-   Add tooltips to bottom row menu
+
+## [3.13.2]
+
+-   Add Gemini 2.5 Flash model to Vertex and Gemini Providers (Thanks monotykamary!)
+-   Add Caching to gemini provider (Thanks arafatkatze!)
+-   Add thinking budget support to Gemini Models (Thanks monotykamary!)
+-   Add !include .file directive support for .clineignore (Thanks watany-dev!)
+-   Improve slash command functionality
+-   Improve prompting for new task tool
+-   Fix o1 temperature being passed to the azure api (Thanks treeleaves30760!)
+-   Fix to make "add new rule file" button functional
+-   Fix Ollama provider timeout, allowing for a larger loading time (Thanks suvarchal!)
+-   Fix Non-UTF-8 File Handling: Improve Encoding Detection to Prevent Garbled Text and Binary Misclassification (Thanks yt3trees!)
+-   Fix settings to not reset by changing providers
+-   Fix terminal outputs missing commas
+-   Fix terminal errors caused by starting non-alphanumeric outputs
+-   Fix auto approve settings becoming unset
+-   Fix Mermaid syntax error in documentation (Thanks tuki0918!)
+-   Remove supportsComputerUse restriction and support browser use through any model that supports images (Thanks arafatkatze!)
+
+## [3.13.1]
+
+-   Fix bug where task cancellation during thinking stream would result in error state
+
+## [3.13.0]
+
+-   Add Cline rules popover under the chat field, allowing you to easily add, enable & disable workspace level or global rule files
+-   Add new slash command menu letting you type “/“ to do quick actions like creating new tasks
+-   Add ability to edit past messages, with options to restore your workspace back to that point
+-   Allow sending a message when selecting an option provided by the question or plan tool
+-   Add command to jump to Cline's chat input
+-   Add support for OpenAI o3 & 4o-mini (Thanks @PeterDaveHello and @arafatkatze!)
+-   Add baseURL option for Google Gemini provider (Thanks @owengo and @olivierhub!)
+-   Add support for Azure's DeepSeek model. (Thanks @yt3trees!)
+-   Add ability for models that support it to receive image responses from MCP servers (Thanks @rikaaa0928!)
+-   Improve search and replace diff editing by making it more flexible with models that fail to follow structured output instructions. (Thanks @chi-cat!)
+-   Add detection of Ctrl+C termination in terminal, improving output reading issues
+-   Fix issue where some commands with large output would cause UI to freeze
+-   Fix token usage tracking issues with vertex provider (Thanks @mzsima!)
+-   Fix issue with xAI reasoning content not being parsed (Thanks @mrubens!)
+
+## [3.12.3]
+
+-   Add copy button to MermaidBlock component (Thanks @cacosub7!)
+-   Add the ability to fetch from global cline rules files
+-   Add icon to indicate when a file outside of the users workspace is edited
+
+## [3.12.2]
+
+-   Add gpt-4.1
+
+## [3.12.1]
+
+-   Use visual checkpoint indicator to make it clear when checkpoints are created
+-   Big shoutout to @samuel871211 for numerous code quality improvements, refactoring contributions, and webview performance improvements!
+-   Use improved context manager
+
+## [3.12.0]
+
+-   Add favorite toggles for models when using the Cline & OpenRouter providers
+-   Add auto-approve options for edits/reads outside of the workspace
+-   Improve diff editing animation for large files
+-   Add indicator showing number of diff edits when Cline edits a file
+-   Add streaming support and reasoning effort option to xAI's Grok 3 Mini
+-   Add settings button to MCP popover to easily modify installed servers
+-   Fix bug where browser tool actions would show unparsed results in the chat view
+-   Fix issue with new checkpoints popover hiding too quickly
+-   Fix duplicate checkpoints bug
+-   Improve Ollama provider with retry mechanism, timeout handling, and improved error handling (thanks suvarchal!)
+
+## [3.11.0]
+
+-   Redesign checkpoint UI to declutter chat view by using a subtle indicator line that expands to a popover on hover, with a new date indicator for when it was created
+-   Add support for xAI's provider's Grok 3 models
+-   Add more robust error tracking for users opted in to telemetry (thank you for helping us make Cline better!)
+
 ## [3.10.1]
 
 -   Add CMD+' keyboard shortcut to add selected text to Cline
 -   Cline now auto focuses the text field when using 'Add to Cline' shortcut
--   Add new 'Create New Task' tool to let Cline start a new task autonomously! 
+-   Add new 'Create New Task' tool to let Cline start a new task autonomously!
 -   Fix Mermaid diagram issues
 -   Fix Gemini provider cost calculation to take new tiered pricing structure into account
 
@@ -23,7 +105,7 @@
 -   Add recommended models for Cline provider
 -   Add ability to detect when user edits files manually so Cline knows to re-read, leading to reduced diff edit errors
 -   Add improvements to file mention searching for faster searching
--   Add scoring logic to file mentions to sort and exlcude results based on relevance
+-   Add scoring logic to file mentions to sort and exclude results based on relevance
 -   Add Support for Bytedance Doubao (Thanks Tunixer!)
 -   Fix to prevent duplicate BOM (Thanks bamps53!)
 
@@ -501,7 +583,7 @@
 -   Adds "Always allow read-only operations" setting to let Claude read files and view directories without needing approval (off by default)
 -   Implement sliding window context management to keep tasks going past 200k tokens
 -   Adds Google Cloud Vertex AI support and updates Claude 3.5 Sonnet max output to 8192 tokens for all providers.
--   Improves system prompt to gaurd against lazy edits (less "//rest of code here")
+-   Improves system prompt to guard against lazy edits (less "//rest of code here")
 
 ## [1.3.0]
 

esbuild.js

@@ -4,11 +4,64 @@ const path = require("path")
 
 const production = process.argv.includes("--production")
 const watch = process.argv.includes("--watch")
-const test = process.env.IS_TEST === "true"
 
 /**
  * @type {import('esbuild').Plugin}
  */
+const aliasResolverPlugin = {
+	name: "alias-resolver",
+	setup(build) {
+		const aliases = {
+			"@": path.resolve(__dirname, "src"),
+			"@api": path.resolve(__dirname, "src/api"),
+			"@core": path.resolve(__dirname, "src/core"),
+			"@integrations": path.resolve(__dirname, "src/integrations"),
+			"@services": path.resolve(__dirname, "src/services"),
+			"@shared": path.resolve(__dirname, "src/shared"),
+			"@utils": path.resolve(__dirname, "src/utils"),
+			"@packages": path.resolve(__dirname, "src/packages"),
+		}
+
+		// For each alias entry, create a resolver
+		Object.entries(aliases).forEach(([alias, aliasPath]) => {
+			const aliasRegex = new RegExp(`^${alias}($|/.*)`)
+			build.onResolve({ filter: aliasRegex }, (args) => {
+				const importPath = args.path.replace(alias, aliasPath)
+
+				// First, check if the path exists as is
+				if (fs.existsSync(importPath)) {
+					const stats = fs.statSync(importPath)
+					if (stats.isDirectory()) {
+						// If it's a directory, try to find index files
+						const extensions = [".ts", ".tsx", ".js", ".jsx"]
+						for (const ext of extensions) {
+							const indexFile = path.join(importPath, `index${ext}`)
+							if (fs.existsSync(indexFile)) {
+								return { path: indexFile }
+							}
+						}
+					} else {
+						// It's a file that exists, so return it
+						return { path: importPath }
+					}
+				}
+
+				// If the path doesn't exist, try appending extensions
+				const extensions = [".ts", ".tsx", ".js", ".jsx"]
+				for (const ext of extensions) {
+					const pathWithExtension = `${importPath}${ext}`
+					if (fs.existsSync(pathWithExtension)) {
+						return { path: pathWithExtension }
+					}
+				}
+
+				// If nothing worked, return the original path and let esbuild handle the error
+				return { path: importPath }
+			})
+		})
+	},
+}
+
 const esbuildProblemMatcherPlugin = {
 	name: "esbuild-problem-matcher",
 
@@ -71,10 +124,10 @@ const extensionConfig = {
 	logLevel: "silent",
 	define: {
 		"process.env.IS_DEV": JSON.stringify(!production),
-		"process.env.IS_TEST": JSON.stringify(test),
 	},
 	plugins: [
 		copyWasmFiles,
+		aliasResolverPlugin,
 		/* add to the end of plugins array */
 		esbuildProblemMatcherPlugin,
 		{

evals/.gitignore

@@ -0,0 +1,3 @@
+repositories
+
+results/evals.db

evals/README.md

@@ -0,0 +1,186 @@
+# Cline Evaluation System
+
+This directory contains the evaluation system for benchmarking Cline against various coding evaluation frameworks.
+
+## Overview
+
+The Cline Evaluation System allows you to:
+
+1. Run Cline against standardized coding benchmarks
+2. Collect comprehensive metrics on performance
+3. Generate detailed reports on evaluation results
+4. Compare performance across different models and benchmarks
+
+## Architecture
+
+The evaluation system consists of two main components:
+
+1. **Test Server**: Enhanced HTTP server in `src/services/test/TestServer.ts` that provides detailed task results
+2. **CLI Tool**: Command-line interface in `evals/cli/` for orchestrating evaluations
+
+## Directory Structure
+
+```
+cline-repo/
+├── src/
+│   ├── services/
+│   │   ├── test/
+│   │   │   ├── TestServer.ts         # Enhanced HTTP server for task execution
+│   │   │   ├── GitHelper.ts          # Git utilities for file tracking
+│   │   │   └── ...
+│   │   └── ...
+│   └── ...
+├── evals/                            # Main directory for evaluation system
+│   ├── cli/                          # CLI tool for orchestrating evaluations
+│   │   ├── src/
+│   │   │   ├── index.ts              # CLI entry point
+│   │   │   ├── commands/             # CLI commands (setup, run, report)
+│   │   │   ├── adapters/             # Benchmark adapters
+│   │   │   ├── db/                   # Database management
+│   │   │   └── utils/                # Utility functions
+│   │   ├── package.json
+│   │   └── tsconfig.json
+│   ├── repositories/                 # Cloned benchmark repositories
+│   │   ├── exercism/                 # Modified Exercism (from pashpashpash/evals)
+│   │   ├── swe-bench/                # SWE-Bench repository
+│   │   ├── swelancer/                # SWELancer repository
+│   │   └── multi-swe/                # Multi-SWE-Bench repository
+│   ├── results/                      # Evaluation results storage
+│   │   ├── runs/                     # Individual run results
+│   │   └── reports/                  # Generated reports
+│   └── README.md                     # This file
+└── ...
+```
+
+## Getting Started
+
+### Prerequisites
+
+- Node.js 16+
+- VSCode with Cline extension installed
+- Git
+
+### Activation Mechanism
+
+The evaluation system uses an `evals.env` file approach to activate test mode in the Cline extension. When an evaluation is run:
+
+1. The CLI creates an `evals.env` file in the workspace directory
+2. The Cline extension activates due to the `workspaceContains:evals.env` activation event
+3. The extension detects this file and automatically enters test mode
+4. After evaluation completes, the file is automatically removed
+
+This approach eliminates the need for environment variables during the build process and allows for targeted activation only when needed for evaluations. The extension remains dormant during normal use, only activating when an evals.env file is present. For more details, see [Evals Env Activation](./docs/evals-env-activation.md).
+
+### Installation
+
+1. Build the CLI tool:
+
+```bash
+cd evals/cli
+npm install
+npm run build
+```
+
+### Usage
+
+#### Setting Up Benchmarks
+
+```bash
+cd evals/cli
+node dist/index.js setup
+```
+
+This will clone and set up all benchmark repositories. You can specify specific benchmarks:
+
+```bash
+node dist/index.js setup --benchmarks exercism
+```
+
+#### Running Evaluations
+
+```bash
+node dist/index.js run --model claude-3-opus-20240229 --benchmark exercism
+```
+
+Options:
+- `--model`: The model to evaluate (default: claude-3-opus-20240229)
+- `--benchmark`: Specific benchmark to run (default: all)
+- `--count`: Number of tasks to run (default: all)
+
+#### Generating Reports
+
+```bash
+node dist/index.js report
+```
+
+Options:
+- `--format`: Report format (json, markdown) (default: markdown)
+- `--output`: Output path for the report
+
+#### Managing Test Mode Activation
+
+The CLI provides a command to manually manage the evals.env file for test mode activation:
+
+```bash
+node dist/index.js evals-env create  # Create evals.env file in current directory
+node dist/index.js evals-env remove  # Remove evals.env file from current directory
+node dist/index.js evals-env check   # Check if evals.env file exists in current directory
+```
+
+Options:
+- `--directory`: Specify a directory other than the current one
+
+## Benchmarks
+
+### Exercism
+
+Modified Exercism exercises from the [pashpashpash/evals](https://github.com/pashpashpash/evals) repository. These are small, focused programming exercises in various languages.
+
+### SWE-Bench (Coming Soon)
+
+Real-world software engineering tasks from the [SWE-bench](https://github.com/SWE-bench/SWE-bench) repository.
+
+### SWELancer (Coming Soon)
+
+Freelance-style programming tasks from the SWELancer benchmark.
+
+### Multi-SWE-Bench (Coming Soon)
+
+Multi-file software engineering tasks from the Multi-SWE-Bench repository.
+
+## Metrics
+
+The evaluation system collects the following metrics:
+
+- **Token Usage**: Input and output tokens
+- **Cost**: Estimated cost of API calls
+- **Duration**: Time taken to complete tasks
+- **Tool Usage**: Number of tool calls and failures
+- **Success Rate**: Percentage of tasks completed successfully
+- **Functional Correctness**: Percentage of tests passed
+
+## Reports
+
+Reports are generated in Markdown or JSON format and include:
+
+- Overall summary
+- Benchmark-specific results
+- Model-specific results
+- Tool usage statistics
+- Charts and visualizations
+
+## Development
+
+### Adding a New Benchmark
+
+1. Create a new adapter in `evals/cli/src/adapters/`
+2. Implement the `BenchmarkAdapter` interface
+3. Register the adapter in `evals/cli/src/adapters/index.ts`
+
+### Extending Metrics
+
+To add new metrics:
+
+1. Update the database schema in `evals/cli/src/db/schema.ts`
+2. Add collection logic in `evals/cli/src/utils/results.ts`
+3. Update report generation in `evals/cli/src/commands/report.ts`

evals/cli/package.json

@@ -0,0 +1,39 @@
+{
+	"name": "cline-evaluation-cli",
+	"version": "0.1.0",
+	"description": "CLI tool for orchestrating Cline evaluations across multiple benchmarks",
+	"main": "dist/index.js",
+	"scripts": {
+		"build": "tsc",
+		"start": "node dist/index.js",
+		"dev": "ts-node src/index.ts",
+		"test": "echo \"Error: no test specified\" && exit 1"
+	},
+	"keywords": [
+		"cline",
+		"evaluation",
+		"benchmark"
+	],
+	"author": "",
+	"license": "MIT",
+	"dependencies": {
+		"better-sqlite3": "^8.0.0",
+		"chalk": "^4.1.2",
+		"commander": "^9.4.1",
+		"execa": "^5.1.1",
+		"node-fetch": "^2.7.0",
+		"ora": "^5.4.1",
+		"sqlite": "^4.1.2",
+		"uuid": "^9.0.0",
+		"yargs": "^17.6.2"
+	},
+	"devDependencies": {
+		"@types/better-sqlite3": "^7.6.3",
+		"@types/node": "^18.11.18",
+		"@types/node-fetch": "^2.6.12",
+		"@types/uuid": "^9.0.0",
+		"@types/yargs": "^17.0.19",
+		"ts-node": "^10.9.1",
+		"typescript": "^4.9.4"
+	}
+}

evals/cli/src/adapters/exercism.ts

@@ -0,0 +1,194 @@
+import * as path from "path"
+import * as fs from "fs"
+import execa from "execa"
+import { BenchmarkAdapter, Task, VerificationResult } from "./types"
+
+const EVALS_DIR = path.resolve(__dirname, "../../../")
+
+/**
+ * Adapter for the modified Exercism benchmark
+ */
+export class ExercismAdapter implements BenchmarkAdapter {
+	name = "exercism"
+
+	/**
+	 * Set up the Exercism benchmark repository
+	 */
+	async setup(): Promise<void> {
+		// Clone repository if needed
+		const exercismDir = path.join(EVALS_DIR, "repositories", "exercism")
+
+		if (!fs.existsSync(exercismDir)) {
+			console.log(`Cloning Exercism repository to ${exercismDir}...`)
+			await execa("git", ["clone", "https://github.com/pashpashpash/evals.git", exercismDir])
+			console.log("Exercism repository cloned successfully")
+		} else {
+			console.log(`Exercism repository already exists at ${exercismDir}`)
+
+			// Pull latest changes
+			console.log("Pulling latest changes...")
+			await execa("git", ["pull"], { cwd: exercismDir })
+			console.log("Repository updated successfully")
+		}
+	}
+
+	/**
+	 * List all available tasks in the Exercism benchmark
+	 */
+	async listTasks(): Promise<Task[]> {
+		const tasks: Task[] = []
+		const exercisesDir = path.join(EVALS_DIR, "repositories", "exercism")
+
+		// Ensure the repository exists
+		if (!fs.existsSync(exercisesDir)) {
+			throw new Error(`Exercism repository not found at ${exercisesDir}. Run setup first.`)
+		}
+
+		// Read language directories
+		const languages = fs
+			.readdirSync(exercisesDir)
+			.filter((dir) => fs.statSync(path.join(exercisesDir, dir)).isDirectory())
+			.filter((dir) => !dir.startsWith(".") && !["node_modules", ".git"].includes(dir))
+
+		for (const language of languages) {
+			const languageDir = path.join(exercisesDir, language)
+
+			// Read exercise directories
+			const exercises = fs.readdirSync(languageDir).filter((dir) => fs.statSync(path.join(languageDir, dir)).isDirectory())
+
+			for (const exercise of exercises) {
+				const exerciseDir = path.join(languageDir, exercise)
+
+				// Read instructions
+				let description = ""
+				const instructionsPath = path.join(exerciseDir, "docs", "instructions.md")
+				if (fs.existsSync(instructionsPath)) {
+					description = fs.readFileSync(instructionsPath, "utf-8")
+				}
+
+				// Determine test commands based on language
+				let testCommands: string[] = []
+				switch (language) {
+					case "javascript":
+						testCommands = ["npm install", "npm test"]
+						break
+					case "python":
+						testCommands = ["python -m pytest -o markers=task *_test.py"]
+						break
+					case "go":
+						testCommands = ["go test"]
+						break
+					case "java":
+						testCommands = ["./gradlew test"]
+						break
+					case "rust":
+						testCommands = ["cargo test"]
+						break
+					default:
+						testCommands = []
+				}
+
+				tasks.push({
+					id: `exercism-${language}-${exercise}`,
+					name: exercise,
+					description,
+					workspacePath: exerciseDir,
+					setupCommands: [],
+					verificationCommands: testCommands,
+					metadata: {
+						language,
+						type: "exercism",
+					},
+				})
+			}
+		}
+
+		return tasks
+	}
+
+	/**
+	 * Prepare a specific task for execution
+	 * @param taskId The ID of the task to prepare
+	 */
+	async prepareTask(taskId: string): Promise<Task> {
+		const tasks = await this.listTasks()
+		const task = tasks.find((t) => t.id === taskId)
+
+		if (!task) {
+			throw new Error(`Task ${taskId} not found`)
+		}
+
+		// Check if Git repository is already initialized
+		const gitDirExists = fs.existsSync(path.join(task.workspacePath, ".git"))
+
+		try {
+			// Initialize Git repository if needed
+			if (!gitDirExists) {
+				await execa("git", ["init"], { cwd: task.workspacePath })
+			}
+
+			// Create a dummy file to ensure there's something to commit
+			const dummyFilePath = path.join(task.workspacePath, ".eval-timestamp")
+			fs.writeFileSync(dummyFilePath, new Date().toISOString())
+
+			// Add all files and commit
+			await execa("git", ["add", "."], { cwd: task.workspacePath })
+
+			try {
+				await execa("git", ["commit", "-m", "Initial commit"], { cwd: task.workspacePath })
+			} catch (error: any) {
+				// If commit fails because there are no changes, that's okay
+				if (!error.stderr?.includes("nothing to commit")) {
+					throw error
+				}
+			}
+		} catch (error: any) {
+			console.warn(`Warning: Git operations failed: ${error.message}`)
+			console.warn("Continuing without Git initialization")
+		}
+
+		return task
+	}
+
+	/**
+	 * Verify the result of a task execution
+	 * @param task The task that was executed
+	 * @param result The result of the task execution
+	 */
+	async verifyResult(task: Task, result: any): Promise<VerificationResult> {
+		// Run verification commands
+		let success = true
+		let output = ""
+
+		for (const command of task.verificationCommands) {
+			try {
+				const [cmd, ...args] = command.split(" ")
+				const { stdout } = await execa(cmd, args, { cwd: task.workspacePath })
+				output += stdout + "\n"
+			} catch (error: any) {
+				success = false
+				if (error.stdout) {
+					output += error.stdout + "\n"
+				}
+				if (error.stderr) {
+					output += error.stderr + "\n"
+				}
+			}
+		}
+
+		// Parse test results
+		const testsPassed = (output.match(/PASS/g) || []).length
+		const testsFailed = (output.match(/FAIL/g) || []).length
+		const testsTotal = testsPassed + testsFailed
+
+		return {
+			success,
+			metrics: {
+				testsPassed,
+				testsFailed,
+				testsTotal,
+				functionalCorrectness: testsTotal > 0 ? testsPassed / testsTotal : 0,
+			},
+		}
+	}
+}

evals/cli/src/adapters/index.ts

@@ -0,0 +1,47 @@
+import { BenchmarkAdapter } from "./types"
+import { ExercismAdapter } from "./exercism"
+import { SWEBenchAdapter } from "./swe-bench"
+import { SWELancerAdapter } from "./swelancer"
+import { MultiSWEAdapter } from "./multi-swe"
+
+// Registry of all available adapters
+const adapters: Record<string, BenchmarkAdapter> = {
+	// Exercism is the primary adapter with real implementation
+	exercism: new ExercismAdapter(),
+
+	// Dummy adapters for testing
+	"swe-bench": new SWEBenchAdapter(),
+	swelancer: new SWELancerAdapter(),
+	"multi-swe": new MultiSWEAdapter(),
+}
+
+/**
+ * Get a specific adapter by name
+ * @param name The name of the adapter to get
+ * @returns The requested adapter
+ * @throws Error if the adapter is not found
+ */
+export function getAdapter(name: string): BenchmarkAdapter {
+	const adapter = adapters[name]
+	if (!adapter) {
+		throw new Error(`Adapter for benchmark '${name}' not found`)
+	}
+	return adapter
+}
+
+/**
+ * Get all available adapters
+ * @returns Array of all registered adapters
+ */
+export function getAllAdapters(): BenchmarkAdapter[] {
+	return Object.values(adapters)
+}
+
+/**
+ * Register a new adapter
+ * @param name The name to register the adapter under
+ * @param adapter The adapter to register
+ */
+export function registerAdapter(name: string, adapter: BenchmarkAdapter): void {
+	adapters[name] = adapter
+}

evals/cli/src/adapters/multi-swe.ts

@@ -0,0 +1,192 @@
+import * as path from "path"
+import * as fs from "fs"
+import execa from "execa"
+import { BenchmarkAdapter, Task, VerificationResult } from "./types"
+
+const EVALS_DIR = path.resolve(__dirname, "../../../")
+
+/**
+ * Dummy adapter for the Multi-SWE-Bench benchmark
+ */
+export class MultiSWEAdapter implements BenchmarkAdapter {
+	name = "multi-swe"
+
+	/**
+	 * Set up the Multi-SWE-Bench benchmark repository (dummy implementation)
+	 */
+	async setup(): Promise<void> {
+		console.log("Multi-SWE-Bench dummy setup completed")
+
+		// Create repositories directory if it doesn't exist
+		const repoDir = path.join(EVALS_DIR, "repositories", "multi-swe")
+		if (!fs.existsSync(repoDir)) {
+			fs.mkdirSync(repoDir, { recursive: true })
+			console.log(`Created dummy Multi-SWE-Bench directory at ${repoDir}`)
+		}
+	}
+
+	/**
+	 * List all available tasks in the Multi-SWE-Bench benchmark (dummy implementation)
+	 */
+	async listTasks(): Promise<Task[]> {
+		return [
+			{
+				id: "multi-swe-task-1",
+				name: "Multi-Language API Integration",
+				description:
+					"Implement a system that integrates a Python backend with a TypeScript frontend and a Rust processing service.",
+				workspacePath: path.join(EVALS_DIR, "repositories", "multi-swe"),
+				setupCommands: [],
+				verificationCommands: [],
+				metadata: {
+					languages: ["python", "typescript", "rust"],
+					complexity: "high",
+					type: "multi-swe",
+				},
+			},
+			{
+				id: "multi-swe-task-2",
+				name: "Cross-Platform Mobile App",
+				description: "Create a cross-platform mobile app using React Native with native modules in Swift and Kotlin.",
+				workspacePath: path.join(EVALS_DIR, "repositories", "multi-swe"),
+				setupCommands: [],
+				verificationCommands: [],
+				metadata: {
+					languages: ["javascript", "swift", "kotlin"],
+					complexity: "medium",
+					type: "multi-swe",
+				},
+			},
+			{
+				id: "multi-swe-task-3",
+				name: "Microservice Architecture",
+				description: "Design and implement a microservice architecture with services written in Go, Node.js, and Java.",
+				workspacePath: path.join(EVALS_DIR, "repositories", "multi-swe"),
+				setupCommands: [],
+				verificationCommands: [],
+				metadata: {
+					languages: ["go", "javascript", "java"],
+					complexity: "high",
+					type: "multi-swe",
+				},
+			},
+		]
+	}
+
+	/**
+	 * Prepare a specific task for execution (dummy implementation)
+	 * @param taskId The ID of the task to prepare
+	 */
+	async prepareTask(taskId: string): Promise<Task> {
+		const tasks = await this.listTasks()
+		const task = tasks.find((t) => t.id === taskId)
+
+		if (!task) {
+			throw new Error(`Task ${taskId} not found`)
+		}
+
+		// Create a dummy workspace for the task
+		const taskDir = path.join(task.workspacePath, taskId)
+		if (!fs.existsSync(taskDir)) {
+			fs.mkdirSync(taskDir, { recursive: true })
+
+			// Create a dummy file for the task
+			fs.writeFileSync(
+				path.join(taskDir, "README.md"),
+				`# ${task.name}\n\n${task.description}\n\nThis is a dummy task for testing purposes.`,
+			)
+
+			// Create additional dummy files based on task type
+			if (task.id === "multi-swe-task-1") {
+				// Python backend
+				fs.mkdirSync(path.join(taskDir, "backend"), { recursive: true })
+				fs.writeFileSync(
+					path.join(taskDir, "backend", "app.py"),
+					`# TODO: Implement Python backend\nfrom flask import Flask\n\napp = Flask(__name__)\n\n@app.route('/')\ndef hello():\n    return "Hello, World!"\n`,
+				)
+
+				// TypeScript frontend
+				fs.mkdirSync(path.join(taskDir, "frontend"), { recursive: true })
+				fs.writeFileSync(
+					path.join(taskDir, "frontend", "app.ts"),
+					`// TODO: Implement TypeScript frontend\nconsole.log('Frontend starting...');\n`,
+				)
+
+				// Rust processing service
+				fs.mkdirSync(path.join(taskDir, "processor"), { recursive: true })
+				fs.writeFileSync(
+					path.join(taskDir, "processor", "main.rs"),
+					`// TODO: Implement Rust processing service\nfn main() {\n    println!("Processor starting...");\n}\n`,
+				)
+			} else if (task.id === "multi-swe-task-2") {
+				// React Native app
+				fs.mkdirSync(path.join(taskDir, "app"), { recursive: true })
+				fs.writeFileSync(
+					path.join(taskDir, "app", "App.js"),
+					`// TODO: Implement React Native app\nimport React from 'react';\nimport { View, Text } from 'react-native';\n\nexport default function App() {\n  return (\n    <View>\n      <Text>Hello, World!</Text>\n    </View>\n  );\n}\n`,
+				)
+
+				// Swift native module
+				fs.mkdirSync(path.join(taskDir, "ios"), { recursive: true })
+				fs.writeFileSync(
+					path.join(taskDir, "ios", "NativeModule.swift"),
+					`// TODO: Implement Swift native module\nimport Foundation\n\n@objc(NativeModule)\nclass NativeModule: NSObject {\n  @objc\n  func hello() -> String {\n    return "Hello from Swift"\n  }\n}\n`,
+				)
+
+				// Kotlin native module
+				fs.mkdirSync(path.join(taskDir, "android"), { recursive: true })
+				fs.writeFileSync(
+					path.join(taskDir, "android", "NativeModule.kt"),
+					`// TODO: Implement Kotlin native module\npackage com.example.app\n\nclass NativeModule {\n  fun hello(): String {\n    return "Hello from Kotlin"\n  }\n}\n`,
+				)
+			} else if (task.id === "multi-swe-task-3") {
+				// Go service
+				fs.mkdirSync(path.join(taskDir, "service-go"), { recursive: true })
+				fs.writeFileSync(
+					path.join(taskDir, "service-go", "main.go"),
+					`// TODO: Implement Go service\npackage main\n\nimport "fmt"\n\nfunc main() {\n\tfmt.Println("Go service starting...")\n}\n`,
+				)
+
+				// Node.js service
+				fs.mkdirSync(path.join(taskDir, "service-node"), { recursive: true })
+				fs.writeFileSync(
+					path.join(taskDir, "service-node", "server.js"),
+					`// TODO: Implement Node.js service\nconsole.log('Node.js service starting...');\n`,
+				)
+
+				// Java service
+				fs.mkdirSync(path.join(taskDir, "service-java"), { recursive: true })
+				fs.writeFileSync(
+					path.join(taskDir, "service-java", "Main.java"),
+					`// TODO: Implement Java service\npublic class Main {\n    public static void main(String[] args) {\n        System.out.println("Java service starting...");\n    }\n}\n`,
+				)
+			}
+		}
+
+		// Update the task's workspace path to the task-specific directory
+		return {
+			...task,
+			workspacePath: taskDir,
+		}
+	}
+
+	/**
+	 * Verify the result of a task execution (dummy implementation)
+	 * @param task The task that was executed
+	 * @param result The result of the task execution
+	 */
+	async verifyResult(task: Task, result: any): Promise<VerificationResult> {
+		// Always return success for dummy implementation
+		return {
+			success: true,
+			metrics: {
+				testsPassed: 1,
+				testsFailed: 0,
+				testsTotal: 1,
+				functionalCorrectness: 1.0,
+				crossLanguageIntegration: 0.9, // Dummy metric specific to Multi-SWE
+				architectureQuality: 0.85, // Dummy metric specific to Multi-SWE
+			},
+		}
+	}
+}

evals/cli/src/adapters/swe-bench.ts

@@ -0,0 +1,125 @@
+import * as path from "path"
+import * as fs from "fs"
+import execa from "execa"
+import { BenchmarkAdapter, Task, VerificationResult } from "./types"
+
+const EVALS_DIR = path.resolve(__dirname, "../../../")
+
+/**
+ * Dummy adapter for the SWE-Bench benchmark
+ */
+export class SWEBenchAdapter implements BenchmarkAdapter {
+	name = "swe-bench"
+
+	/**
+	 * Set up the SWE-Bench benchmark repository (dummy implementation)
+	 */
+	async setup(): Promise<void> {
+		console.log("SWE-Bench dummy setup completed")
+
+		// Create repositories directory if it doesn't exist
+		const repoDir = path.join(EVALS_DIR, "repositories", "swe-bench")
+		if (!fs.existsSync(repoDir)) {
+			fs.mkdirSync(repoDir, { recursive: true })
+			console.log(`Created dummy SWE-Bench directory at ${repoDir}`)
+		}
+	}
+
+	/**
+	 * List all available tasks in the SWE-Bench benchmark (dummy implementation)
+	 */
+	async listTasks(): Promise<Task[]> {
+		return [
+			{
+				id: "swe-bench-task-1",
+				name: "Fix React Component Bug",
+				description: "Fix a bug in a React component where the state is not properly updated.",
+				workspacePath: path.join(EVALS_DIR, "repositories", "swe-bench"),
+				setupCommands: [],
+				verificationCommands: [],
+				metadata: {
+					repository: "facebook/react",
+					issue: "#12345",
+					type: "swe-bench",
+				},
+			},
+			{
+				id: "swe-bench-task-2",
+				name: "Optimize Database Query",
+				description: "Optimize a slow database query in a Django application.",
+				workspacePath: path.join(EVALS_DIR, "repositories", "swe-bench"),
+				setupCommands: [],
+				verificationCommands: [],
+				metadata: {
+					repository: "django/django",
+					issue: "#6789",
+					type: "swe-bench",
+				},
+			},
+			{
+				id: "swe-bench-task-3",
+				name: "Fix Memory Leak",
+				description: "Fix a memory leak in a Node.js application.",
+				workspacePath: path.join(EVALS_DIR, "repositories", "swe-bench"),
+				setupCommands: [],
+				verificationCommands: [],
+				metadata: {
+					repository: "nodejs/node",
+					issue: "#9876",
+					type: "swe-bench",
+				},
+			},
+		]
+	}
+
+	/**
+	 * Prepare a specific task for execution (dummy implementation)
+	 * @param taskId The ID of the task to prepare
+	 */
+	async prepareTask(taskId: string): Promise<Task> {
+		const tasks = await this.listTasks()
+		const task = tasks.find((t) => t.id === taskId)
+
+		if (!task) {
+			throw new Error(`Task ${taskId} not found`)
+		}
+
+		// Create a dummy workspace for the task
+		const taskDir = path.join(task.workspacePath, taskId)
+		if (!fs.existsSync(taskDir)) {
+			fs.mkdirSync(taskDir, { recursive: true })
+
+			// Create a dummy file for the task
+			fs.writeFileSync(
+				path.join(taskDir, "README.md"),
+				`# ${task.name}\n\n${task.description}\n\nThis is a dummy task for testing purposes.`,
+			)
+		}
+
+		// Update the task's workspace path to the task-specific directory
+		return {
+			...task,
+			workspacePath: taskDir,
+		}
+	}
+
+	/**
+	 * Verify the result of a task execution (dummy implementation)
+	 * @param task The task that was executed
+	 * @param result The result of the task execution
+	 */
+	async verifyResult(task: Task, result: any): Promise<VerificationResult> {
+		// Always return success for dummy implementation
+		return {
+			success: true,
+			metrics: {
+				testsPassed: 1,
+				testsFailed: 0,
+				testsTotal: 1,
+				functionalCorrectness: 1.0,
+				performanceImprovement: 0.25, // Dummy metric specific to SWE-Bench
+				codeQuality: 0.9, // Dummy metric specific to SWE-Bench
+			},
+		}
+	}
+}

evals/cli/src/adapters/swelancer.ts

@@ -0,0 +1,143 @@
+import * as path from "path"
+import * as fs from "fs"
+import execa from "execa"
+import { BenchmarkAdapter, Task, VerificationResult } from "./types"
+
+const EVALS_DIR = path.resolve(__dirname, "../../../")
+
+/**
+ * Dummy adapter for the SWELancer benchmark
+ */
+export class SWELancerAdapter implements BenchmarkAdapter {
+	name = "swelancer"
+
+	/**
+	 * Set up the SWELancer benchmark repository (dummy implementation)
+	 */
+	async setup(): Promise<void> {
+		console.log("SWELancer dummy setup completed")
+
+		// Create repositories directory if it doesn't exist
+		const repoDir = path.join(EVALS_DIR, "repositories", "swelancer")
+		if (!fs.existsSync(repoDir)) {
+			fs.mkdirSync(repoDir, { recursive: true })
+			console.log(`Created dummy SWELancer directory at ${repoDir}`)
+		}
+	}
+
+	/**
+	 * List all available tasks in the SWELancer benchmark (dummy implementation)
+	 */
+	async listTasks(): Promise<Task[]> {
+		return [
+			{
+				id: "swelancer-task-1",
+				name: "Create Landing Page",
+				description: "Create a responsive landing page for a new product using HTML, CSS, and JavaScript.",
+				workspacePath: path.join(EVALS_DIR, "repositories", "swelancer"),
+				setupCommands: [],
+				verificationCommands: [],
+				metadata: {
+					client: "TechStartup Inc.",
+					difficulty: "medium",
+					type: "swelancer",
+				},
+			},
+			{
+				id: "swelancer-task-2",
+				name: "Build REST API",
+				description: "Create a RESTful API for a blog application using Node.js and Express.",
+				workspacePath: path.join(EVALS_DIR, "repositories", "swelancer"),
+				setupCommands: [],
+				verificationCommands: [],
+				metadata: {
+					client: "BlogCo",
+					difficulty: "hard",
+					type: "swelancer",
+				},
+			},
+			{
+				id: "swelancer-task-3",
+				name: "Fix CSS Layout Issues",
+				description: "Fix layout issues in a responsive website across different screen sizes.",
+				workspacePath: path.join(EVALS_DIR, "repositories", "swelancer"),
+				setupCommands: [],
+				verificationCommands: [],
+				metadata: {
+					client: "DesignAgency",
+					difficulty: "easy",
+					type: "swelancer",
+				},
+			},
+		]
+	}
+
+	/**
+	 * Prepare a specific task for execution (dummy implementation)
+	 * @param taskId The ID of the task to prepare
+	 */
+	async prepareTask(taskId: string): Promise<Task> {
+		const tasks = await this.listTasks()
+		const task = tasks.find((t) => t.id === taskId)
+
+		if (!task) {
+			throw new Error(`Task ${taskId} not found`)
+		}
+
+		// Create a dummy workspace for the task
+		const taskDir = path.join(task.workspacePath, taskId)
+		if (!fs.existsSync(taskDir)) {
+			fs.mkdirSync(taskDir, { recursive: true })
+
+			// Create a dummy file for the task
+			fs.writeFileSync(
+				path.join(taskDir, "README.md"),
+				`# ${task.name}\n\n${task.description}\n\nThis is a dummy task for testing purposes.`,
+			)
+
+			// Create additional dummy files based on task type
+			if (task.id === "swelancer-task-1") {
+				fs.writeFileSync(
+					path.join(taskDir, "index.html"),
+					`<!DOCTYPE html>\n<html>\n<head>\n  <title>Landing Page</title>\n</head>\n<body>\n  <!-- TODO: Implement landing page -->\n</body>\n</html>`,
+				)
+			} else if (task.id === "swelancer-task-2") {
+				fs.writeFileSync(
+					path.join(taskDir, "server.js"),
+					`// TODO: Implement REST API\nconsole.log('Server starting...');`,
+				)
+			} else if (task.id === "swelancer-task-3") {
+				fs.writeFileSync(
+					path.join(taskDir, "styles.css"),
+					`/* TODO: Fix layout issues */\nbody {\n  margin: 0;\n  padding: 0;\n}`,
+				)
+			}
+		}
+
+		// Update the task's workspace path to the task-specific directory
+		return {
+			...task,
+			workspacePath: taskDir,
+		}
+	}
+
+	/**
+	 * Verify the result of a task execution (dummy implementation)
+	 * @param task The task that was executed
+	 * @param result The result of the task execution
+	 */
+	async verifyResult(task: Task, result: any): Promise<VerificationResult> {
+		// Always return success for dummy implementation
+		return {
+			success: true,
+			metrics: {
+				testsPassed: 1,
+				testsFailed: 0,
+				testsTotal: 1,
+				functionalCorrectness: 1.0,
+				clientSatisfaction: 0.95, // Dummy metric specific to SWELancer
+				timeEfficiency: 0.85, // Dummy metric specific to SWELancer
+			},
+		}
+	}
+}

evals/cli/src/adapters/types.ts

@@ -0,0 +1,31 @@
+/**
+ * Represents a task to be executed
+ */
+export interface Task {
+	id: string
+	name: string
+	description: string
+	workspacePath: string
+	setupCommands: string[]
+	verificationCommands: string[]
+	metadata: Record<string, any>
+}
+
+/**
+ * Result of verifying a task execution
+ */
+export interface VerificationResult {
+	success: boolean
+	metrics: Record<string, any>
+}
+
+/**
+ * Interface for benchmark adapters
+ */
+export interface BenchmarkAdapter {
+	name: string
+	setup(): Promise<void>
+	listTasks(): Promise<Task[]>
+	prepareTask(taskId: string): Promise<Task>
+	verifyResult(task: Task, result: any): Promise<VerificationResult>
+}

evals/cli/src/commands/evals-env.ts

@@ -0,0 +1,53 @@
+import * as path from "path"
+import chalk from "chalk"
+import { createEvalsEnvFile, removeEvalsEnvFile, checkEvalsEnvFile } from "../utils/evals-env"
+
+interface EvalsEnvOptions {
+	action: "create" | "remove" | "check"
+	directory?: string
+}
+
+/**
+ * Handler for the evals-env command
+ * @param options Command options
+ */
+export async function evalsEnvHandler(options: EvalsEnvOptions): Promise<void> {
+	// Determine the directory to use - default to repository root instead of current directory
+	const currentDir = process.cwd()
+	const repoRoot = path.resolve(currentDir, "..", "..") // Navigate up from evals/cli to root
+	const directory = options.directory || repoRoot
+
+	console.log(chalk.blue(`Working with directory: ${directory}`))
+
+	// Perform the requested action
+	switch (options.action) {
+		case "create":
+			console.log(chalk.blue("Creating evals.env file..."))
+			createEvalsEnvFile(directory)
+			console.log(chalk.green("The Cline extension should now detect this file and enter test mode."))
+			console.log(chalk.yellow("Note: You may need to reload VSCode for the changes to take effect."))
+			break
+
+		case "remove":
+			console.log(chalk.blue("Removing evals.env file..."))
+			removeEvalsEnvFile(directory)
+			console.log(chalk.green("The Cline extension should now exit test mode."))
+			console.log(chalk.yellow("Note: You may need to reload VSCode for the changes to take effect."))
+			break
+
+		case "check":
+			console.log(chalk.blue("Checking for evals.env file..."))
+			const exists = checkEvalsEnvFile(directory)
+			if (exists) {
+				console.log(chalk.green("The Cline extension should be in test mode."))
+			} else {
+				console.log(chalk.yellow("The Cline extension should not be in test mode."))
+			}
+			break
+
+		default:
+			console.error(chalk.red(`Unknown action: ${options.action}`))
+			console.log(chalk.yellow("Valid actions are: create, remove, check"))
+			break
+	}
+}

evals/cli/src/commands/report.ts

@@ -0,0 +1,237 @@
+import * as fs from "fs"
+import * as path from "path"
+import chalk from "chalk"
+import ora from "ora"
+import { ResultsDatabase } from "../db"
+import { generateMarkdownReport } from "../utils/markdown"
+
+interface ReportOptions {
+	format?: "json" | "markdown"
+	output?: string
+}
+
+/**
+ * Handler for the report command
+ * @param options Command options
+ */
+export async function reportHandler(options: ReportOptions): Promise<void> {
+	const format = options.format || "markdown"
+	const db = new ResultsDatabase()
+
+	try {
+		const spinner = ora("Generating report...").start()
+
+		// Get all runs
+		const runs = db.getRuns()
+
+		console.log(chalk.blue(`Found ${runs.length} evaluation runs`))
+
+		if (runs.length === 0) {
+			spinner.fail("No evaluation runs found")
+			return
+		}
+
+		// Generate summary report
+		const summary = {
+			runs: runs.length,
+			models: [...new Set(runs.map((run) => run.model))],
+			benchmarks: [...new Set(runs.map((run) => run.benchmark))],
+			tasks: 0,
+			successRate: 0,
+			averageTokens: 0,
+			averageCost: 0,
+			averageDuration: 0,
+			totalToolCalls: 0,
+			totalToolFailures: 0,
+			toolSuccessRate: 0,
+			toolUsage: {} as Record<string, { calls: number; failures: number }>,
+		}
+
+		let totalTasks = 0
+		let successfulTasks = 0
+		let totalTokens = 0
+		let totalCost = 0
+		let totalDuration = 0
+		let totalToolCalls = 0
+		let totalToolFailures = 0
+
+		for (const run of runs) {
+			const tasks = db.getRunTasks(run.id)
+			totalTasks += tasks.length
+
+			for (const task of tasks) {
+				if (task.success) {
+					successfulTasks++
+				}
+
+				const metrics = db.getTaskMetrics(task.id)
+
+				const tokensIn = metrics.find((m) => m.name === "tokensIn")?.value || 0
+				const tokensOut = metrics.find((m) => m.name === "tokensOut")?.value || 0
+				totalTokens += tokensIn + tokensOut
+
+				totalCost += metrics.find((m) => m.name === "cost")?.value || 0
+				totalDuration += metrics.find((m) => m.name === "duration")?.value || 0
+
+				// Collect tool call metrics
+				totalToolCalls += task.total_tool_calls || 0
+				totalToolFailures += task.total_tool_failures || 0
+
+				// Get detailed tool usage
+				const toolCalls = db.getTaskToolCalls(task.id)
+
+				for (const toolCall of toolCalls) {
+					if (!summary.toolUsage[toolCall.tool_name]) {
+						summary.toolUsage[toolCall.tool_name] = {
+							calls: 0,
+							failures: 0,
+						}
+					}
+
+					summary.toolUsage[toolCall.tool_name].calls += toolCall.call_count
+					summary.toolUsage[toolCall.tool_name].failures += toolCall.failure_count
+				}
+			}
+		}
+
+		// Calculate tool success rate
+		summary.totalToolCalls = totalToolCalls
+		summary.totalToolFailures = totalToolFailures
+		summary.toolSuccessRate = totalToolCalls > 0 ? 1 - totalToolFailures / totalToolCalls : 1.0
+
+		summary.tasks = totalTasks
+		summary.successRate = totalTasks > 0 ? successfulTasks / totalTasks : 0
+		summary.averageTokens = totalTasks > 0 ? totalTokens / totalTasks : 0
+		summary.averageCost = totalTasks > 0 ? totalCost / totalTasks : 0
+		summary.averageDuration = totalTasks > 0 ? totalDuration / totalTasks : 0
+
+		// Generate benchmark-specific reports
+		const benchmarkReports: Record<string, any> = {}
+
+		for (const benchmark of summary.benchmarks) {
+			const benchmarkRuns = runs.filter((run) => run.benchmark === benchmark)
+			const benchmarkSummary = {
+				runs: benchmarkRuns.length,
+				models: [...new Set(benchmarkRuns.map((run) => run.model))],
+				tasks: 0,
+				successRate: 0,
+				averageTokens: 0,
+				averageCost: 0,
+				averageDuration: 0,
+			}
+
+			let benchmarkTasks = 0
+			let benchmarkSuccessfulTasks = 0
+			let benchmarkTotalTokens = 0
+			let benchmarkTotalCost = 0
+			let benchmarkTotalDuration = 0
+
+			for (const run of benchmarkRuns) {
+				const tasks = db.getRunTasks(run.id)
+				benchmarkTasks += tasks.length
+
+				for (const task of tasks) {
+					if (task.success) {
+						benchmarkSuccessfulTasks++
+					}
+
+					const metrics = db.getTaskMetrics(task.id)
+
+					const tokensIn = metrics.find((m) => m.name === "tokensIn")?.value || 0
+					const tokensOut = metrics.find((m) => m.name === "tokensOut")?.value || 0
+					benchmarkTotalTokens += tokensIn + tokensOut
+
+					benchmarkTotalCost += metrics.find((m) => m.name === "cost")?.value || 0
+					benchmarkTotalDuration += metrics.find((m) => m.name === "duration")?.value || 0
+				}
+			}
+
+			benchmarkSummary.tasks = benchmarkTasks
+			benchmarkSummary.successRate = benchmarkTasks > 0 ? benchmarkSuccessfulTasks / benchmarkTasks : 0
+			benchmarkSummary.averageTokens = benchmarkTasks > 0 ? benchmarkTotalTokens / benchmarkTasks : 0
+			benchmarkSummary.averageCost = benchmarkTasks > 0 ? benchmarkTotalCost / benchmarkTasks : 0
+			benchmarkSummary.averageDuration = benchmarkTasks > 0 ? benchmarkTotalDuration / benchmarkTasks : 0
+
+			benchmarkReports[benchmark] = benchmarkSummary
+		}
+
+		// Generate model-specific reports
+		const modelReports: Record<string, any> = {}
+
+		for (const model of summary.models) {
+			const modelRuns = runs.filter((run) => run.model === model)
+			const modelSummary = {
+				runs: modelRuns.length,
+				benchmarks: [...new Set(modelRuns.map((run) => run.benchmark))],
+				tasks: 0,
+				successRate: 0,
+				averageTokens: 0,
+				averageCost: 0,
+				averageDuration: 0,
+			}
+
+			let modelTasks = 0
+			let modelSuccessfulTasks = 0
+			let modelTotalTokens = 0
+			let modelTotalCost = 0
+			let modelTotalDuration = 0
+
+			for (const run of modelRuns) {
+				const tasks = db.getRunTasks(run.id)
+				modelTasks += tasks.length
+
+				for (const task of tasks) {
+					if (task.success) {
+						modelSuccessfulTasks++
+					}
+
+					const metrics = db.getTaskMetrics(task.id)
+
+					const tokensIn = metrics.find((m) => m.name === "tokensIn")?.value || 0
+					const tokensOut = metrics.find((m) => m.name === "tokensOut")?.value || 0
+					modelTotalTokens += tokensIn + tokensOut
+
+					modelTotalCost += metrics.find((m) => m.name === "cost")?.value || 0
+					modelTotalDuration += metrics.find((m) => m.name === "duration")?.value || 0
+				}
+			}
+
+			modelSummary.tasks = modelTasks
+			modelSummary.successRate = modelTasks > 0 ? modelSuccessfulTasks / modelTasks : 0
+			modelSummary.averageTokens = modelTasks > 0 ? modelTotalTokens / modelTasks : 0
+			modelSummary.averageCost = modelTasks > 0 ? modelTotalCost / modelTasks : 0
+			modelSummary.averageDuration = modelTasks > 0 ? modelTotalDuration / modelTasks : 0
+
+			modelReports[model] = modelSummary
+		}
+
+		// Save reports
+		const reportDir = path.join(path.resolve(__dirname, "../../../"), "results", "reports")
+		fs.mkdirSync(reportDir, { recursive: true })
+
+		const timestamp = new Date().toISOString().replace(/:/g, "-")
+
+		if (format === "json") {
+			// Save JSON reports
+			fs.writeFileSync(path.join(reportDir, `summary-${timestamp}.json`), JSON.stringify(summary, null, 2))
+
+			fs.writeFileSync(path.join(reportDir, `benchmarks-${timestamp}.json`), JSON.stringify(benchmarkReports, null, 2))
+
+			fs.writeFileSync(path.join(reportDir, `models-${timestamp}.json`), JSON.stringify(modelReports, null, 2))
+
+			spinner.succeed(`JSON reports generated in ${reportDir}`)
+		} else {
+			// Generate markdown report
+			const outputPath = options.output || path.join(reportDir, `report-${timestamp}.md`)
+
+			generateMarkdownReport(summary, benchmarkReports, modelReports, outputPath)
+
+			spinner.succeed(`Markdown report generated at ${outputPath}`)
+		}
+	} catch (error: any) {
+		console.error(chalk.red(`Error generating report: ${error.message}`))
+		console.error(error.stack)
+	} finally {
+		db.close()
+	}
+}

evals/cli/src/commands/run.ts

@@ -0,0 +1,133 @@
+import * as path from "path"
+import { v4 as uuidv4 } from "uuid"
+import chalk from "chalk"
+import ora from "ora"
+import { getAdapter } from "../adapters"
+import { ResultsDatabase } from "../db"
+import { spawnVSCode, cleanupVSCode } from "../utils/vscode"
+import { sendTaskToServer } from "../utils/task"
+import { storeTaskResult } from "../utils/results"
+
+interface RunOptions {
+	benchmark?: string
+	model: string
+	count?: number
+	apiKey?: string
+}
+
+/**
+ * Handler for the run command
+ * @param options Command options
+ */
+export async function runHandler(options: RunOptions): Promise<void> {
+	// Determine which benchmarks to run
+	const benchmarks = options.benchmark ? [options.benchmark] : ["exercism"] // Default to exercism for now
+	const model = options.model
+	const count = options.count || Infinity
+
+	console.log(chalk.blue(`Running evaluations for model: ${model}`))
+	console.log(chalk.blue(`Benchmarks: ${benchmarks.join(", ")}`))
+
+	// Create a run for each benchmark
+	for (const benchmark of benchmarks) {
+		const runId = uuidv4()
+		const db = new ResultsDatabase()
+
+		console.log(chalk.green(`\nStarting run for benchmark: ${benchmark}`))
+
+		// Create run in database
+		db.createRun(runId, model, benchmark)
+
+		// Get adapter for this benchmark
+		try {
+			const adapter = getAdapter(benchmark)
+
+			// List tasks
+			const spinner = ora("Listing tasks...").start()
+			const tasks = await adapter.listTasks()
+			spinner.succeed(`Found ${tasks.length} tasks for ${benchmark}`)
+
+			// Limit number of tasks if specified
+			const tasksToRun = tasks.slice(0, count)
+
+			console.log(chalk.blue(`Running ${tasksToRun.length} tasks...`))
+
+			// Run each task
+			for (let i = 0; i < tasksToRun.length; i++) {
+				const task = tasksToRun[i]
+
+				console.log(chalk.cyan(`\nTask ${i + 1}/${tasksToRun.length}: ${task.name}`))
+
+				// Prepare task
+				const prepareSpinner = ora("Preparing task...").start()
+				const preparedTask = await adapter.prepareTask(task.id)
+				prepareSpinner.succeed("Task prepared")
+
+				// Spawn VSCode
+				console.log("Spawning VSCode...")
+				await spawnVSCode(preparedTask.workspacePath)
+
+				// Send task to server
+				const sendSpinner = ora("Sending task to server...").start()
+				try {
+					const result = await sendTaskToServer(preparedTask.description, options.apiKey)
+					sendSpinner.succeed("Task completed")
+
+					// Verify result
+					const verifySpinner = ora("Verifying result...").start()
+					const verification = await adapter.verifyResult(preparedTask, result)
+
+					if (verification.success) {
+						verifySpinner.succeed(
+							`Verification successful: ${verification.metrics.testsPassed}/${verification.metrics.testsTotal} tests passed`,
+						)
+					} else {
+						verifySpinner.fail(
+							`Verification failed: ${verification.metrics.testsPassed}/${verification.metrics.testsTotal} tests passed`,
+						)
+					}
+
+					// Store result
+					const storeSpinner = ora("Storing result...").start()
+					await storeTaskResult(runId, preparedTask, result, verification)
+					storeSpinner.succeed("Result stored")
+
+					console.log(chalk.green(`Task completed. Success: ${verification.success}`))
+
+					// Clean up VS Code and temporary files
+					const cleanupSpinner = ora("Cleaning up...").start()
+					try {
+						await cleanupVSCode(preparedTask.workspacePath)
+						cleanupSpinner.succeed("Cleanup completed")
+					} catch (cleanupError: any) {
+						cleanupSpinner.fail(`Cleanup failed: ${cleanupError.message}`)
+						console.error(chalk.yellow(cleanupError.stack))
+					}
+				} catch (error: any) {
+					sendSpinner.fail(`Task failed: ${error.message}`)
+					console.error(chalk.red(error.stack))
+
+					// Clean up VS Code and temporary files even if the task failed
+					const cleanupSpinner = ora("Cleaning up...").start()
+					try {
+						await cleanupVSCode(preparedTask.workspacePath)
+						cleanupSpinner.succeed("Cleanup completed")
+					} catch (cleanupError: any) {
+						cleanupSpinner.fail(`Cleanup failed: ${cleanupError.message}`)
+						console.error(chalk.yellow(cleanupError.stack))
+					}
+				}
+			}
+
+			// Mark run as complete
+			db.completeRun(runId)
+
+			console.log(chalk.green(`\nRun complete for benchmark: ${benchmark}`))
+		} catch (error: any) {
+			console.error(chalk.red(`Error running benchmark ${benchmark}: ${error.message}`))
+			console.error(error.stack)
+		}
+	}
+
+	console.log(chalk.green("\nAll evaluations complete"))
+}

evals/cli/src/commands/setup.ts

@@ -0,0 +1,72 @@
+import * as path from "path"
+import * as fs from "fs"
+import execa from "execa"
+import chalk from "chalk"
+import ora from "ora"
+import { getAllAdapters } from "../adapters/index"
+import { BenchmarkAdapter } from "../adapters/types"
+
+interface SetupOptions {
+	benchmarks: string
+}
+
+/**
+ * Handler for the setup command
+ * @param options Command options
+ */
+export async function setupHandler(options: SetupOptions): Promise<void> {
+	const benchmarks = options.benchmarks.split(",")
+
+	console.log(chalk.blue(`Setting up benchmarks: ${benchmarks.join(", ")}`))
+
+	// Create directories
+	const evalsDir = path.resolve(__dirname, "../../../")
+	const reposDir = path.join(evalsDir, "repositories")
+	const resultsDir = path.join(evalsDir, "results")
+
+	const spinner = ora("Creating directory structure").start()
+
+	try {
+		fs.mkdirSync(reposDir, { recursive: true })
+		fs.mkdirSync(resultsDir, { recursive: true })
+		fs.mkdirSync(path.join(resultsDir, "runs"), { recursive: true })
+		fs.mkdirSync(path.join(resultsDir, "reports"), { recursive: true })
+		spinner.succeed("Directory structure created")
+	} catch (error) {
+		spinner.fail(`Failed to create directory structure: ${(error as Error).message}`)
+		throw error
+	}
+
+	// Set up each benchmark
+	try {
+		const adapters = getAllAdapters().filter((adapter: BenchmarkAdapter) => benchmarks.includes(adapter.name))
+
+		if (adapters.length === 0) {
+			console.warn(chalk.yellow("No valid benchmarks specified. Available benchmarks:"))
+			console.warn(
+				chalk.yellow(
+					getAllAdapters()
+						.map((a: BenchmarkAdapter) => a.name)
+						.join(", "),
+				),
+			)
+			return
+		}
+
+		for (const adapter of adapters) {
+			const setupSpinner = ora(`Setting up ${adapter.name}...`).start()
+			try {
+				await adapter.setup()
+				setupSpinner.succeed(`${adapter.name} setup complete`)
+			} catch (error) {
+				setupSpinner.fail(`Failed to set up ${adapter.name}: ${(error as Error).message}`)
+				throw error
+			}
+		}
+
+		console.log(chalk.green("Setup complete"))
+	} catch (error) {
+		console.error(chalk.red(`Setup failed: ${(error as Error).message}`))
+		throw error
+	}
+}

evals/cli/src/db/index.ts

@@ -0,0 +1,211 @@
+import * as path from "path"
+import * as fs from "fs"
+import Database from "better-sqlite3"
+import { SCHEMA } from "./schema"
+
+const EVALS_DIR = path.resolve(__dirname, "../../../")
+
+/**
+ * Database class for storing evaluation results
+ */
+export class ResultsDatabase {
+	db: Database.Database
+
+	constructor() {
+		// Ensure results directory exists
+		const resultsDir = path.join(EVALS_DIR, "results")
+		fs.mkdirSync(resultsDir, { recursive: true })
+
+		// Create database file
+		const dbPath = path.join(resultsDir, "evals.db")
+		this.db = new Database(dbPath)
+
+		// Initialize schema
+		this.initSchema()
+	}
+
+	/**
+	 * Initialize the database schema
+	 */
+	private initSchema(): void {
+		this.db.exec(SCHEMA)
+	}
+
+	/**
+	 * Create a new evaluation run
+	 * @param id Run ID
+	 * @param model Model name
+	 * @param benchmark Benchmark name
+	 */
+	createRun(id: string, model: string, benchmark: string): void {
+		const stmt = this.db.prepare(`
+      INSERT INTO runs (id, timestamp, model, benchmark)
+      VALUES (?, ?, ?, ?)
+    `)
+
+		stmt.run(id, Date.now(), model, benchmark)
+	}
+
+	/**
+	 * Mark a run as completed
+	 * @param id Run ID
+	 */
+	completeRun(id: string): void {
+		const stmt = this.db.prepare(`
+      UPDATE runs SET completed = 1 WHERE id = ?
+    `)
+
+		stmt.run(id)
+	}
+
+	/**
+	 * Create a new task
+	 * @param id Task ID
+	 * @param runId Run ID
+	 * @param taskId Original task ID
+	 */
+	createTask(id: string, runId: string, taskId: string): void {
+		const stmt = this.db.prepare(`
+      INSERT INTO tasks (id, run_id, task_id, timestamp)
+      VALUES (?, ?, ?, ?)
+    `)
+
+		stmt.run(id, runId, taskId, Date.now())
+	}
+
+	/**
+	 * Mark a task as completed
+	 * @param id Task ID
+	 * @param success Whether the task was successful
+	 * @param toolCalls Total tool calls
+	 * @param toolFailures Total tool failures
+	 */
+	completeTask(id: string, success: boolean, toolCalls: number = 0, toolFailures: number = 0): void {
+		const stmt = this.db.prepare(`
+      UPDATE tasks 
+      SET success = ?, total_tool_calls = ?, total_tool_failures = ? 
+      WHERE id = ?
+    `)
+
+		stmt.run(success ? 1 : 0, toolCalls, toolFailures, id)
+	}
+
+	/**
+	 * Add a metric to a task
+	 * @param taskId Task ID
+	 * @param name Metric name
+	 * @param value Metric value
+	 */
+	addMetric(taskId: string, name: string, value: number): void {
+		const stmt = this.db.prepare(`
+      INSERT INTO metrics (task_id, name, value)
+      VALUES (?, ?, ?)
+    `)
+
+		stmt.run(taskId, name, value)
+	}
+
+	/**
+	 * Add a tool call record
+	 * @param taskId Task ID
+	 * @param toolName Tool name
+	 * @param callCount Number of calls
+	 * @param failureCount Number of failures
+	 */
+	addToolCall(taskId: string, toolName: string, callCount: number, failureCount: number): void {
+		const stmt = this.db.prepare(`
+      INSERT INTO tool_calls (task_id, tool_name, call_count, failure_count)
+      VALUES (?, ?, ?, ?)
+    `)
+
+		stmt.run(taskId, toolName, callCount, failureCount)
+	}
+
+	/**
+	 * Add a file record
+	 * @param taskId Task ID
+	 * @param filePath File path
+	 * @param status File status (created, modified, deleted)
+	 */
+	addFile(taskId: string, filePath: string, status: "created" | "modified" | "deleted"): void {
+		const stmt = this.db.prepare(`
+      INSERT INTO files (task_id, path, status)
+      VALUES (?, ?, ?)
+    `)
+
+		stmt.run(taskId, filePath, status)
+	}
+
+	/**
+	 * Get all runs
+	 * @returns Array of runs
+	 */
+	getRuns(): any[] {
+		const stmt = this.db.prepare(`
+      SELECT * FROM runs ORDER BY timestamp DESC
+    `)
+
+		return stmt.all()
+	}
+
+	/**
+	 * Get all tasks for a run
+	 * @param runId Run ID
+	 * @returns Array of tasks
+	 */
+	getRunTasks(runId: string): any[] {
+		const stmt = this.db.prepare(`
+      SELECT * FROM tasks WHERE run_id = ? ORDER BY timestamp ASC
+    `)
+
+		return stmt.all(runId)
+	}
+
+	/**
+	 * Get all metrics for a task
+	 * @param taskId Task ID
+	 * @returns Array of metrics
+	 */
+	getTaskMetrics(taskId: string): any[] {
+		const stmt = this.db.prepare(`
+      SELECT name, value FROM metrics WHERE task_id = ?
+    `)
+
+		return stmt.all(taskId)
+	}
+
+	/**
+	 * Get all tool calls for a task
+	 * @param taskId Task ID
+	 * @returns Array of tool calls
+	 */
+	getTaskToolCalls(taskId: string): any[] {
+		const stmt = this.db.prepare(`
+      SELECT tool_name, call_count, failure_count 
+      FROM tool_calls 
+      WHERE task_id = ?
+    `)
+
+		return stmt.all(taskId)
+	}
+
+	/**
+	 * Get all files for a task
+	 * @param taskId Task ID
+	 * @returns Array of files
+	 */
+	getTaskFiles(taskId: string): any[] {
+		const stmt = this.db.prepare(`
+      SELECT path, status FROM files WHERE task_id = ?
+    `)
+
+		return stmt.all(taskId)
+	}
+
+	/**
+	 * Close the database connection
+	 */
+	close(): void {
+		this.db.close()
+	}
+}

evals/cli/src/db/schema.ts

@@ -0,0 +1,48 @@
+/**
+ * SQL schema for the evaluation database
+ */
+export const SCHEMA = `
+CREATE TABLE IF NOT EXISTS runs (
+  id TEXT PRIMARY KEY,
+  timestamp INTEGER NOT NULL,
+  model TEXT NOT NULL,
+  benchmark TEXT NOT NULL,
+  completed INTEGER NOT NULL DEFAULT 0
+);
+
+CREATE TABLE IF NOT EXISTS tasks (
+  id TEXT PRIMARY KEY,
+  run_id TEXT NOT NULL,
+  task_id TEXT NOT NULL,
+  timestamp INTEGER NOT NULL,
+  success INTEGER NOT NULL DEFAULT 0,
+  total_tool_calls INTEGER DEFAULT 0,
+  total_tool_failures INTEGER DEFAULT 0,
+  FOREIGN KEY (run_id) REFERENCES runs(id)
+);
+
+CREATE TABLE IF NOT EXISTS metrics (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  task_id TEXT NOT NULL,
+  name TEXT NOT NULL,
+  value REAL NOT NULL,
+  FOREIGN KEY (task_id) REFERENCES tasks(id)
+);
+
+CREATE TABLE IF NOT EXISTS tool_calls (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  task_id TEXT NOT NULL,
+  tool_name TEXT NOT NULL,
+  call_count INTEGER NOT NULL,
+  failure_count INTEGER NOT NULL,
+  FOREIGN KEY (task_id) REFERENCES tasks(id)
+);
+
+CREATE TABLE IF NOT EXISTS files (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  task_id TEXT NOT NULL,
+  path TEXT NOT NULL,
+  status TEXT NOT NULL,
+  FOREIGN KEY (task_id) REFERENCES tasks(id)
+);
+`

evals/cli/src/index.ts

@@ -0,0 +1,86 @@
+#!/usr/bin/env node
+import { Command } from "commander"
+import chalk from "chalk"
+import { setupHandler } from "./commands/setup"
+import { runHandler } from "./commands/run"
+import { reportHandler } from "./commands/report"
+import { evalsEnvHandler } from "./commands/evals-env"
+
+// Create the CLI program
+const program = new Command()
+
+// Set up CLI metadata
+program.name("cline-eval").description("CLI tool for orchestrating Cline evaluations across multiple benchmarks").version("0.1.0")
+
+// Setup command
+program
+	.command("setup")
+	.description("Clone and set up benchmark repositories")
+	.option(
+		"-b, --benchmarks <benchmarks>",
+		"Comma-separated list of benchmarks to set up",
+		"exercism,swe-bench,swelancer,multi-swe",
+	)
+	.action(async (options) => {
+		try {
+			await setupHandler(options)
+		} catch (error) {
+			console.error(chalk.red(`Error during setup: ${error instanceof Error ? error.message : String(error)}`))
+			process.exit(1)
+		}
+	})
+
+// Run command
+program
+	.command("run")
+	.description("Run evaluations")
+	.option("-b, --benchmark <benchmark>", "Specific benchmark to run")
+	.option("-m, --model <model>", "Model to evaluate", "claude-3-opus-20240229")
+	.option("-c, --count <count>", "Number of tasks to run", parseInt)
+	.option("-k, --api-key <apiKey>", "Cline API key to use for evaluations")
+	.action(async (options) => {
+		try {
+			await runHandler(options)
+		} catch (error) {
+			console.error(chalk.red(`Error during run: ${error instanceof Error ? error.message : String(error)}`))
+			process.exit(1)
+		}
+	})
+
+// Report command
+program
+	.command("report")
+	.description("Generate reports")
+	.option("-f, --format <format>", "Report format (json, markdown)", "markdown")
+	.option("-o, --output <path>", "Output path for the report")
+	.action(async (options) => {
+		try {
+			await reportHandler(options)
+		} catch (error) {
+			console.error(chalk.red(`Error generating report: ${error instanceof Error ? error.message : String(error)}`))
+			process.exit(1)
+		}
+	})
+
+// Evals-env command
+program
+	.command("evals-env")
+	.description("Manage evals.env files for test mode activation")
+	.argument("<action>", "Action to perform: create, remove, or check")
+	.option("-d, --directory <directory>", "Directory to create/remove/check evals.env file in (defaults to current directory)")
+	.action(async (action, options) => {
+		try {
+			await evalsEnvHandler({ action, ...options })
+		} catch (error) {
+			console.error(chalk.red(`Error managing evals.env file: ${error instanceof Error ? error.message : String(error)}`))
+			process.exit(1)
+		}
+	})
+
+// Parse command line arguments
+program.parse(process.argv)
+
+// If no arguments provided, show help
+if (process.argv.length === 2) {
+	program.help()
+}

evals/cli/src/utils/evals-env.ts

@@ -0,0 +1,79 @@
+import * as fs from "fs"
+import * as path from "path"
+import chalk from "chalk"
+
+/**
+ * Creates an evals.env file in the specified directory
+ * @param directory The directory where the evals.env file should be created
+ * @returns True if the file was created, false if it already exists
+ */
+export function createEvalsEnvFile(directory: string): boolean {
+	const evalsEnvPath = path.join(directory, "evals.env")
+
+	// Check if the file already exists
+	if (fs.existsSync(evalsEnvPath)) {
+		console.log(chalk.yellow(`evals.env file already exists at ${evalsEnvPath}`))
+		return false
+	}
+
+	// Create the file
+	try {
+		const content = `# This file activates Cline test mode
+# Created at: ${new Date().toISOString()}
+# 
+# This file is automatically detected by the Cline extension
+# and enables test mode for automated evaluations.
+#
+# Delete this file to deactivate test mode.
+`
+		fs.writeFileSync(evalsEnvPath, content)
+		console.log(chalk.green(`Created evals.env file at ${evalsEnvPath}`))
+		return true
+	} catch (error) {
+		console.error(chalk.red(`Error creating evals.env file: ${error}`))
+		return false
+	}
+}
+
+/**
+ * Removes an evals.env file from the specified directory
+ * @param directory The directory where the evals.env file should be removed
+ * @returns True if the file was removed, false if it doesn't exist
+ */
+export function removeEvalsEnvFile(directory: string): boolean {
+	const evalsEnvPath = path.join(directory, "evals.env")
+
+	// Check if the file exists
+	if (!fs.existsSync(evalsEnvPath)) {
+		console.log(chalk.yellow(`No evals.env file found at ${evalsEnvPath}`))
+		return false
+	}
+
+	// Remove the file
+	try {
+		fs.unlinkSync(evalsEnvPath)
+		console.log(chalk.green(`Removed evals.env file from ${evalsEnvPath}`))
+		return true
+	} catch (error) {
+		console.error(chalk.red(`Error removing evals.env file: ${error}`))
+		return false
+	}
+}
+
+/**
+ * Checks if an evals.env file exists in the specified directory
+ * @param directory The directory to check for an evals.env file
+ * @returns True if the file exists, false otherwise
+ */
+export function checkEvalsEnvFile(directory: string): boolean {
+	const evalsEnvPath = path.join(directory, "evals.env")
+	const exists = fs.existsSync(evalsEnvPath)
+
+	if (exists) {
+		console.log(chalk.green(`evals.env file found at ${evalsEnvPath}`))
+	} else {
+		console.log(chalk.yellow(`No evals.env file found at ${evalsEnvPath}`))
+	}
+
+	return exists
+}

evals/cli/src/utils/extensions.ts

@@ -0,0 +1,131 @@
+import execa from "execa"
+import * as fs from "fs"
+import * as path from "path"
+import * as os from "os"
+
+/**
+ * List of VSCode extensions to install for evaluation environments
+ * These extensions provide language support and other useful features
+ */
+export const REQUIRED_EXTENSIONS = [
+	"golang.go", // Go language support
+	"dbaeumer.vscode-eslint", // ESLint support
+	"redhat.java", // Java support
+	"ms-python.python", // Python support
+	"rust-lang.rust-analyzer", // Rust support
+	"ms-vscode.cpptools", // C/C++ support
+]
+
+/**
+ * Install required VSCode extensions in the specified extensions directory
+ * @param extensionsDir The directory where extensions should be installed
+ * @returns Promise that resolves when all extensions are installed
+ */
+export async function installRequiredExtensions(extensionsDir: string): Promise<void> {
+	console.log("Installing required VSCode extensions...")
+
+	// Create the extensions directory if it doesn't exist
+	if (!fs.existsSync(extensionsDir)) {
+		fs.mkdirSync(extensionsDir, { recursive: true })
+	}
+
+	// Install each extension
+	for (const extension of REQUIRED_EXTENSIONS) {
+		try {
+			console.log(`Installing extension: ${extension}...`)
+			await execa("code", ["--extensions-dir", extensionsDir, "--install-extension", extension, "--force"])
+			console.log(`✅ Extension ${extension} installed successfully`)
+		} catch (error: any) {
+			console.warn(`⚠️ Failed to install extension ${extension}: ${error.message}`)
+			// Continue with other extensions even if one fails
+		}
+	}
+
+	console.log("✅ All required extensions installed")
+}
+
+/**
+ * Check if a VSCode extension is installed in the specified directory
+ * @param extensionsDir The directory to check for installed extensions
+ * @param extensionId The ID of the extension to check
+ * @returns True if the extension is installed, false otherwise
+ */
+export function isExtensionInstalled(extensionsDir: string, extensionId: string): boolean {
+	// Extensions are installed in directories named publisher.name-version
+	// We need to check if any directory starts with the extensionId
+	const extensionPrefix = extensionId.toLowerCase() + "-"
+
+	try {
+		const files = fs.readdirSync(extensionsDir)
+		return files.some((file) => {
+			const lowerCaseFile = file.toLowerCase()
+			return lowerCaseFile === extensionId.toLowerCase() || lowerCaseFile.startsWith(extensionPrefix)
+		})
+	} catch (error) {
+		return false
+	}
+}
+
+/**
+ * Get the path to the VSCode settings file in the specified user data directory
+ * @param userDataDir The VSCode user data directory
+ * @returns The path to the settings.json file
+ */
+export function getSettingsPath(userDataDir: string): string {
+	const settingsDir = path.join(userDataDir, "User")
+	fs.mkdirSync(settingsDir, { recursive: true })
+	return path.join(settingsDir, "settings.json")
+}
+
+/**
+ * Configure extension settings in the VSCode user data directory
+ * @param userDataDir The VSCode user data directory
+ */
+export function configureExtensionSettings(userDataDir: string): void {
+	const settingsPath = getSettingsPath(userDataDir)
+
+	// Read existing settings if they exist
+	let settings = {}
+	if (fs.existsSync(settingsPath)) {
+		try {
+			settings = JSON.parse(fs.readFileSync(settingsPath, "utf8"))
+		} catch (error) {
+			console.warn(`Error reading settings file: ${error}`)
+		}
+	}
+
+	// Add or update extension-specific settings
+	const updatedSettings = {
+		...settings,
+		// Go extension settings
+		"go.toolsManagement.autoUpdate": false,
+		"go.survey.prompt": false,
+
+		// ESLint settings
+		"eslint.enable": true,
+		"eslint.run": "onSave",
+
+		// Java settings
+		"java.configuration.checkProjectSettingsExclusions": false,
+		"java.configure.checkForOutdatedExtensions": false,
+		"java.help.firstView": false,
+
+		// Python settings
+		"python.experiments.enabled": false,
+		"python.showStartPage": false,
+
+		// Rust settings
+		"rust-analyzer.checkOnSave.command": "check",
+
+		// C/C++ settings
+		"C_Cpp.intelliSenseEngine": "default",
+
+		// General extension settings
+		"extensions.autoUpdate": false,
+		"extensions.ignoreRecommendations": true,
+	}
+
+	// Write updated settings
+	fs.writeFileSync(settingsPath, JSON.stringify(updatedSettings, null, 2))
+	console.log("✅ Extension settings configured")
+}

evals/cli/src/utils/markdown.ts

@@ -0,0 +1,109 @@
+import * as fs from "fs"
+import * as path from "path"
+
+/**
+ * Generate a markdown report from evaluation results
+ * @param summary Overall summary
+ * @param benchmarkReports Benchmark-specific reports
+ * @param modelReports Model-specific reports
+ * @param outputPath Output file path
+ */
+export function generateMarkdownReport(
+	summary: any,
+	benchmarkReports: Record<string, any>,
+	modelReports: Record<string, any>,
+	outputPath: string,
+): void {
+	let markdown = `# Cline Evaluation Report\n\n`
+
+	// Generate summary section
+	markdown += `## Summary\n\n`
+	markdown += `- **Total Runs:** ${summary.runs}\n`
+	markdown += `- **Models:** ${summary.models.join(", ")}\n`
+	markdown += `- **Benchmarks:** ${summary.benchmarks.join(", ")}\n`
+	markdown += `- **Total Tasks:** ${summary.tasks}\n`
+	markdown += `- **Success Rate:** ${(summary.successRate * 100).toFixed(2)}%\n`
+	markdown += `- **Average Tokens:** ${Math.round(summary.averageTokens)}\n`
+	markdown += `- **Average Cost:** $${summary.averageCost.toFixed(4)}\n`
+	markdown += `- **Average Duration:** ${(summary.averageDuration / 1000).toFixed(2)}s\n`
+	markdown += `- **Total Tool Calls:** ${summary.totalToolCalls}\n`
+	markdown += `- **Tool Success Rate:** ${(summary.toolSuccessRate * 100).toFixed(2)}%\n\n`
+
+	// Generate tool usage section
+	markdown += `## Tool Usage\n\n`
+	markdown += `| Tool | Calls | Failures | Success Rate |\n`
+	markdown += `| ---- | ----- | -------- | ------------ |\n`
+
+	for (const [toolName, metrics] of Object.entries(summary.toolUsage)) {
+		const calls = (metrics as any).calls
+		const failures = (metrics as any).failures
+		const successRate = calls > 0 ? (1 - failures / calls) * 100 : 100
+
+		markdown += `| ${toolName} | ${calls} | ${failures} | ${successRate.toFixed(2)}% |\n`
+	}
+
+	// Generate benchmark results section
+	markdown += `\n## Benchmark Results\n\n`
+
+	for (const [benchmark, report] of Object.entries(benchmarkReports)) {
+		markdown += `### ${benchmark}\n\n`
+		markdown += `- **Runs:** ${report.runs}\n`
+		markdown += `- **Models:** ${report.models.join(", ")}\n`
+		markdown += `- **Tasks:** ${report.tasks}\n`
+		markdown += `- **Success Rate:** ${(report.successRate * 100).toFixed(2)}%\n`
+		markdown += `- **Average Tokens:** ${Math.round(report.averageTokens)}\n`
+		markdown += `- **Average Cost:** $${report.averageCost.toFixed(4)}\n`
+		markdown += `- **Average Duration:** ${(report.averageDuration / 1000).toFixed(2)}s\n\n`
+	}
+
+	// Generate model results section
+	markdown += `## Model Results\n\n`
+
+	for (const [model, report] of Object.entries(modelReports)) {
+		markdown += `### ${model}\n\n`
+		markdown += `- **Runs:** ${report.runs}\n`
+		markdown += `- **Benchmarks:** ${report.benchmarks.join(", ")}\n`
+		markdown += `- **Tasks:** ${report.tasks}\n`
+		markdown += `- **Success Rate:** ${(report.successRate * 100).toFixed(2)}%\n`
+		markdown += `- **Average Tokens:** ${Math.round(report.averageTokens)}\n`
+		markdown += `- **Average Cost:** $${report.averageCost.toFixed(4)}\n`
+		markdown += `- **Average Duration:** ${(report.averageDuration / 1000).toFixed(2)}s\n\n`
+	}
+
+	// Add charts using Mermaid
+	markdown += `## Charts\n\n`
+
+	// Success rate by benchmark chart
+	markdown += `### Success Rate by Benchmark\n\n`
+	markdown += "```mermaid\n"
+	markdown += "graph TD\n"
+	markdown += "  title[Success Rate by Benchmark]\n"
+	markdown += "  style title fill:none,stroke:none\n\n"
+
+	for (const [benchmark, report] of Object.entries(benchmarkReports)) {
+		const successRate = (report.successRate * 100).toFixed(2)
+		markdown += `  ${benchmark}[${benchmark}: ${successRate}%]\n`
+	}
+
+	markdown += "```\n\n"
+
+	// Success rate by model chart
+	markdown += `### Success Rate by Model\n\n`
+	markdown += "```mermaid\n"
+	markdown += "graph TD\n"
+	markdown += "  title[Success Rate by Model]\n"
+	markdown += "  style title fill:none,stroke:none\n\n"
+
+	for (const [model, report] of Object.entries(modelReports)) {
+		const successRate = (report.successRate * 100).toFixed(2)
+		markdown += `  ${model.replace(/[-\.]/g, "_")}[${model}: ${successRate}%]\n`
+	}
+
+	markdown += "```\n\n"
+
+	// Add timestamp
+	markdown += `\n\n---\n\nReport generated on ${new Date().toISOString()}\n`
+
+	// Write markdown to file
+	fs.writeFileSync(outputPath, markdown)
+}

evals/cli/src/utils/results.ts

@@ -0,0 +1,79 @@
+import { v4 as uuidv4 } from "uuid"
+import { ResultsDatabase } from "../db"
+import { Task } from "../adapters/types"
+
+/**
+ * Store task result in the database
+ * @param runId The run ID
+ * @param task The task that was executed
+ * @param result The result from the test server
+ * @param verification The verification result
+ */
+export async function storeTaskResult(runId: string, task: Task, result: any, verification: any): Promise<void> {
+	const db = new ResultsDatabase()
+	const taskId = uuidv4()
+
+	try {
+		// Extract metrics from the result
+		const { metrics } = result
+		const totalToolCalls = metrics?.totalToolCalls || 0
+		const totalToolFailures = metrics?.totalToolFailures || 0
+
+		// Create task with tool metrics
+		db.createTask(taskId, runId, task.id)
+		db.completeTask(taskId, verification.success, totalToolCalls, totalToolFailures)
+
+		// Store metrics
+		if (metrics) {
+			// Store token metrics
+			if (metrics.tokensIn) db.addMetric(taskId, "tokensIn", metrics.tokensIn)
+			if (metrics.tokensOut) db.addMetric(taskId, "tokensOut", metrics.tokensOut)
+			if (metrics.cost) db.addMetric(taskId, "cost", metrics.cost)
+			if (metrics.duration) db.addMetric(taskId, "duration", metrics.duration)
+
+			// Store tool call metrics
+			if (metrics.toolCalls) {
+				for (const [toolName, callCount] of Object.entries(metrics.toolCalls)) {
+					const failureCount = metrics.toolFailures?.[toolName] || 0
+					db.addToolCall(taskId, toolName, callCount as number, failureCount)
+				}
+			}
+		}
+
+		// Store verification metrics
+		if (verification.metrics) {
+			for (const [key, value] of Object.entries(verification.metrics)) {
+				if (typeof value === "number") {
+					db.addMetric(taskId, key, value)
+				}
+			}
+		}
+
+		// Store file changes
+		if (result.files) {
+			// Store created files
+			if (result.files.created) {
+				for (const file of result.files.created) {
+					db.addFile(taskId, file, "created")
+				}
+			}
+
+			// Store modified files
+			if (result.files.modified) {
+				for (const file of result.files.modified) {
+					db.addFile(taskId, file, "modified")
+				}
+			}
+
+			// Store deleted files
+			if (result.files.deleted) {
+				for (const file of result.files.deleted) {
+					db.addFile(taskId, file, "deleted")
+				}
+			}
+		}
+	} finally {
+		// Close the database connection
+		db.close()
+	}
+}

evals/cli/src/utils/task.ts

@@ -0,0 +1,52 @@
+import fetch from "node-fetch"
+import chalk from "chalk"
+
+/**
+ * Send a task to the Cline test server
+ * @param task The task description to send
+ * @param apiKey Optional Cline API key to use for the task
+ * @returns The result of the task execution
+ */
+export async function sendTaskToServer(task: string, apiKey?: string): Promise<any> {
+	const SERVER_URL = "http://localhost:9876/task"
+
+	try {
+		console.log(chalk.blue(`Sending task to server: ${task.substring(0, 100)}${task.length > 100 ? "..." : ""}`))
+
+		const response = await fetch(SERVER_URL, {
+			method: "POST",
+			headers: {
+				"Content-Type": "application/json",
+			},
+			body: JSON.stringify({
+				task,
+				apiKey,
+			}),
+		})
+
+		if (!response.ok) {
+			const errorText = await response.text()
+			throw new Error(`Server responded with status ${response.status}: ${errorText}`)
+		}
+
+		const result = await response.json()
+
+		if (!result.success) {
+			throw new Error(`Task execution failed: ${result.error || "Unknown error"}`)
+		}
+
+		if (result.timeout) {
+			throw new Error("Task execution timed out")
+		}
+
+		return result
+	} catch (error: any) {
+		if (error.code === "ECONNREFUSED") {
+			throw new Error(
+				"Could not connect to the test server. Make sure VSCode is running with the Cline extension and the test server is active.",
+			)
+		}
+
+		throw error
+	}
+}

evals/cli/src/utils/vscode.ts

@@ -0,0 +1,615 @@
+import execa from "execa"
+import * as path from "path"
+import * as fs from "fs"
+import fetch from "node-fetch"
+import * as os from "os"
+import { installRequiredExtensions, configureExtensionSettings } from "./extensions"
+
+// Store temporary directories for cleanup
+interface VSCodeResources {
+	tempUserDataDir: string
+	tempExtensionsDir: string
+	vscodePid?: number
+}
+
+// Global map to track resources for each workspace
+const workspaceResources = new Map<string, VSCodeResources>()
+
+/**
+ * Spawn a VSCode instance with the Cline extension
+ * @param workspacePath The workspace path to open
+ * @param vsixPath Optional path to a VSIX file to install
+ * @returns The resources created for this VS Code instance
+ */
+export async function spawnVSCode(workspacePath: string, vsixPath?: string): Promise<VSCodeResources> {
+	// Ensure the workspace path exists
+	if (!fs.existsSync(workspacePath)) {
+		throw new Error(`Workspace path does not exist: ${workspacePath}`)
+	}
+
+	// If no VSIX path is provided, build one with IS_TEST=true
+	if (!vsixPath) {
+		try {
+			// Build the VSIX (no longer need to set IS_TEST=true as we'll use evals.env file)
+			console.log("Building VSIX...")
+			const clineRoot = path.resolve(process.cwd(), "..", "..")
+			await execa("npx", ["vsce", "package"], {
+				cwd: clineRoot,
+				stdio: "inherit",
+			})
+
+			// Find the generated VSIX file(s)
+			const files = fs.readdirSync(clineRoot)
+			const vsixFiles = files.filter((file) => file.endsWith(".vsix"))
+
+			if (vsixFiles.length > 0) {
+				// Get file stats to find the most recent one
+				const vsixFilesWithStats = vsixFiles.map((file) => {
+					const filePath = path.join(clineRoot, file)
+					return {
+						file,
+						path: filePath,
+						mtime: fs.statSync(filePath).mtime,
+					}
+				})
+
+				// Sort by modification time (most recent first)
+				vsixFilesWithStats.sort((a, b) => b.mtime.getTime() - a.mtime.getTime())
+
+				// Use the most recent VSIX
+				vsixPath = vsixFilesWithStats[0].path
+				console.log(`Using most recent VSIX: ${vsixPath} (modified ${vsixFilesWithStats[0].mtime.toISOString()})`)
+
+				// Log all found VSIX files for debugging
+				if (vsixFiles.length > 1) {
+					console.log(`Found ${vsixFiles.length} VSIX files:`)
+					vsixFilesWithStats.forEach((f) => {
+						console.log(`  - ${f.file} (modified ${f.mtime.toISOString()})`)
+					})
+				}
+			} else {
+				console.warn("Could not find generated VSIX file")
+			}
+		} catch (error) {
+			console.warn("Failed to build test VSIX:", error)
+		}
+	}
+
+	// Create a temporary user data directory for this VS Code instance
+	const tempUserDataDir = path.join(os.tmpdir(), `vscode-cline-eval-${Date.now()}`)
+	fs.mkdirSync(tempUserDataDir, { recursive: true })
+	console.log(`Created temporary user data directory: ${tempUserDataDir}`)
+
+	// Create a temporary extensions directory to ensure no other extensions are loaded
+	const tempExtensionsDir = path.join(os.tmpdir(), `vscode-cline-eval-ext-${Date.now()}`)
+	fs.mkdirSync(tempExtensionsDir, { recursive: true })
+	console.log(`Created temporary extensions directory: ${tempExtensionsDir}`)
+
+	// Create evals.env file in the workspace to trigger test mode
+	console.log(`Creating evals.env file in workspace: ${workspacePath}`)
+	const evalsEnvPath = path.join(workspacePath, "evals.env")
+	fs.writeFileSync(
+		evalsEnvPath,
+		`# This file activates Cline test mode
+# Created at: ${new Date().toISOString()}
+# 
+# This file is automatically detected by the Cline extension
+# and enables test mode for automated evaluations.
+#
+# Delete this file to deactivate test mode.
+`,
+	)
+
+	// Create settings.json in the temporary user data directory to disable workspace trust
+	// and configure Cline to auto-open on startup
+	const settingsDir = path.join(tempUserDataDir, "User")
+	fs.mkdirSync(settingsDir, { recursive: true })
+	const settingsPath = path.join(settingsDir, "settings.json")
+	const settings = {
+		// Disable workspace trust
+		"security.workspace.trust.enabled": false,
+		"security.workspace.trust.startupPrompt": "never",
+		"security.workspace.trust.banner": "never",
+		"security.workspace.trust.emptyWindow": true,
+
+		// Configure startup behavior
+		"workbench.startupEditor": "none",
+
+		// Auto-open Cline on startup
+		"cline.autoOpenOnStartup": true,
+
+		// Show the activity bar and sidebar
+		"workbench.activityBar.visible": true,
+		"workbench.sideBar.visible": true,
+		"workbench.view.extension.saoudrizwan.claude-dev-ActivityBar.visible": true,
+		"workbench.view.alwaysShowHeaderActions": true,
+		"workbench.editor.openSideBySideDirection": "right",
+
+		// Disable GitLens from opening automatically
+		"gitlens.views.repositories.autoReveal": false,
+		"gitlens.views.fileHistory.autoReveal": false,
+		"gitlens.views.lineHistory.autoReveal": false,
+		"gitlens.views.compare.autoReveal": false,
+		"gitlens.views.search.autoReveal": false,
+		"gitlens.showWelcomeOnInstall": false,
+		"gitlens.showWhatsNewAfterUpgrades": false,
+
+		// Disable other extensions that might compete for startup focus
+		"extensions.autoUpdate": false,
+	}
+	fs.writeFileSync(settingsPath, JSON.stringify(settings, null, 2))
+	console.log(`Created settings.json to disable workspace trust and auto-open Cline`)
+
+	// Create keybindings.json to automatically open Cline on startup
+	const keybindingsPath = path.join(settingsDir, "keybindings.json")
+	const keybindings = [
+		{
+			key: "alt+c",
+			command: "workbench.view.extension.saoudrizwan.claude-dev-ActivityBar",
+			when: "viewContainer.workbench.view.extension.saoudrizwan.claude-dev-ActivityBar.enabled",
+		},
+		{
+			key: "alt+shift+c",
+			command: "cline.openInNewTab",
+			when: "viewContainer.workbench.view.extension.saoudrizwan.claude-dev-ActivityBar.enabled",
+		},
+	]
+	fs.writeFileSync(keybindingsPath, JSON.stringify(keybindings, null, 2))
+	console.log(`Created keybindings.json to help with Cline activation`)
+
+	// Build the command arguments with custom user data directory
+	const args = [
+		// Use a custom user data directory to isolate this instance
+		"--user-data-dir",
+		tempUserDataDir,
+		// Use a custom extensions directory to ensure only our extension is loaded
+		"--extensions-dir",
+		tempExtensionsDir,
+		// Disable workspace trust
+		"--disable-workspace-trust",
+		"-n",
+		workspacePath,
+		// Force the extension to be activated on startup
+		"--start-up-extension",
+		"saoudrizwan.claude-dev",
+		// Run a command on startup to open Cline
+		"--command",
+		"workbench.view.extension.saoudrizwan.claude-dev-ActivityBar",
+		// Additional flags to help with extension activation
+		"--disable-gpu=false",
+		"--max-memory=4096",
+	]
+
+	// Create a startup script to run commands after VS Code launches
+	const startupScriptPath = path.join(settingsDir, "startup.js")
+	const startupScript = `
+		// This script will be executed when VS Code starts
+		setTimeout(() => {
+			// Try to open Cline in the sidebar
+			require('vscode').commands.executeCommand('workbench.view.extension.saoudrizwan.claude-dev-ActivityBar');
+			
+			// Also try to open Cline in a tab as a fallback
+			setTimeout(() => {
+				require('vscode').commands.executeCommand('cline.openInNewTab');
+			}, 5000);
+		}, 5000);
+	`
+	fs.writeFileSync(startupScriptPath, startupScript)
+	console.log(`Created startup script to activate Cline`)
+
+	// If a VSIX is provided, install it
+	if (vsixPath) {
+		if (!fs.existsSync(vsixPath)) {
+			throw new Error(`VSIX file does not exist: ${vsixPath}`)
+		}
+		args.unshift("--install-extension", vsixPath)
+	}
+
+	// Install required extensions
+	console.log("Installing required VSCode extensions...")
+	await installRequiredExtensions(tempExtensionsDir)
+
+	// Configure extension settings
+	console.log("Configuring extension settings...")
+	configureExtensionSettings(tempUserDataDir)
+
+	// Execute the command
+	try {
+		// We don't need to install extensions globally anymore since we're using a custom user data directory
+		// The VSIX will be installed in the isolated environment if provided in the args
+
+		// Launch VS Code
+		console.log("Launching VS Code...")
+		await execa("code", args, {
+			stdio: "inherit",
+		})
+
+		// Wait longer for VSCode to initialize and extension to load
+		console.log("Waiting for VS Code to initialize...")
+		await new Promise((resolve) => setTimeout(resolve, 30000))
+
+		// Create a JavaScript file that will be loaded as a VS Code extension
+		const extensionDir = path.join(tempExtensionsDir, "cline-activator")
+		fs.mkdirSync(extensionDir, { recursive: true })
+
+		// Create package.json for the extension
+		const packageJsonPath = path.join(extensionDir, "package.json")
+		const packageJson = {
+			name: "cline-activator",
+			displayName: "Cline Activator",
+			description: "Activates Cline and starts the test server",
+			version: "0.0.1",
+			engines: {
+				vscode: "^1.60.0",
+			},
+			main: "./extension.js",
+			activationEvents: ["*"],
+			contributes: {
+				commands: [
+					{
+						command: "cline-activator.activate",
+						title: "Activate Cline",
+					},
+				],
+			},
+		}
+		fs.writeFileSync(packageJsonPath, JSON.stringify(packageJson, null, 2))
+
+		// Create extension.js
+		const extensionJsPath = path.join(extensionDir, "extension.js")
+		const extensionJs = `
+			const vscode = require('vscode');
+			
+			/**
+			 * @param {vscode.ExtensionContext} context
+			 */
+			function activate(context) {
+				console.log('Cline Activator is now active!');
+				
+				// Register the command to activate Cline
+				let disposable = vscode.commands.registerCommand('cline-activator.activate', async function () {
+					try {
+						// Make sure the Cline extension is activated
+						const extension = vscode.extensions.getExtension('saoudrizwan.claude-dev');
+						if (!extension) {
+							console.error('Cline extension not found');
+							return;
+						}
+						
+						if (!extension.isActive) {
+							console.log('Activating Cline extension...');
+							await extension.activate();
+						}
+						
+						// Show the Cline sidebar
+						console.log('Opening Cline sidebar...');
+						await vscode.commands.executeCommand('workbench.view.extension.saoudrizwan.claude-dev-ActivityBar');
+						
+						// Wait a moment for the sidebar to initialize
+						await new Promise(resolve => setTimeout(resolve, 2000));
+						
+						// Also open Cline in a tab as a fallback
+						console.log('Opening Cline in a tab...');
+						await vscode.commands.executeCommand('cline.openInNewTab');
+						
+						// Wait a moment for the tab to initialize
+						await new Promise(resolve => setTimeout(resolve, 2000));
+						
+						// Create the test server if it doesn't exist
+						console.log('Creating test server...');
+						
+						// Get the visible webview instance
+						const clineRootPath = '${path.resolve(process.cwd(), "..", "..")}';
+						const visibleWebview = require(path.join(clineRootPath, 'src', 'core', 'webview')).WebviewProvider.getVisibleInstance();
+						if (visibleWebview) {
+							require(path.join(clineRootPath, 'src', 'services', 'test', 'TestServer')).createTestServer(visibleWebview);
+							console.log('Test server created successfully');
+						} else {
+							console.error('No visible webview instance found');
+						}
+					} catch (error) {
+						console.error('Error activating Cline:', error);
+					}
+				});
+				
+				context.subscriptions.push(disposable);
+				
+				// Automatically run the command after a delay
+				setTimeout(() => {
+					vscode.commands.executeCommand('cline-activator.activate');
+				}, 5000);
+			}
+			
+			function deactivate() {}
+			
+			module.exports = {
+				activate,
+				deactivate
+			}
+		`
+		fs.writeFileSync(extensionJsPath, extensionJs)
+		console.log(`Created Cline Activator extension`)
+
+		// Try multiple approaches to activate the extension
+		let serverStarted = false
+
+		// Create an activation script to run in VS Code
+		const activationScriptPath = path.join(settingsDir, "activate-cline.js")
+		const activationScript = `
+			// This script will be executed to activate Cline and start the test server
+			const vscode = require('vscode');
+			
+			// Execute the cline-activator.activate command
+			vscode.commands.executeCommand('cline-activator.activate');
+		`
+		fs.writeFileSync(activationScriptPath, activationScript)
+		console.log(`Created activation script to run in VS Code`)
+
+		// Execute the activation script
+		try {
+			console.log("Executing activation script to start Cline and test server...")
+			await execa(
+				"code",
+				[
+					"--user-data-dir",
+					tempUserDataDir,
+					"--extensions-dir",
+					tempExtensionsDir,
+					"--folder-uri",
+					`file://${workspacePath}`,
+					"--execute",
+					activationScriptPath,
+				],
+				{
+					stdio: "inherit",
+				},
+			)
+
+			// Wait for the test server to start
+			console.log("Waiting for test server to start...")
+			for (let i = 0; i < 30; i++) {
+				try {
+					// Try to connect to the test server
+					const response = await fetch("http://localhost:9876/task", {
+						method: "OPTIONS",
+						headers: {
+							"Content-Type": "application/json",
+						},
+					})
+
+					if (response.status === 204) {
+						console.log("Test server is running!")
+						serverStarted = true
+						break
+					}
+				} catch (error) {
+					// Server not started yet, wait and try again
+					await new Promise((resolve) => setTimeout(resolve, 1000))
+				}
+			}
+		} catch (error) {
+			console.warn("Failed to execute activation script:", error)
+		}
+
+		if (!serverStarted) {
+			console.warn("Test server did not start after multiple attempts")
+			console.log("You may need to manually open the Cline extension in VS Code")
+		}
+
+		// Store the resources for this workspace
+		const resources: VSCodeResources = {
+			tempUserDataDir,
+			tempExtensionsDir,
+		}
+
+		// Store in the global map
+		workspaceResources.set(workspacePath, resources)
+
+		// Return the resources
+		return resources
+	} catch (error: any) {
+		throw new Error(`Failed to spawn VSCode: ${error.message}`)
+	}
+}
+
+/**
+ * Clean up VS Code resources and shut down the test server
+ * @param workspacePath The workspace path to clean up resources for
+ */
+export async function cleanupVSCode(workspacePath: string): Promise<void> {
+	console.log(`Cleaning up VS Code resources for workspace: ${workspacePath}`)
+
+	// Get the resources for this workspace
+	const resources = workspaceResources.get(workspacePath)
+	if (!resources) {
+		console.log(`No resources found for workspace: ${workspacePath}`)
+		return
+	}
+
+	// Try to shut down the test server
+	try {
+		console.log("Shutting down test server...")
+		await fetch("http://localhost:9876/shutdown", {
+			method: "POST",
+			headers: {
+				"Content-Type": "application/json",
+			},
+		}).catch(() => {
+			// Ignore errors, the server might already be down
+		})
+	} catch (error) {
+		console.warn(`Error shutting down test server: ${error}`)
+	}
+
+	// Try to gracefully close VS Code instead of killing it
+	try {
+		console.log("Attempting to gracefully close VS Code...")
+
+		// Create a settings file that will disable the crash reporter and the exit confirmation dialog
+		const settingsDir = path.join(resources.tempUserDataDir, "User")
+		const settingsPath = path.join(settingsDir, "settings.json")
+
+		// Read existing settings if they exist
+		let settings = {}
+		if (fs.existsSync(settingsPath)) {
+			try {
+				settings = JSON.parse(fs.readFileSync(settingsPath, "utf8"))
+			} catch (error) {
+				console.warn(`Error reading settings file: ${error}`)
+			}
+		}
+
+		// Update settings to disable crash reporter and exit confirmation
+		settings = {
+			...settings,
+			"window.confirmBeforeClose": "never",
+			"telemetry.enableCrashReporter": false,
+			"window.restoreWindows": "none",
+			"window.newWindowDimensions": "default",
+		}
+
+		// Write updated settings
+		fs.writeFileSync(settingsPath, JSON.stringify(settings, null, 2))
+
+		// On macOS, use AppleScript to quit VS Code gracefully
+		if (process.platform === "darwin") {
+			try {
+				// First try AppleScript to quit VS Code gracefully
+				await execa("osascript", ["-e", 'tell application "Visual Studio Code" to quit'])
+
+				// Wait a moment for VS Code to close
+				await new Promise((resolve) => setTimeout(resolve, 2000))
+			} catch (appleScriptError) {
+				console.warn(`Error using AppleScript to quit VS Code: ${appleScriptError}`)
+			}
+		} else if (process.platform === "win32") {
+			// On Windows, try to use taskkill without /F first
+			try {
+				await execa("taskkill", ["/IM", "code.exe"])
+
+				// Wait a moment for VS Code to close
+				await new Promise((resolve) => setTimeout(resolve, 2000))
+			} catch (taskkillError) {
+				console.warn(`Error using taskkill to quit VS Code: ${taskkillError}`)
+			}
+		} else {
+			// On Linux, try to use SIGTERM first
+			try {
+				// Find VS Code processes
+				const { stdout } = await execa("ps", ["aux"])
+				const lines = stdout.split("\n")
+
+				for (const line of lines) {
+					if (line.includes(resources.tempUserDataDir)) {
+						const parts = line.trim().split(/\s+/)
+						const pid = parseInt(parts[1])
+
+						if (pid && !isNaN(pid)) {
+							console.log(`Sending SIGTERM to VS Code process with PID: ${pid}`)
+							try {
+								// Use SIGTERM instead of SIGKILL for a graceful shutdown
+								process.kill(pid, "SIGTERM")
+							} catch (killError) {
+								console.warn(`Failed to terminate process ${pid}: ${killError}`)
+							}
+						}
+					}
+				}
+
+				// Wait a moment for VS Code to close
+				await new Promise((resolve) => setTimeout(resolve, 2000))
+			} catch (psError) {
+				console.warn(`Error listing processes: ${psError}`)
+			}
+		}
+
+		// If graceful methods failed, fall back to forceful termination as a last resort
+		// Check if VS Code is still running with the temp user data dir
+		let vsCodeStillRunning = false
+
+		if (process.platform !== "win32") {
+			try {
+				const { stdout } = await execa("ps", ["aux"])
+				vsCodeStillRunning = stdout.split("\n").some((line) => line.includes(resources.tempUserDataDir))
+			} catch (error) {
+				console.warn(`Error checking if VS Code is still running: ${error}`)
+			}
+		} else {
+			try {
+				const { stdout } = await execa("tasklist", ["/FI", `IMAGENAME eq code.exe`])
+				vsCodeStillRunning = stdout.includes("code.exe")
+			} catch (error) {
+				console.warn(`Error checking if VS Code is still running: ${error}`)
+			}
+		}
+
+		// If VS Code is still running, use forceful termination as a last resort
+		if (vsCodeStillRunning) {
+			console.log("Graceful shutdown failed, falling back to forceful termination...")
+
+			if (process.platform === "win32") {
+				try {
+					await execa("taskkill", ["/IM", "code.exe", "/F"])
+				} catch (error) {
+					console.warn(`Error forcefully terminating VS Code: ${error}`)
+				}
+			} else {
+				try {
+					const { stdout } = await execa("ps", ["aux"])
+					const lines = stdout.split("\n")
+
+					for (const line of lines) {
+						if (line.includes(resources.tempUserDataDir)) {
+							const parts = line.trim().split(/\s+/)
+							const pid = parseInt(parts[1])
+
+							if (pid && !isNaN(pid)) {
+								console.log(`Forcefully killing VS Code process with PID: ${pid}`)
+								try {
+									process.kill(pid, "SIGKILL")
+								} catch (killError) {
+									console.warn(`Failed to kill process ${pid}: ${killError}`)
+								}
+							}
+						}
+					}
+				} catch (error) {
+					console.warn(`Error forcefully terminating VS Code: ${error}`)
+				}
+			}
+		}
+	} catch (error) {
+		console.warn(`Error closing VS Code: ${error}`)
+	}
+
+	// Clean up temporary directories and evals.env file
+	try {
+		console.log(`Removing temporary user data directory: ${resources.tempUserDataDir}`)
+		fs.rmSync(resources.tempUserDataDir, { recursive: true, force: true })
+	} catch (error) {
+		console.warn(`Error removing temporary user data directory: ${error}`)
+	}
+
+	try {
+		console.log(`Removing temporary extensions directory: ${resources.tempExtensionsDir}`)
+		fs.rmSync(resources.tempExtensionsDir, { recursive: true, force: true })
+	} catch (error) {
+		console.warn(`Error removing temporary extensions directory: ${error}`)
+	}
+
+	// Remove the evals.env file
+	try {
+		const evalsEnvPath = path.join(workspacePath, "evals.env")
+		if (fs.existsSync(evalsEnvPath)) {
+			console.log(`Removing evals.env file: ${evalsEnvPath}`)
+			fs.unlinkSync(evalsEnvPath)
+		}
+	} catch (error) {
+		console.warn(`Error removing evals.env file: ${error}`)
+	}
+
+	// Remove from the global map
+	workspaceResources.delete(workspacePath)
+
+	console.log("Cleanup completed")
+}

evals/cli/tsconfig.json

@@ -0,0 +1,17 @@
+{
+	"compilerOptions": {
+		"target": "ES2020",
+		"module": "commonjs",
+		"lib": ["ES2020"],
+		"declaration": true,
+		"outDir": "dist",
+		"rootDir": "src",
+		"strict": true,
+		"esModuleInterop": true,
+		"skipLibCheck": true,
+		"forceConsistentCasingInFileNames": true,
+		"resolveJsonModule": true
+	},
+	"include": ["src/**/*"],
+	"exclude": ["node_modules", "dist"]
+}

locales/ar-sa/README.md

@@ -2,7 +2,7 @@
 العربية | <a href="https://github.com/cline/cline/blob/main/locales/es/README.md" target="_blank">الإسبانية</a> | <a href="https://github.com/cline/cline/blob/main/locales/de/README.md" target="_blank">الألمانية</a> | <a href="https://github.com/cline/cline/blob/main/locales/ja/README.md" target="_blank">اليابانية</a> | <a href="https://github.com/cline/cline/blob/main/locales/zh-cn/README.md" target="_blank">الصينية المبسطة</a> | <a href="https://github.com/cline/cline/blob/main/locales/zh-tw/README.md" target="_blank">الصينية التقليدية</a> | <a href="https://github.com/cline/cline/blob/main/locales/pt-BR/README.md" target="_blank">البرتغالية</a>
 </sub></div>
 
-# Cline – \#1 على OpenRouter
+# Cline
 
 <p align="center">
   <img src="https://media.githubusercontent.com/media/cline/cline/main/assets/docs/demo.gif" width="100%" />

locales/de/README.md

@@ -1,4 +1,4 @@
-# Cline – \#1 auf OpenRouter
+# Cline
 
 <p align="center">
         <img src="https://media.githubusercontent.com/media/cline/cline/main/assets/docs/demo.gif" width="100%" />

locales/es/README.md

@@ -1,4 +1,4 @@
-# Cline – #1 en OpenRouter
+# Cline
 
 <p align="center">
         <img src="https://media.githubusercontent.com/media/cline/cline/main/assets/docs/demo.gif" width="100%" />

locales/ja/CONTRIBUTING.md

@@ -1,4 +1,4 @@
-# Clineへの貢献
+# Cline
 
 Clineへの貢献に興味をお持ちいただきありがとうございます。
 

locales/ja/README.md

@@ -1,4 +1,4 @@
-# Cline – OpenRouterでのナンバーワン
+# Cline
 
 <p align="center">
     <img src="https://media.githubusercontent.com/media/cline/cline/main/assets/docs/demo.gif" width="100%" />

locales/ko/CONTRIBUTING.md

@@ -1,4 +1,4 @@
-# Cline에 기여하기
+# Cline
 
 Cline에 기여하는 것에 관심을 가져주셔서 감사합니다! 버그 수정, 기능 추가, 문서 개선 등 모든 기여는 Cline을 더욱 스마트하게 만드는 데 기여합니다. 활기차고 환영하는 커뮤니티를 유지하기 위해 모든 구성원은 [행동 강령](CODE_OF_CONDUCT.md)을 준수해야 합니다.
 

locales/ko/README.md

@@ -1,4 +1,4 @@
-# Cline - 최고의 OpenRouter
+# Cline
 
 <p align="center">
     <img src="https://media.githubusercontent.com/media/cline/cline/main/assets/docs/demo.gif" width="100%" />

locales/pt-BR/README.md

@@ -1,4 +1,4 @@
-# Cline – #1 no OpenRouter
+# Cline
 
 <p align="center">
         <img src="https://media.githubusercontent.com/media/cline/cline/main/assets/docs/demo.gif" width="100%" />

locales/zh-cn/README.md

@@ -1,4 +1,4 @@
-# Cline – OpenRouter 排名第一的 AI 工具
+# Cline
 
 <p align="center">
     <img src="https://media.githubusercontent.com/media/cline/cline/main/assets/docs/demo.gif" width="100%" />
@@ -26,17 +26,17 @@
 </table>
 </div>
 
-认识 Cline，一个可以使用你的 **CLI** 和 **编辑器** 的 AI 助手。
+认识 Cline —— 一个可以使用你的 **终端** 和 **编辑器** 的 AI 助手。
 
-感谢 [Claude 3.7 Sonnet 的代理编码能力](https://www.anthropic.com/claude/sonnet)，Cline 可以一步步处理复杂的软件开发任务。通过允许他创建和编辑文件、探索大型项目、使用浏览器和执行终端命令（在你授予权限后），他可以提供超越代码完成或技术支持的帮助。Cline 甚至可以使用 Model Context Protocol (MCP) 创建新工具并扩展自己的能力。虽然自主 AI 脚本传统上在沙盒环境中运行，但此扩展提供了一个人机交互的 GUI 来批准每个文件更改和终端命令，提供了一种安全且可访问的方式来探索代理 AI 的潜力。
+得益于 [Claude 3.7 Sonnet 的代理式编码能力](https://www.anthropic.com/claude/sonnet)，Cline 能够逐步处理复杂的软件开发任务。借助于一系列工具，他可以创建和编辑文件、浏览大型项目、使用浏览器，并在你授权后执行终端命令，从而在代码补全或技术支持之外提供更深入的帮助。Cline 甚至还能使用 Model Context Protocol（MCP）来创建新工具，并扩展自身的能力。虽然传统的自动化 AI 脚本通常运行在沙盒环境中，但这个扩展提供了一个人类参与审核的图形界面（GUI），用于审批每一次文件变更和终端命令，从而为探索代理式 AI 的潜力提供了一种安全且易于使用的方式。
 
-1. 输入你的任务并添加图像，将模型转换为功能应用程序或通过截图修复错误。
-2. Cline 首先分析你的文件结构和源代码 AST，运行正则表达式搜索，并阅读相关文件以了解现有项目。通过仔细管理添加到上下文中的信息，Cline 即使在大型复杂项目中也能提供有价值的帮助，而不会使上下文窗口过载。
-3. 一旦 Cline 获得所需信息，他可以：
-   - 创建和编辑文件 + 监控 linter/编译器错误，从而主动修复诸如缺少导入和语法错误等问题。
-   - 直接在你的终端中执行命令并监控其输出，从而在编辑文件后对开发服务器问题做出反应。
-   - 对于 Web 开发任务，Cline 可以在无头浏览器中启动网站，点击、输入、滚动并捕获截图和控制台日志，从而修复运行时错误和视觉错误。
-4. 当任务完成时，Cline 将通过终端命令如 `open -a "Google Chrome" index.html` 向你展示结果，你可以通过点击按钮运行该命令。
+1. 输入你的任务，并添加图片，以将界面原型（mockup）转换为功能应用，或通过截图修复 bug。
+2. Cline 会从分析你的文件结构和源代码的抽象语法树（AST）开始，同时执行正则搜索并读取相关文件，以便尽快熟悉项目上下文。通过精细地管理上下文中引入的信息，即使面对大型复杂项目，Cline 也能在不超出上下文窗口限制的前提下提供有效协助。
+3. 一旦获取了所需信息，Cline 能够：
+   - 创建和编辑文件，并在过程中监控 linter 或编译器错误，主动修复诸如缺少导入、语法错误等问题。
+   - 直接在你的终端中执行命令，并在运行过程中监控输出，例如在修改文件后自动响应开发服务器问题。
+   - 针对 Web 开发任务，Cline 可以在无头浏览器中打开网站，进行点击、输入、滚动操作，并采集截图与控制台日志，从而修复运行时错误和界面问题。
+4. 当任务完成后，Cline 会通过类似 `open -a "Google Chrome" index.html` 的终端命令将结果展示给你，你只需点击按钮即可执行。
 
 > [!TIP]
 > 使用 `CMD/CTRL + Shift + P` 快捷键打开命令面板并输入 "Cline: Open In New Tab" 将扩展作为标签在编辑器中打开。这让你可以与文件资源管理器并排使用 Cline，更清楚地看到他如何改变你的工作空间。
@@ -49,7 +49,7 @@
 
 Cline 支持 OpenRouter、Anthropic、OpenAI、Google Gemini、AWS Bedrock、Azure 和 GCP Vertex 等 API 提供商。你还可以配置任何兼容 OpenAI 的 API，或通过 LM Studio/Ollama 使用本地模型。如果你使用 OpenRouter，扩展会获取他们的最新模型列表，让你在新模型可用时立即使用。
 
-扩展还会跟踪整个任务循环和单个请求的总令牌和 API 使用成本，让你在每一步都了解支出情况。
+此外，该扩展还会记录整个任务流程中以及每次请求的总 token 数和 API 使用费用，确保你在每一步都能清楚了解花费情况。
 
 <!-- 透明像素以在浮动图像后创建换行 -->
 

locales/zh-tw/README.md

@@ -2,7 +2,7 @@
 <a href="https://github.com/cline/cline/blob/main/README.md" target="_blank">English</a> | <a href="https://github.com/cline/cline/blob/main/locales/es/README.md" target="_blank">Español</a> | <a href="https://github.com/cline/cline/blob/main/locales/de/README.md" target="_blank">Deutsch</a> | <a href="https://github.com/cline/cline/blob/main/locales/ja/README.md" target="_blank">日本語</a> | <a href="https://github.com/cline/cline/blob/main/locales/zh-cn/README.md" target="_blank">简体中文</a> | 繁體中文 | <a href="https://github.com/cline/cline/blob/main/locales/ko/README.md" target="_blank">한국어</a>
 </sub></div>
 
-# Cline – OpenRouter 第一名的 AI 工具
+# Cline
 
 <p align="center">
     <img src="https://media.githubusercontent.com/media/cline/cline/main/assets/docs/demo.gif" width="100%" />
@@ -32,7 +32,7 @@
 
 認識 Cline，一個可以使用您的**命令列介面** (CLI) 和**程式編輯器** (Editor) 的 AI 助理。
 
-感謝 [Claude 3.7 Sonnet 的代理式程式設計能力](https://www.anthropic.com/claude/sonnet)，Cline 能夠逐步處理複雜的軟體開發任務。透過能讓他建立和編輯檔案、探索大型專案、使用瀏覽器，以及執行終端機指令（在您授權後）的工具，他能以超越程式碼自動完成或技術支援的方式協助您。Cline 甚至能使用模型上下文協定（Model Context Protocol，MCP）來建立新工具並擴展自己的功能。雖然自主 AI 腳本傳統上會在沙箱環境中執行，但這個擴充套件提供了人機互動的圖形介面，讓您可以核准每個檔案變更和終端機指令，提供一個安全且容易使用的方式來探索代理式 AI 的潛力。
+感謝 [Claude 3.7 Sonnet 的代理式程式設計能力](https://www.anthropic.com/claude/sonnet)，Cline 能夠逐步處理複雜的軟體開發任務。透過能讓他建立和編輯檔案、探索大型專案、使用瀏覽器，以及執行終端機指令（在您授權後）的工具，從而在程式碼補全或技術支援之外提供更深入的協助。Cline 甚至能使用模型上下文協定（Model Context Protocol，MCP）來建立新工具並擴展自己的功能。雖然自主 AI 腳本傳統上會在沙箱環境中執行，但這個擴充套件提供了人機互動的圖形介面，讓您可以核准每個檔案變更和終端機指令，提供一個安全且容易使用的方式來探索代理式 AI 的潛力。
 
 1. 輸入您的任務，並可以加入圖片來將設計稿轉換成功能性應用程式，或使用截圖來修正錯誤。
 2. Cline 會先分析您的檔案結構和程式碼 AST、執行正規表達式搜尋，並讀取相關檔案，以便在現有專案中快速掌握狀況。透過仔細管理加入上下文的資訊，Cline 可以在不超過上下文視窗的情況下，為大型且複雜的專案提供有價值的協助。

package-lock.json

@@ -1,20 +1,22 @@
 {
 	"name": "claude-dev",
-	"version": "3.10.0",
+	"version": "3.13.1",
 	"lockfileVersion": 3,
 	"requires": true,
 	"packages": {
 		"": {
 			"name": "claude-dev",
-			"version": "3.10.0",
+			"version": "3.13.1",
 			"license": "Apache-2.0",
 			"dependencies": {
 				"@anthropic-ai/bedrock-sdk": "^0.12.4",
 				"@anthropic-ai/sdk": "^0.37.0",
 				"@anthropic-ai/vertex-sdk": "^0.6.4",
 				"@aws-sdk/client-bedrock-runtime": "^3.758.0",
+				"@bufbuild/protobuf": "^2.2.5",
 				"@google-cloud/vertexai": "^1.9.3",
-				"@google/generative-ai": "^0.18.0",
+				"@google/genai": "^0.9.0",
+				"@grpc/grpc-js": "^1.9.15",
 				"@mistralai/mistralai": "^1.5.0",
 				"@modelcontextprotocol/sdk": "^1.7.0",
 				"@opentelemetry/api": "^1.4.1",
@@ -38,8 +40,10 @@
 				"fzf": "^0.5.2",
 				"get-folder-size": "^5.0.0",
 				"globby": "^14.0.2",
+				"iconv-lite": "^0.6.3",
 				"ignore": "^7.0.3",
 				"isbinaryfile": "^5.0.2",
+				"jschardet": "^3.1.4",
 				"mammoth": "^1.8.0",
 				"monaco-vscode-textmate-theme-converter": "^0.1.7",
 				"ollama": "^0.5.13",
@@ -79,15 +83,20 @@
 				"@vscode/test-cli": "^0.0.9",
 				"@vscode/test-electron": "^2.4.0",
 				"chai": "^4.3.10",
+				"chalk": "^5.3.0",
 				"esbuild": "^0.25.0",
 				"eslint": "^8.57.0",
+				"grpc-tools": "^1.13.0",
 				"husky": "^9.1.7",
 				"npm-run-all": "^4.1.5",
 				"prettier": "^3.3.3",
+				"protoc-gen-ts": "^0.8.7",
 				"proxyquire": "^2.1.3",
 				"should": "^13.2.3",
 				"sinon": "^19.0.2",
 				"ts-node": "^10.9.2",
+				"ts-proto": "^2.6.1",
+				"tsconfig-paths": "^4.2.0",
 				"typescript": "^5.4.5"
 			},
 			"engines": {
@@ -3957,6 +3966,12 @@
 			"dev": true,
 			"license": "MIT"
 		},
+		"node_modules/@bufbuild/protobuf": {
+			"version": "2.2.5",
+			"resolved": "https://registry.npmjs.org/@bufbuild/protobuf/-/protobuf-2.2.5.tgz",
+			"integrity": "sha512-/g5EzJifw5GF8aren8wZ/G5oMuPoGeS6MQD3ca8ddcvdXR5UELUfdTZITCGNhNXynY/AYl3Z4plmxdj/tRl/hQ==",
+			"license": "(Apache-2.0 AND BSD-3-Clause)"
+		},
 		"node_modules/@changesets/apply-release-plan": {
 			"version": "7.0.8",
 			"resolved": "https://registry.npmjs.org/@changesets/apply-release-plan/-/apply-release-plan-7.0.8.tgz",
@@ -5553,11 +5568,17 @@
 				"node": ">=18.0.0"
 			}
 		},
-		"node_modules/@google/generative-ai": {
-			"version": "0.18.0",
-			"resolved": "https://registry.npmjs.org/@google/generative-ai/-/generative-ai-0.18.0.tgz",
-			"integrity": "sha512-AhaIWSpk2tuhYHrBhUqC0xrWWznmYEja1/TRDIb+5kruBU5kUzMlFsXCQNO9PzyTZ4clUJ3CX/Rvy+Xm9x+w3g==",
+		"node_modules/@google/genai": {
+			"version": "0.9.0",
+			"resolved": "https://registry.npmjs.org/@google/genai/-/genai-0.9.0.tgz",
+			"integrity": "sha512-FD2RizYGInsvfjeaN6O+wQGpRnGVglS1XWrGQr8K7D04AfMmvPodDSw94U9KyFtsVLzWH9kmlPyFM+G4jbmkqg==",
 			"license": "Apache-2.0",
+			"dependencies": {
+				"google-auth-library": "^9.14.2",
+				"ws": "^8.18.0",
+				"zod": "^3.22.4",
+				"zod-to-json-schema": "^3.22.4"
+			},
 			"engines": {
 				"node": ">=18.0.0"
 			}
@@ -5566,6 +5587,7 @@
 			"version": "1.9.15",
 			"resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.9.15.tgz",
 			"integrity": "sha512-nqE7Hc0AzI+euzUwDAy0aY5hCp10r734gMGRdU+qOPX0XSceI2ULrcXB5U2xSc5VkWwalCj4M7GzCAygZl2KoQ==",
+			"license": "Apache-2.0",
 			"dependencies": {
 				"@grpc/proto-loader": "^0.7.8",
 				"@types/node": ">=12.12.47"
@@ -5892,6 +5914,90 @@
 				"node": ">=8"
 			}
 		},
+		"node_modules/@mapbox/node-pre-gyp": {
+			"version": "1.0.11",
+			"resolved": "https://registry.npmjs.org/@mapbox/node-pre-gyp/-/node-pre-gyp-1.0.11.tgz",
+			"integrity": "sha512-Yhlar6v9WQgUp/He7BdgzOz8lqMQ8sU+jkCq7Wx8Myc5YFJLbEe7lgui/V7G1qB1DJykHSGwreceSaD60Y0PUQ==",
+			"dev": true,
+			"license": "BSD-3-Clause",
+			"dependencies": {
+				"detect-libc": "^2.0.0",
+				"https-proxy-agent": "^5.0.0",
+				"make-dir": "^3.1.0",
+				"node-fetch": "^2.6.7",
+				"nopt": "^5.0.0",
+				"npmlog": "^5.0.1",
+				"rimraf": "^3.0.2",
+				"semver": "^7.3.5",
+				"tar": "^6.1.11"
+			},
+			"bin": {
+				"node-pre-gyp": "bin/node-pre-gyp"
+			}
+		},
+		"node_modules/@mapbox/node-pre-gyp/node_modules/agent-base": {
+			"version": "6.0.2",
+			"resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz",
+			"integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==",
+			"dev": true,
+			"license": "MIT",
+			"dependencies": {
+				"debug": "4"
+			},
+			"engines": {
+				"node": ">= 6.0.0"
+			}
+		},
+		"node_modules/@mapbox/node-pre-gyp/node_modules/detect-libc": {
+			"version": "2.0.4",
+			"resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.4.tgz",
+			"integrity": "sha512-3UDv+G9CsCKO1WKMGw9fwq/SWJYbI0c5Y7LU1AXYoDdbhE2AHQ6N6Nb34sG8Fj7T5APy8qXDCKuuIHd1BR0tVA==",
+			"dev": true,
+			"license": "Apache-2.0",
+			"engines": {
+				"node": ">=8"
+			}
+		},
+		"node_modules/@mapbox/node-pre-gyp/node_modules/https-proxy-agent": {
+			"version": "5.0.1",
+			"resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz",
+			"integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==",
+			"dev": true,
+			"license": "MIT",
+			"dependencies": {
+				"agent-base": "6",
+				"debug": "4"
+			},
+			"engines": {
+				"node": ">= 6"
+			}
+		},
+		"node_modules/@mapbox/node-pre-gyp/node_modules/make-dir": {
+			"version": "3.1.0",
+			"resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz",
+			"integrity": "sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==",
+			"dev": true,
+			"license": "MIT",
+			"dependencies": {
+				"semver": "^6.0.0"
+			},
+			"engines": {
+				"node": ">=8"
+			},
+			"funding": {
+				"url": "https://github.com/sponsors/sindresorhus"
+			}
+		},
+		"node_modules/@mapbox/node-pre-gyp/node_modules/make-dir/node_modules/semver": {
+			"version": "6.3.1",
+			"resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz",
+			"integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==",
+			"dev": true,
+			"license": "ISC",
+			"bin": {
+				"semver": "bin/semver.js"
+			}
+		},
 		"node_modules/@mistralai/mistralai": {
 			"version": "1.5.0",
 			"resolved": "https://registry.npmjs.org/@mistralai/mistralai/-/mistralai-1.5.0.tgz",
@@ -9183,6 +9289,13 @@
 				"node": ">=10.0.0"
 			}
 		},
+		"node_modules/abbrev": {
+			"version": "1.1.1",
+			"resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz",
+			"integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==",
+			"dev": true,
+			"license": "ISC"
+		},
 		"node_modules/abort-controller": {
 			"version": "3.0.0",
 			"resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz",
@@ -9368,6 +9481,36 @@
 			"integrity": "sha512-lYe4Gx7QT+MKGbDsA+Z+he/Wtef0BiwDOlK/XkBrdfsh9J/jPPXbX0tE9x9cl27Tmu5gg3QUbUrQYa/y+KOHPQ==",
 			"license": "ISC"
 		},
+		"node_modules/are-we-there-yet": {
+			"version": "2.0.0",
+			"resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-2.0.0.tgz",
+			"integrity": "sha512-Ci/qENmwHnsYo9xKIcUJN5LeDKdJ6R1Z1j9V/J5wyq8nh/mYPEpIKJbBZXtZjG04HiK7zV/p6Vs9952MrMeUIw==",
+			"deprecated": "This package is no longer supported.",
+			"dev": true,
+			"license": "ISC",
+			"dependencies": {
+				"delegates": "^1.0.0",
+				"readable-stream": "^3.6.0"
+			},
+			"engines": {
+				"node": ">=10"
+			}
+		},
+		"node_modules/are-we-there-yet/node_modules/readable-stream": {
+			"version": "3.6.2",
+			"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz",
+			"integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==",
+			"dev": true,
+			"license": "MIT",
+			"dependencies": {
+				"inherits": "^2.0.3",
+				"string_decoder": "^1.1.1",
+				"util-deprecate": "^1.0.1"
+			},
+			"engines": {
+				"node": ">= 6"
+			}
+		},
 		"node_modules/arg": {
 			"version": "4.1.3",
 			"resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz",
@@ -9919,6 +10062,19 @@
 				"url": "https://github.com/sponsors/sindresorhus"
 			}
 		},
+		"node_modules/case-anything": {
+			"version": "2.1.13",
+			"resolved": "https://registry.npmjs.org/case-anything/-/case-anything-2.1.13.tgz",
+			"integrity": "sha512-zlOQ80VrQ2Ue+ymH5OuM/DlDq64mEm+B9UTdHULv5osUMD6HalNTblf2b1u/m6QecjsnOkBpqVZ+XPwIVsy7Ng==",
+			"dev": true,
+			"license": "MIT",
+			"engines": {
+				"node": ">=12.13"
+			},
+			"funding": {
+				"url": "https://github.com/sponsors/mesqueeb"
+			}
+		},
 		"node_modules/chai": {
 			"version": "4.3.10",
 			"resolved": "https://registry.npmjs.org/chai/-/chai-4.3.10.tgz",
@@ -9938,35 +10094,18 @@
 			}
 		},
 		"node_modules/chalk": {
-			"version": "4.1.2",
-			"resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
-			"integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
+			"version": "5.4.1",
+			"resolved": "https://registry.npmjs.org/chalk/-/chalk-5.4.1.tgz",
+			"integrity": "sha512-zgVZuo2WcZgfUEmsn6eO3kINexW8RAE4maiQ8QNs8CtpPCSyMiYsULR3HQYkm3w8FIA3SberyMJMSldGsW+U3w==",
 			"dev": true,
 			"license": "MIT",
-			"dependencies": {
-				"ansi-styles": "^4.1.0",
-				"supports-color": "^7.1.0"
-			},
 			"engines": {
-				"node": ">=10"
+				"node": "^12.17.0 || ^14.13 || >=16.0.0"
 			},
 			"funding": {
 				"url": "https://github.com/chalk/chalk?sponsor=1"
 			}
 		},
-		"node_modules/chalk/node_modules/supports-color": {
-			"version": "7.2.0",
-			"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
-			"integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
-			"dev": true,
-			"license": "MIT",
-			"dependencies": {
-				"has-flag": "^4.0.0"
-			},
-			"engines": {
-				"node": ">=8"
-			}
-		},
 		"node_modules/chardet": {
 			"version": "0.7.0",
 			"resolved": "https://registry.npmjs.org/chardet/-/chardet-0.7.0.tgz",
@@ -10043,6 +10182,16 @@
 				"url": "https://paulmillr.com/funding/"
 			}
 		},
+		"node_modules/chownr": {
+			"version": "2.0.0",
+			"resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz",
+			"integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==",
+			"dev": true,
+			"license": "ISC",
+			"engines": {
+				"node": ">=10"
+			}
+		},
 		"node_modules/chrome-launcher": {
 			"version": "1.1.2",
 			"resolved": "https://registry.npmjs.org/chrome-launcher/-/chrome-launcher-1.1.2.tgz",
@@ -10583,6 +10732,13 @@
 				"node": ">=0.4.0"
 			}
 		},
+		"node_modules/delegates": {
+			"version": "1.0.0",
+			"resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz",
+			"integrity": "sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==",
+			"dev": true,
+			"license": "MIT"
+		},
 		"node_modules/depd": {
 			"version": "2.0.0",
 			"resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz",
@@ -10612,6 +10768,19 @@
 				"node": ">=8"
 			}
 		},
+		"node_modules/detect-libc": {
+			"version": "1.0.3",
+			"resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz",
+			"integrity": "sha512-pGjwhsmsp4kL2RTz08wcOlGN83otlqHeD/Z5T8GXZB+/YcpQ/dgo+lbU8ZsGxV0HIvqqxo9l7mqYwyYMD9bKDg==",
+			"dev": true,
+			"license": "Apache-2.0",
+			"bin": {
+				"detect-libc": "bin/detect-libc.js"
+			},
+			"engines": {
+				"node": ">=0.10"
+			}
+		},
 		"node_modules/devtools-protocol": {
 			"version": "0.0.1342118",
 			"resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1342118.tgz",
@@ -10722,6 +10891,16 @@
 				"url": "https://github.com/fb55/domutils?sponsor=1"
 			}
 		},
+		"node_modules/dprint-node": {
+			"version": "1.0.8",
+			"resolved": "https://registry.npmjs.org/dprint-node/-/dprint-node-1.0.8.tgz",
+			"integrity": "sha512-iVKnUtYfGrYcW1ZAlfR/F59cUVL8QIhWoBJoSjkkdua/dkWIgjZfiLMeTjiB06X0ZLkQ0M2C1VbUj/CxkIf1zg==",
+			"dev": true,
+			"license": "MIT",
+			"dependencies": {
+				"detect-libc": "^1.0.3"
+			}
+		},
 		"node_modules/duck": {
 			"version": "0.1.12",
 			"resolved": "https://registry.npmjs.org/duck/-/duck-0.1.12.tgz",
@@ -11192,6 +11371,23 @@
 				"concat-map": "0.0.1"
 			}
 		},
+		"node_modules/eslint/node_modules/chalk": {
+			"version": "4.1.2",
+			"resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
+			"integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
+			"dev": true,
+			"license": "MIT",
+			"dependencies": {
+				"ansi-styles": "^4.1.0",
+				"supports-color": "^7.1.0"
+			},
+			"engines": {
+				"node": ">=10"
+			},
+			"funding": {
+				"url": "https://github.com/chalk/chalk?sponsor=1"
+			}
+		},
 		"node_modules/eslint/node_modules/glob-parent": {
 			"version": "6.0.2",
 			"resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz",
@@ -11240,6 +11436,19 @@
 				"node": ">=8"
 			}
 		},
+		"node_modules/eslint/node_modules/supports-color": {
+			"version": "7.2.0",
+			"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+			"integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
+			"dev": true,
+			"license": "MIT",
+			"dependencies": {
+				"has-flag": "^4.0.0"
+			},
+			"engines": {
+				"node": ">=8"
+			}
+		},
 		"node_modules/espree": {
 			"version": "9.6.1",
 			"resolved": "https://registry.npmjs.org/espree/-/espree-9.6.1.tgz",
@@ -12031,6 +12240,32 @@
 				"node": ">=6 <7 || >=8"
 			}
 		},
+		"node_modules/fs-minipass": {
+			"version": "2.1.0",
+			"resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz",
+			"integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==",
+			"dev": true,
+			"license": "ISC",
+			"dependencies": {
+				"minipass": "^3.0.0"
+			},
+			"engines": {
+				"node": ">= 8"
+			}
+		},
+		"node_modules/fs-minipass/node_modules/minipass": {
+			"version": "3.3.6",
+			"resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz",
+			"integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==",
+			"dev": true,
+			"license": "ISC",
+			"dependencies": {
+				"yallist": "^4.0.0"
+			},
+			"engines": {
+				"node": ">=8"
+			}
+		},
 		"node_modules/fs.realpath": {
 			"version": "1.0.0",
 			"resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
@@ -12434,9 +12669,10 @@
 			}
 		},
 		"node_modules/google-auth-library": {
-			"version": "9.14.0",
-			"resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-9.14.0.tgz",
-			"integrity": "sha512-Y/eq+RWVs55Io/anIsm24sDS8X79Tq948zVLGaa7+KlJYYqaGwp1YI37w48nzrNi12RgnzMrQD4NzdmCowT90g==",
+			"version": "9.15.1",
+			"resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-9.15.1.tgz",
+			"integrity": "sha512-Jb6Z0+nvECVz+2lzSMt9u98UsoakXxA2HGHMCxh+so3n90XgYWkq5dur19JAJV7ONiJY22yBTyJB1TSkvPq9Ng==",
+			"license": "Apache-2.0",
 			"dependencies": {
 				"base64-js": "^1.3.0",
 				"ecdsa-sig-formatter": "^1.0.11",
@@ -12474,6 +12710,20 @@
 			"dev": true,
 			"license": "MIT"
 		},
+		"node_modules/grpc-tools": {
+			"version": "1.13.0",
+			"resolved": "https://registry.npmjs.org/grpc-tools/-/grpc-tools-1.13.0.tgz",
+			"integrity": "sha512-7CbkJ1yWPfX0nHjbYG58BQThNhbICXBZynzCUxCb3LzX5X9B3hQbRY2STiRgIEiLILlK9fgl0z0QVGwPCdXf5g==",
+			"dev": true,
+			"hasInstallScript": true,
+			"dependencies": {
+				"@mapbox/node-pre-gyp": "^1.0.5"
+			},
+			"bin": {
+				"grpc_tools_node_protoc": "bin/protoc.js",
+				"grpc_tools_node_protoc_plugin": "bin/protoc_plugin.js"
+			}
+		},
 		"node_modules/gtoken": {
 			"version": "7.1.0",
 			"resolved": "https://registry.npmjs.org/gtoken/-/gtoken-7.1.0.tgz",
@@ -13435,6 +13685,14 @@
 			"integrity": "sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A==",
 			"license": "MIT"
 		},
+		"node_modules/jschardet": {
+			"version": "3.1.4",
+			"resolved": "https://registry.npmjs.org/jschardet/-/jschardet-3.1.4.tgz",
+			"integrity": "sha512-/kmVISmrwVwtyYU40iQUOp3SUPk2dhNCMsZBQX0R1/jZ8maaXJ/oZIzUOiyOqcgtLnETFKYChbJ5iDC/eWmFHg==",
+			"engines": {
+				"node": ">=0.1.90"
+			}
+		},
 		"node_modules/json-bigint": {
 			"version": "1.0.0",
 			"resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz",
@@ -13471,6 +13729,19 @@
 			"dev": true,
 			"license": "MIT"
 		},
+		"node_modules/json5": {
+			"version": "2.2.3",
+			"resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz",
+			"integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==",
+			"dev": true,
+			"license": "MIT",
+			"bin": {
+				"json5": "lib/cli.js"
+			},
+			"engines": {
+				"node": ">=6"
+			}
+		},
 		"node_modules/jsonfile": {
 			"version": "4.0.0",
 			"resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz",
@@ -13659,6 +13930,36 @@
 				"url": "https://github.com/sponsors/sindresorhus"
 			}
 		},
+		"node_modules/log-symbols/node_modules/chalk": {
+			"version": "4.1.2",
+			"resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
+			"integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
+			"dev": true,
+			"license": "MIT",
+			"dependencies": {
+				"ansi-styles": "^4.1.0",
+				"supports-color": "^7.1.0"
+			},
+			"engines": {
+				"node": ">=10"
+			},
+			"funding": {
+				"url": "https://github.com/chalk/chalk?sponsor=1"
+			}
+		},
+		"node_modules/log-symbols/node_modules/supports-color": {
+			"version": "7.2.0",
+			"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
+			"integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
+			"dev": true,
+			"license": "MIT",
+			"dependencies": {
+				"has-flag": "^4.0.0"
+			},
+			"engines": {
+				"node": ">=8"
+			}
+		},
 		"node_modules/long": {
 			"version": "5.2.4",
 			"resolved": "https://registry.npmjs.org/long/-/long-5.2.4.tgz",
@@ -13910,12 +14211,52 @@
 				"node": ">=16 || 14 >=14.17"
 			}
 		},
+		"node_modules/minizlib": {
+			"version": "2.1.2",
+			"resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz",
+			"integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==",
+			"dev": true,
+			"license": "MIT",
+			"dependencies": {
+				"minipass": "^3.0.0",
+				"yallist": "^4.0.0"
+			},
+			"engines": {
+				"node": ">= 8"
+			}
+		},
+		"node_modules/minizlib/node_modules/minipass": {
+			"version": "3.3.6",
+			"resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz",
+			"integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==",
+			"dev": true,
+			"license": "ISC",
+			"dependencies": {
+				"yallist": "^4.0.0"
+			},
+			"engines": {
+				"node": ">=8"
+			}
+		},
 		"node_modules/mitt": {
 			"version": "3.0.1",
 			"resolved": "https://registry.npmjs.org/mitt/-/mitt-3.0.1.tgz",
 			"integrity": "sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==",
 			"license": "MIT"
 		},
+		"node_modules/mkdirp": {
+			"version": "1.0.4",
+			"resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz",
+			"integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==",
+			"dev": true,
+			"license": "MIT",
+			"bin": {
+				"mkdirp": "bin/cmd.js"
+			},
+			"engines": {
+				"node": ">=10"
+			}
+		},
 		"node_modules/mocha": {
 			"version": "10.6.0",
 			"resolved": "https://registry.npmjs.org/mocha/-/mocha-10.6.0.tgz",
@@ -14283,6 +14624,22 @@
 			"integrity": "sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==",
 			"license": "MIT"
 		},
+		"node_modules/nopt": {
+			"version": "5.0.0",
+			"resolved": "https://registry.npmjs.org/nopt/-/nopt-5.0.0.tgz",
+			"integrity": "sha512-Tbj67rffqceeLpcRXrT7vKAN8CwfPeIBgM7E6iBkmKLV7bEMwpGgYLGv0jACUsECaa/vuxP0IjEont6umdMgtQ==",
+			"dev": true,
+			"license": "ISC",
+			"dependencies": {
+				"abbrev": "1"
+			},
+			"bin": {
+				"nopt": "bin/nopt.js"
+			},
+			"engines": {
+				"node": ">=6"
+			}
+		},
 		"node_modules/normalize-package-data": {
 			"version": "2.5.0",
 			"resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.5.0.tgz",
@@ -14544,6 +14901,84 @@
 				"url": "https://github.com/sponsors/sindresorhus"
 			}
 		},
+		"node_modules/npmlog": {
+			"version": "5.0.1",
+			"resolved": "https://registry.npmjs.org/npmlog/-/npmlog-5.0.1.tgz",
+			"integrity": "sha512-AqZtDUWOMKs1G/8lwylVjrdYgqA4d9nu8hc+0gzRxlDb1I10+FHBGMXs6aiQHFdCUUlqH99MUMuLfzWDNDtfxw==",
+			"deprecated": "This package is no longer supported.",
+			"dev": true,
+			"license": "ISC",
+			"dependencies": {
+				"are-we-there-yet": "^2.0.0",
+				"console-control-strings": "^1.1.0",
+				"gauge": "^3.0.0",
+				"set-blocking": "^2.0.0"
+			}
+		},
+		"node_modules/npmlog/node_modules/emoji-regex": {
+			"version": "8.0.0",
+			"resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
+			"integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==",
+			"dev": true,
+			"license": "MIT"
+		},
+		"node_modules/npmlog/node_modules/gauge": {
+			"version": "3.0.2",
+			"resolved": "https://registry.npmjs.org/gauge/-/gauge-3.0.2.tgz",
+			"integrity": "sha512-+5J6MS/5XksCuXq++uFRsnUd7Ovu1XenbeuIuNRJxYWjgQbPuFhT14lAvsWfqfAmnwluf1OwMjz39HjfLPci0Q==",
+			"deprecated": "This package is no longer supported.",
+			"dev": true,
+			"license": "ISC",
+			"dependencies": {
+				"aproba": "^1.0.3 || ^2.0.0",
+				"color-support": "^1.1.2",
+				"console-control-strings": "^1.0.0",
+				"has-unicode": "^2.0.1",
+				"object-assign": "^4.1.1",
+				"signal-exit": "^3.0.0",
+				"string-width": "^4.2.3",
+				"strip-ansi": "^6.0.1",
+				"wide-align": "^1.1.2"
+			},
+			"engines": {
+				"node": ">=10"
+			}
+		},
+		"node_modules/npmlog/node_modules/signal-exit": {
+			"version": "3.0.7",
+			"resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz",
+			"integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==",
+			"dev": true,
+			"license": "ISC"
+		},
+		"node_modules/npmlog/node_modules/string-width": {
+			"version": "4.2.3",
+			"resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
+			"integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
+			"dev": true,
+			"license": "MIT",
+			"dependencies": {
+				"emoji-regex": "^8.0.0",
+				"is-fullwidth-code-point": "^3.0.0",
+				"strip-ansi": "^6.0.1"
+			},
+			"engines": {
+				"node": ">=8"
+			}
+		},
+		"node_modules/npmlog/node_modules/strip-ansi": {
+			"version": "6.0.1",
+			"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
+			"integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
+			"dev": true,
+			"license": "MIT",
+			"dependencies": {
+				"ansi-regex": "^5.0.1"
+			},
+			"engines": {
+				"node": ">=8"
+			}
+		},
 		"node_modules/nth-check": {
 			"version": "2.1.1",
 			"resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz",
@@ -14768,19 +15203,6 @@
 				"url": "https://github.com/sponsors/sindresorhus"
 			}
 		},
-		"node_modules/ora/node_modules/chalk": {
-			"version": "5.3.0",
-			"resolved": "https://registry.npmjs.org/chalk/-/chalk-5.3.0.tgz",
-			"integrity": "sha512-dLitG79d+GV1Nb/VYcCDFivJeK1hiukt9QjRNVOsUtTy1rR1YJsmpGGTZ3qJos+uw7WmWF4wUwBd9jxjocFC2w==",
-			"dev": true,
-			"license": "MIT",
-			"engines": {
-				"node": "^12.17.0 || ^14.13 || >=16.0.0"
-			},
-			"funding": {
-				"url": "https://github.com/chalk/chalk?sponsor=1"
-			}
-		},
 		"node_modules/ora/node_modules/emoji-regex": {
 			"version": "10.3.0",
 			"resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-10.3.0.tgz",
@@ -15366,6 +15788,20 @@
 				"node": ">=12.0.0"
 			}
 		},
+		"node_modules/protoc-gen-ts": {
+			"version": "0.8.7",
+			"resolved": "https://registry.npmjs.org/protoc-gen-ts/-/protoc-gen-ts-0.8.7.tgz",
+			"integrity": "sha512-jr4VJey2J9LVYCV7EVyVe53g1VMw28cCmYJhBe5e3YX5wiyiDwgxWxeDf9oTqAe4P1bN/YGAkW2jhlH8LohwiQ==",
+			"dev": true,
+			"license": "MIT",
+			"bin": {
+				"protoc-gen-ts": "protoc-gen-ts.js"
+			},
+			"funding": {
+				"type": "individual",
+				"url": "https://www.buymeacoffee.com/thesayyn"
+			}
+		},
 		"node_modules/proxy-addr": {
 			"version": "2.0.7",
 			"resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz",
@@ -16011,6 +16447,13 @@
 				"node": ">= 18"
 			}
 		},
+		"node_modules/set-blocking": {
+			"version": "2.0.0",
+			"resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz",
+			"integrity": "sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==",
+			"dev": true,
+			"license": "ISC"
+		},
 		"node_modules/set-function-length": {
 			"version": "1.2.2",
 			"resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz",
@@ -16712,6 +17155,24 @@
 				"node": ">=6"
 			}
 		},
+		"node_modules/tar": {
+			"version": "6.2.1",
+			"resolved": "https://registry.npmjs.org/tar/-/tar-6.2.1.tgz",
+			"integrity": "sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A==",
+			"dev": true,
+			"license": "ISC",
+			"dependencies": {
+				"chownr": "^2.0.0",
+				"fs-minipass": "^2.0.0",
+				"minipass": "^5.0.0",
+				"minizlib": "^2.1.1",
+				"mkdirp": "^1.0.3",
+				"yallist": "^4.0.0"
+			},
+			"engines": {
+				"node": ">=10"
+			}
+		},
 		"node_modules/tar-fs": {
 			"version": "3.0.8",
 			"resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.0.8.tgz",
@@ -16737,6 +17198,16 @@
 				"streamx": "^2.15.0"
 			}
 		},
+		"node_modules/tar/node_modules/minipass": {
+			"version": "5.0.0",
+			"resolved": "https://registry.npmjs.org/minipass/-/minipass-5.0.0.tgz",
+			"integrity": "sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ==",
+			"dev": true,
+			"license": "ISC",
+			"engines": {
+				"node": ">=8"
+			}
+		},
 		"node_modules/term-size": {
 			"version": "2.2.1",
 			"resolved": "https://registry.npmjs.org/term-size/-/term-size-2.2.1.tgz",
@@ -16970,6 +17441,57 @@
 				"node": ">=0.3.1"
 			}
 		},
+		"node_modules/ts-poet": {
+			"version": "6.11.0",
+			"resolved": "https://registry.npmjs.org/ts-poet/-/ts-poet-6.11.0.tgz",
+			"integrity": "sha512-r5AGF8vvb+GjBsnqiTqbLhN1/U2FJt6BI+k0dfCrkKzWvUhNlwMmq9nDHuucHs45LomgHjZPvYj96dD3JawjJA==",
+			"dev": true,
+			"license": "Apache-2.0",
+			"dependencies": {
+				"dprint-node": "^1.0.8"
+			}
+		},
+		"node_modules/ts-proto": {
+			"version": "2.7.0",
+			"resolved": "https://registry.npmjs.org/ts-proto/-/ts-proto-2.7.0.tgz",
+			"integrity": "sha512-BGHjse2wTOeswOqnnPKinpxmbaRd882so/e1En6ww59YMG7AO9Kg4vPpJcbVfrpBixPRDqHafXD/RDyd2T99GA==",
+			"dev": true,
+			"license": "ISC",
+			"dependencies": {
+				"@bufbuild/protobuf": "^2.0.0",
+				"case-anything": "^2.1.13",
+				"ts-poet": "^6.7.0",
+				"ts-proto-descriptors": "2.0.0"
+			},
+			"bin": {
+				"protoc-gen-ts_proto": "protoc-gen-ts_proto"
+			}
+		},
+		"node_modules/ts-proto-descriptors": {
+			"version": "2.0.0",
+			"resolved": "https://registry.npmjs.org/ts-proto-descriptors/-/ts-proto-descriptors-2.0.0.tgz",
+			"integrity": "sha512-wHcTH3xIv11jxgkX5OyCSFfw27agpInAd6yh89hKG6zqIXnjW9SYqSER2CVQxdPj4czeOhGagNvZBEbJPy7qkw==",
+			"dev": true,
+			"license": "ISC",
+			"dependencies": {
+				"@bufbuild/protobuf": "^2.0.0"
+			}
+		},
+		"node_modules/tsconfig-paths": {
+			"version": "4.2.0",
+			"resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-4.2.0.tgz",
+			"integrity": "sha512-NoZ4roiN7LnbKn9QqE1amc9DJfzvZXxF4xDavcOWt1BPkdx+m+0gJuPM+S0vCe7zTJMYUP0R8pO2XMr+Y8oLIg==",
+			"dev": true,
+			"license": "MIT",
+			"dependencies": {
+				"json5": "^2.2.2",
+				"minimist": "^1.2.6",
+				"strip-bom": "^3.0.0"
+			},
+			"engines": {
+				"node": ">=6"
+			}
+		},
 		"node_modules/tslib": {
 			"version": "1.14.1",
 			"resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz",
@@ -17770,6 +18292,13 @@
 				"node": ">=10"
 			}
 		},
+		"node_modules/yallist": {
+			"version": "4.0.0",
+			"resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
+			"integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
+			"dev": true,
+			"license": "ISC"
+		},
 		"node_modules/yargs": {
 			"version": "17.7.2",
 			"resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz",

package.json

@@ -2,7 +2,7 @@
 	"name": "claude-dev",
 	"displayName": "Cline",
 	"description": "Autonomous coding agent right in your IDE, capable of creating/editing files, running commands, using the browser, and more with your permission every step of the way.",
-	"version": "3.10.1",
+	"version": "3.13.3",
 	"icon": "assets/icons/icon.png",
 	"engines": {
 		"vscode": "^1.84.0"
@@ -39,7 +39,9 @@
 		"ai",
 		"llama"
 	],
-	"activationEvents": [],
+	"activationEvents": [
+		"workspaceContains:evals.env"
+	],
 	"main": "./dist/extension.js",
 	"contributes": {
 		"viewsContainers": {
@@ -113,8 +115,8 @@
 				"category": "Cline"
 			},
 			{
-				"command": "cline.fixWithCline",
-				"title": "Fix with Cline",
+				"command": "cline.focusChatInput",
+				"title": "Jump to Chat Input",
 				"category": "Cline"
 			}
 		],
@@ -224,21 +226,6 @@
 					},
 					"description": "Settings for VSCode Language Model API"
 				},
-				"cline.mcp.mode": {
-					"type": "string",
-					"enum": [
-						"full",
-						"server-use-only",
-						"off"
-					],
-					"enumDescriptions": [
-						"Enable all MCP functionality (server use and build instructions)",
-						"Enable MCP server use only (excludes instructions about building MCP servers)",
-						"Disable all MCP functionality"
-					],
-					"default": "full",
-					"description": "Controls MCP inclusion in prompts, reduces token usage if you only need access to certain functionality."
-				},
 				"cline.enableCheckpoints": {
 					"type": "boolean",
 					"default": true,
@@ -302,13 +289,10 @@
 		"compile": "npm run check-types && npm run lint && node esbuild.js",
 		"watch": "npm-run-all -p watch:*",
 		"watch:esbuild": "node esbuild.js --watch",
-		"watch:esbuild:test": "IS_TEST=true node esbuild.js --watch",
 		"watch:tsc": "tsc --noEmit --watch --project tsconfig.json",
 		"package": "npm run build:webview && npm run check-types && npm run lint && node esbuild.js --production",
-		"package:test": "IS_TEST=true npm run build:webview:test && npm run check-types && npm run lint && IS_TEST=true node esbuild.js --production",
-		"build:webview:test": "cd webview-ui && IS_TEST=true npm run build",
-		"watch:test": "IS_TEST=true npm-run-all -p watch:tsc watch:esbuild:test",
-		"compile-tests": "tsc -p ./tsconfig.test.json --outDir out",
+		"protos": "node proto/build-proto.js && prettier src/shared/proto --write && prettier src/core/controller --write",
+		"compile-tests": "node ./scripts/build-tests.js",
 		"watch-tests": "tsc -p . -w --outDir out",
 		"pretest": "npm run compile-tests && npm run compile && npm run lint",
 		"check-types": "tsc --noEmit",
@@ -349,15 +333,20 @@
 		"@vscode/test-cli": "^0.0.9",
 		"@vscode/test-electron": "^2.4.0",
 		"chai": "^4.3.10",
+		"chalk": "^5.3.0",
 		"esbuild": "^0.25.0",
 		"eslint": "^8.57.0",
+		"grpc-tools": "^1.13.0",
 		"husky": "^9.1.7",
 		"npm-run-all": "^4.1.5",
 		"prettier": "^3.3.3",
+		"protoc-gen-ts": "^0.8.7",
 		"proxyquire": "^2.1.3",
 		"should": "^13.2.3",
 		"sinon": "^19.0.2",
 		"ts-node": "^10.9.2",
+		"ts-proto": "^2.6.1",
+		"tsconfig-paths": "^4.2.0",
 		"typescript": "^5.4.5"
 	},
 	"dependencies": {
@@ -365,8 +354,10 @@
 		"@anthropic-ai/sdk": "^0.37.0",
 		"@anthropic-ai/vertex-sdk": "^0.6.4",
 		"@aws-sdk/client-bedrock-runtime": "^3.758.0",
+		"@bufbuild/protobuf": "^2.2.5",
 		"@google-cloud/vertexai": "^1.9.3",
-		"@google/generative-ai": "^0.18.0",
+		"@google/genai": "^0.9.0",
+		"@grpc/grpc-js": "^1.9.15",
 		"@mistralai/mistralai": "^1.5.0",
 		"@modelcontextprotocol/sdk": "^1.7.0",
 		"@opentelemetry/api": "^1.4.1",
@@ -390,8 +381,10 @@
 		"fzf": "^0.5.2",
 		"get-folder-size": "^5.0.0",
 		"globby": "^14.0.2",
+		"iconv-lite": "^0.6.3",
 		"ignore": "^7.0.3",
 		"isbinaryfile": "^5.0.2",
+		"jschardet": "^3.1.4",
 		"mammoth": "^1.8.0",
 		"monaco-vscode-textmate-theme-converter": "^0.1.7",
 		"ollama": "^0.5.13",

proto/account.proto

@@ -0,0 +1,15 @@
+syntax = "proto3";
+
+package cline;
+option java_package = "bot.cline.proto";
+option java_multiple_files = true;
+
+import "common.proto";
+
+// Service for account-related operations
+service AccountService {
+    // Handles the user clicking the login link in the UI.
+    // Generates a secure nonce for state validation, stores it in secrets,
+    // and opens the authentication URL in the external browser.
+    rpc accountLoginClicked(EmptyRequest) returns (String);
+}

proto/browser.proto

@@ -0,0 +1,50 @@
+syntax = "proto3";
+
+package cline;
+option java_package = "bot.cline.proto";
+option java_multiple_files = true;
+
+import "common.proto";
+
+service BrowserService {
+  rpc getBrowserConnectionInfo(EmptyRequest) returns (BrowserConnectionInfo);
+  rpc testBrowserConnection(StringRequest) returns (BrowserConnection);
+  rpc discoverBrowser(EmptyRequest) returns (BrowserConnection);
+  rpc getDetectedChromePath(EmptyRequest) returns (ChromePath);
+  rpc updateBrowserSettings(UpdateBrowserSettingsRequest) returns (Boolean);
+}
+
+message BrowserConnectionInfo {
+  bool is_connected = 1;
+  bool is_remote = 2;
+  optional string host = 3;
+}
+
+message BrowserConnection {
+  bool success = 1;
+  string message = 2;
+  optional string endpoint = 3;
+}
+
+message ChromePath {
+  string path = 1;
+  bool is_bundled = 2;
+}
+
+message Viewport {
+  int32 width = 1;
+  int32 height = 2;
+}
+
+message BrowserSettings {
+  Viewport viewport = 1;
+  optional string remote_browser_host = 2;
+  optional bool remote_browser_enabled = 3;
+}
+
+message UpdateBrowserSettingsRequest {
+  Metadata metadata = 1;
+  Viewport viewport = 2;
+  optional string remote_browser_host = 3;
+  optional bool remote_browser_enabled = 4;
+}

proto/build-proto.js

@@ -0,0 +1,147 @@
+#!/usr/bin/env node
+
+import * as fs from "fs/promises"
+import * as path from "path"
+import { execSync } from "child_process"
+import { globby } from "globby"
+import chalk from "chalk"
+
+import { createRequire } from "module"
+const require = createRequire(import.meta.url)
+const protoc = path.join(require.resolve("grpc-tools"), "../bin/protoc")
+const tsProtoPlugin = require.resolve("ts-proto/protoc-gen-ts_proto")
+
+// Get script directory and root directory
+const SCRIPT_DIR = path.dirname(new URL(import.meta.url).pathname)
+const ROOT_DIR = path.resolve(SCRIPT_DIR, "..")
+
+async function main() {
+	console.log(chalk.bold.blue("Starting Protocol Buffer code generation..."))
+
+	// Define output directories
+	const TS_OUT_DIR = path.join(ROOT_DIR, "src", "shared", "proto")
+
+	// Create output directory if it doesn't exist
+	await fs.mkdir(TS_OUT_DIR, { recursive: true })
+
+	// Clean up existing generated files
+	console.log(chalk.cyan("Cleaning up existing generated TypeScript files..."))
+	const existingFiles = await globby("**/*.ts", { cwd: TS_OUT_DIR })
+	for (const file of existingFiles) {
+		await fs.unlink(path.join(TS_OUT_DIR, file))
+	}
+
+	// Process all proto files
+	console.log(chalk.cyan("Processing proto files from"), SCRIPT_DIR)
+	const protoFiles = await globby("*.proto", { cwd: SCRIPT_DIR })
+
+	for (const protoFile of protoFiles) {
+		console.log(chalk.cyan(`Generating TypeScript code for ${protoFile}...`))
+
+		// Build the protoc command with proper path handling for cross-platform
+		const protocCommand = [
+			protoc,
+			`--plugin=protoc-gen-ts_proto="${tsProtoPlugin}"`,
+			`--ts_proto_out="${TS_OUT_DIR}"`,
+			"--ts_proto_opt=outputServices=generic-definitions,env=node,esModuleInterop=true,useDate=false,useOptionals=messages",
+			`--proto_path="${SCRIPT_DIR}"`,
+			`"${path.join(SCRIPT_DIR, protoFile)}"`,
+		].join(" ")
+
+		try {
+			const execOptions = {
+				stdio: "inherit",
+			}
+			execSync(protocCommand, execOptions)
+		} catch (error) {
+			console.error(chalk.red(`Error generating TypeScript for ${protoFile}:`), error)
+			process.exit(1)
+		}
+	}
+
+	console.log(chalk.green("Protocol Buffer code generation completed successfully."))
+	console.log(chalk.green(`TypeScript files generated in: ${TS_OUT_DIR}`))
+
+	// Generate method registration files
+	await generateMethodRegistrations()
+
+	// Make the script executable
+	try {
+		await fs.chmod(path.join(SCRIPT_DIR, "build-proto.js"), 0o755)
+	} catch (error) {
+		console.warn(chalk.yellow("Warning: Could not make script executable:"), error)
+	}
+}
+
+async function generateMethodRegistrations() {
+	console.log(chalk.cyan("Generating method registration files..."))
+
+	const serviceDirs = [
+		path.join(ROOT_DIR, "src", "core", "controller", "account"),
+		path.join(ROOT_DIR, "src", "core", "controller", "browser"),
+		path.join(ROOT_DIR, "src", "core", "controller", "checkpoints"),
+		path.join(ROOT_DIR, "src", "core", "controller", "file"),
+		path.join(ROOT_DIR, "src", "core", "controller", "mcp"),
+		path.join(ROOT_DIR, "src", "core", "controller", "task"),
+		// Add more service directories here as needed
+	]
+
+	for (const serviceDir of serviceDirs) {
+		try {
+			await fs.access(serviceDir)
+		} catch (error) {
+			console.log(chalk.gray(`Skipping ${serviceDir} - directory does not exist`))
+			continue
+		}
+
+		const serviceName = path.basename(serviceDir)
+		const registryFile = path.join(serviceDir, "methods.ts")
+
+		console.log(chalk.cyan(`Generating method registrations for ${serviceName}...`))
+
+		// Get all TypeScript files in the service directory
+		const files = await globby("*.ts", { cwd: serviceDir })
+
+		// Filter out index.ts and methods.ts
+		const implementationFiles = files.filter((file) => file !== "index.ts" && file !== "methods.ts")
+
+		// Create the output file with header
+		let content = `// AUTO-GENERATED FILE - DO NOT MODIFY DIRECTLY
+// Generated by proto/build-proto.js
+
+// Import all method implementations
+import { registerMethod } from "./index"\n`
+
+		// Add imports for all implementation files
+		for (const file of implementationFiles) {
+			const baseName = path.basename(file, ".ts")
+			content += `import { ${baseName} } from "./${baseName}"\n`
+		}
+
+		// Add registration function
+		content += `\n// Register all ${serviceName} service methods
+export function registerAllMethods(): void {
+\t// Register each method with the registry\n`
+
+		// Add registration statements
+		for (const file of implementationFiles) {
+			const baseName = path.basename(file, ".ts")
+			content += `\tregisterMethod("${baseName}", ${baseName})\n`
+		}
+
+		// Close the function
+		content += `}`
+
+		// Write the file
+		await fs.writeFile(registryFile, content)
+		console.log(chalk.green(`Generated ${registryFile}`))
+	}
+
+	console.log(chalk.green("Method registration files generated successfully."))
+}
+
+// Run the main function
+main().catch((error) => {
+	console.error(chalk.red("Error:"), error)
+	process.exit(1)
+})

proto/checkpoints.proto

@@ -0,0 +1,19 @@
+syntax = "proto3";
+
+package cline;
+option java_package = "bot.cline.proto";
+option java_multiple_files = true;
+
+import "common.proto";
+
+service CheckpointsService {
+  rpc checkpointDiff(Int64Request) returns (Empty);
+  rpc checkpointRestore(CheckpointRestoreRequest) returns (Empty);
+}
+
+message CheckpointRestoreRequest {
+  Metadata metadata = 1;
+  int64 number = 2;
+  string restore_type = 3;
+  optional int64 offset = 4;
+}

proto/common.proto

@@ -0,0 +1,51 @@
+syntax = "proto3";
+
+package cline;
+option java_package = "bot.cline.proto";
+option java_multiple_files = true;
+
+message Metadata {
+}
+
+message EmptyRequest {
+  Metadata metadata = 1;
+}
+
+message Empty {
+}
+
+message StringRequest {
+  Metadata metadata = 1;
+  string value = 2;
+}
+
+message String {
+  string value = 1;
+}
+
+message Int64Request {
+  Metadata metadata = 1;
+  int64 value = 2;
+}
+
+message Int64 {
+  int64 value = 1;
+}
+
+message BytesRequest {
+  Metadata metadata = 1;
+  bytes value = 2;
+}
+
+message Bytes {
+  bytes value = 1;
+}
+
+message BooleanRequest {
+  Metadata metadata = 1;
+  bool value = 2;
+}
+
+message Boolean {
+  bool value = 1;
+}

proto/file.proto

@@ -0,0 +1,16 @@
+syntax = "proto3";
+
+package cline;
+option java_package = "bot.cline.proto";
+option java_multiple_files = true;
+
+import "common.proto";
+
+// Service for file-related operations
+service FileService {
+  // Opens a file in the editor
+  rpc openFile(StringRequest) returns (Empty);
+  
+  // Opens an image in the system viewer
+  rpc openImage(StringRequest) returns (Empty);
+}

proto/mcp.proto

@@ -0,0 +1,76 @@
+syntax = "proto3";
+
+package cline;
+option java_package = "bot.cline.proto";
+option java_multiple_files = true;
+
+import "common.proto";
+
+service McpService {
+  rpc toggleMcpServer(ToggleMcpServerRequest) returns (McpServers);
+  rpc updateMcpTimeout(UpdateMcpTimeoutRequest) returns (McpServers);
+  rpc addRemoteMcpServer(AddRemoteMcpServerRequest) returns (McpServers);
+}
+
+message ToggleMcpServerRequest {
+  Metadata metadata = 1;
+  string server_name = 2;
+  bool disabled = 3;
+}
+
+message UpdateMcpTimeoutRequest {
+  Metadata metadata = 1;
+  string server_name = 2;
+  int32 timeout = 3;
+}
+
+message AddRemoteMcpServerRequest {
+  Metadata metadata = 1;
+  string server_name = 2;
+  string server_url = 3;
+}
+
+message McpTool {
+  string name = 1;
+  optional string description = 2;
+  optional string input_schema = 3;
+  optional bool auto_approve = 4;
+}
+
+message McpResource {
+  string uri = 1;
+  string name = 2;
+  optional string mime_type = 3;
+  optional string description = 4;
+}
+
+message McpResourceTemplate {
+  string uri_template = 1;
+  string name = 2;
+  optional string mime_type = 3;
+  optional string description = 4;
+}
+
+enum McpServerStatus {
+  // Protobuf enums (in proto3) must have a zero value defined, which serves as the default if the field isn't explicitly set. 
+  // To align with the required nature of the TypeScript type and avoid an unnecessary UNSPECIFIED state, we map one of the existing statuses to this zero value.
+  MCP_SERVER_STATUS_DISCONNECTED = 0; // default
+  MCP_SERVER_STATUS_CONNECTED = 1;
+  MCP_SERVER_STATUS_CONNECTING = 2;
+}
+
+message McpServer {
+  string name = 1;
+  string config = 2;
+  McpServerStatus status = 3;
+  optional string error = 4;
+  repeated McpTool tools = 5;
+  repeated McpResource resources = 6;
+  repeated McpResourceTemplate resource_templates = 7;
+  optional bool disabled = 8;
+  optional int32 timeout = 9;
+}
+
+message McpServers {
+  repeated McpServer mcp_servers = 1;
+}

proto/package.json

@@ -0,0 +1,3 @@
+{
+	"type": "module"
+}

proto/task.proto

@@ -0,0 +1,23 @@
+syntax = "proto3";
+
+package cline;
+option java_package = "bot.cline.proto";
+option java_multiple_files = true;
+
+import "common.proto";
+
+service TaskService {
+  // Cancels the currently running task
+  rpc cancelTask(EmptyRequest) returns (Empty);
+  // Clears the current task
+  rpc clearTask(EmptyRequest) returns (Empty);
+  // Creates a new task with the given text and optional images
+  rpc newTask(NewTaskRequest) returns (Empty);
+}
+
+// Request message for creating a new task
+message NewTaskRequest {
+  Metadata metadata = 1;
+  string text = 2;
+  repeated string images = 3;
+}

scripts/build-tests.js

@@ -0,0 +1,61 @@
+const { execSync } = require("child_process")
+const esbuild = require("esbuild")
+
+const watch = process.argv.includes("--watch")
+
+/**
+ * @type {import('esbuild').Plugin}
+ */
+const esbuildProblemMatcherPlugin = {
+	name: "esbuild-problem-matcher",
+
+	setup(build) {
+		build.onStart(() => {
+			console.log("[watch] build started")
+		})
+		build.onEnd((result) => {
+			result.errors.forEach(({ text, location }) => {
+				console.error(`✘ [ERROR] ${text}`)
+				console.error(`    ${location.file}:${location.line}:${location.column}:`)
+			})
+			console.log("[watch] build finished")
+		})
+	},
+}
+
+const srcConfig = {
+	bundle: true,
+	minify: false,
+	sourcemap: true,
+	sourcesContent: true,
+	logLevel: "silent",
+	entryPoints: ["src/packages/**/*.ts"],
+	outdir: "out/packages",
+	format: "cjs",
+	platform: "node",
+	define: {
+		"process.env.IS_DEV": "true",
+		"process.env.IS_TEST": "true",
+	},
+	external: ["vscode"],
+	plugins: [esbuildProblemMatcherPlugin],
+}
+
+async function main() {
+	const srcCtx = await esbuild.context(srcConfig)
+
+	if (watch) {
+		await srcCtx.watch()
+	} else {
+		await srcCtx.rebuild()
+
+		await srcCtx.dispose()
+	}
+}
+
+execSync("tsc -p ./tsconfig.test.json --outDir out", { encoding: "utf-8" })
+
+main().catch((e) => {
+	console.error(e)
+	process.exit(1)
+})

src/api/providers/__tests__/ollama.test.ts

@@ -0,0 +1,228 @@
+import { describe, it, beforeEach, afterEach, before } from "mocha"
+import "should"
+import sinon from "sinon"
+import { Anthropic } from "@anthropic-ai/sdk"
+import { OllamaHandler } from "../ollama"
+import { ApiHandlerOptions } from "@shared/api"
+import axios from "axios"
+
+describe("OllamaHandler", () => {
+	let ollamaAvailable = false
+
+	// Check if Ollama is running before running tests
+	before(async function () {
+		this.timeout(5000)
+		try {
+			await axios.get("http://localhost:11434/api/version", { timeout: 2000 })
+			ollamaAvailable = true
+		} catch (error) {
+			console.log("Ollama server not available, skipping tests")
+			ollamaAvailable = false
+		}
+	})
+	let handler: OllamaHandler
+	let options: ApiHandlerOptions
+	let clock: sinon.SinonFakeTimers
+
+	beforeEach(() => {
+		options = {
+			ollamaModelId: "llama2",
+			ollamaBaseUrl: "http://localhost:11434",
+		}
+		handler = new OllamaHandler(options)
+		// Use fake timers for testing timeouts
+		clock = sinon.useFakeTimers()
+	})
+
+	afterEach(() => {
+		clock.restore()
+		sinon.restore()
+	})
+
+	describe("createMessage", () => {
+		it("should handle successful responses", async function () {
+			if (!ollamaAvailable) {
+				this.skip()
+			}
+			this.timeout(5000)
+			// Mock the Ollama client's chat method
+			const chatStub = sinon.stub(handler["client"], "chat").resolves({
+				[Symbol.asyncIterator]: async function* () {
+					yield {
+						message: { content: "Hello, world!" },
+						eval_count: 10,
+						prompt_eval_count: 20,
+					}
+				},
+			} as any)
+
+			const systemPrompt = "You are a helpful assistant."
+			const messages: Anthropic.Messages.MessageParam[] = [{ role: "user", content: "Hello" }]
+
+			const result = []
+			const usageInfo = []
+
+			// Collect the results
+			for await (const chunk of handler.createMessage(systemPrompt, messages)) {
+				if (chunk.type === "text") {
+					result.push(chunk.text)
+				} else if (chunk.type === "usage") {
+					usageInfo.push({
+						inputTokens: chunk.inputTokens,
+						outputTokens: chunk.outputTokens,
+					})
+				}
+			}
+
+			// Verify the results
+			result.should.deepEqual(["Hello, world!"])
+			usageInfo.should.deepEqual([{ inputTokens: 20, outputTokens: 10 }])
+			chatStub.calledOnce.should.be.true()
+		})
+
+		it("should handle timeout errors", async function () {
+			if (!ollamaAvailable) {
+				this.skip()
+			}
+			this.timeout(10000)
+			// Restore real timers for this test
+			clock.restore()
+
+			// Create a handler with a very short timeout for testing
+			const testHandler = new OllamaHandler(options)
+
+			// Replace the createMessage method with one that has a shorter timeout
+			testHandler.createMessage = async function* (systemPrompt, messages) {
+				try {
+					// Create a promise that rejects after a short timeout
+					const timeoutPromise = new Promise<never>((_, reject) => {
+						setTimeout(() => reject(new Error("Ollama request timed out after 120 seconds")), 100)
+					})
+
+					// Create a promise that never resolves
+					const neverPromise = new Promise(() => {})
+
+					// Race them
+					await Promise.race([timeoutPromise, neverPromise])
+				} catch (error: any) {
+					// Enhance error reporting
+					console.error(`Ollama API error: ${error.message}`)
+					throw error
+				}
+			}
+
+			const systemPrompt = "You are a helpful assistant."
+			const messages: Anthropic.Messages.MessageParam[] = [{ role: "user", content: "Hello" }]
+
+			// Start the request and catch the error
+			let errorMessage = ""
+			try {
+				for await (const _ of testHandler.createMessage(systemPrompt, messages)) {
+					// This should not be reached
+				}
+			} catch (error: any) {
+				errorMessage = error.message
+			}
+
+			// Check the result
+			errorMessage.should.equal("Ollama request timed out after 120 seconds")
+
+			// Restore the fake timers for other tests
+			clock = sinon.useFakeTimers()
+		})
+
+		it("should retry on errors when using the withRetry decorator", async function () {
+			if (!ollamaAvailable) {
+				this.skip()
+			}
+			this.timeout(10000)
+			// Restore real timers for this test
+			clock.restore()
+
+			// Mock the Ollama client's chat method to fail on first call and succeed on second
+			const chatStub = sinon.stub(handler["client"], "chat")
+
+			// First call throws an error
+			chatStub.onFirstCall().rejects(new Error("API Error"))
+
+			// Second call succeeds
+			chatStub.onSecondCall().resolves({
+				[Symbol.asyncIterator]: async function* () {
+					yield {
+						message: { content: "Success after retry" },
+					}
+				},
+			} as any)
+
+			const systemPrompt = "You are a helpful assistant."
+			const messages: Anthropic.Messages.MessageParam[] = [{ role: "user", content: "Hello" }]
+
+			const result = []
+
+			// Add a small delay to ensure the retry mechanism has time to work
+			await new Promise((resolve) => setTimeout(resolve, 100))
+
+			// Collect the results
+			for await (const chunk of handler.createMessage(systemPrompt, messages)) {
+				if (chunk.type === "text") {
+					result.push(chunk.text)
+				}
+			}
+
+			// Verify the results
+			result.should.deepEqual(["Success after retry"])
+			chatStub.calledTwice.should.be.true()
+
+			// Restore the fake timers for other tests
+			clock = sinon.useFakeTimers()
+		})
+
+		it("should handle stream processing errors", async function () {
+			if (!ollamaAvailable) {
+				this.skip()
+			}
+			this.timeout(10000)
+			// Restore real timers for this test
+			clock.restore()
+
+			// Create a handler with a custom implementation for testing
+			const testHandler = new OllamaHandler(options)
+
+			// Replace the createMessage method with one that simulates a stream error
+			testHandler.createMessage = async function* (systemPrompt, messages) {
+				// First yield a successful chunk
+				yield {
+					type: "text",
+					text: "Partial response",
+				}
+
+				// Then throw an error in the stream
+				throw new Error("Ollama stream processing error: Stream error")
+			}
+
+			const systemPrompt = "You are a helpful assistant."
+			const messages: Anthropic.Messages.MessageParam[] = [{ role: "user", content: "Hello" }]
+
+			const result = []
+
+			// Collect the results and catch the error
+			let errorMessage = ""
+			try {
+				for await (const chunk of testHandler.createMessage(systemPrompt, messages)) {
+					if (chunk.type === "text") {
+						result.push(chunk.text)
+					}
+				}
+			} catch (error: any) {
+				errorMessage = error.message
+			}
+
+			// Verify the results
+			errorMessage.should.equal("Ollama stream processing error: Stream error")
+			result.should.deepEqual(["Partial response"])
+
+			// Restore the fake timers for other tests
+			clock = sinon.useFakeTimers()
+		})
+	})
+})

src/api/providers/anthropic.ts

@@ -1,7 +1,7 @@
 import { Anthropic } from "@anthropic-ai/sdk"
 import { Stream as AnthropicStream } from "@anthropic-ai/sdk/streaming"
 import { withRetry } from "../retry"
-import { anthropicDefaultModelId, AnthropicModelId, anthropicModels, ApiHandlerOptions, ModelInfo } from "../../shared/api"
+import { anthropicDefaultModelId, AnthropicModelId, anthropicModels, ApiHandlerOptions, ModelInfo } from "@shared/api"
 import { ApiHandler } from "../index"
 import { ApiStream } from "../transform/stream"
 
@@ -114,7 +114,7 @@ export class AnthropicHandler implements ApiHandler {
 				break
 			}
 			default: {
-				stream = (await this.client.messages.create({
+				stream = await this.client.messages.create({
 					model: modelId,
 					max_tokens: model.info.maxTokens || 8192,
 					temperature: 0,
@@ -123,7 +123,7 @@ export class AnthropicHandler implements ApiHandler {
 					// tools,
 					// tool_choice: { type: "auto" },
 					stream: true,
-				})) as any
+				})
 				break
 			}
 		}

src/api/providers/asksage.ts

@@ -7,7 +7,7 @@ import {
 	askSageModels,
 	askSageDefaultModelId,
 	askSageDefaultURL,
-} from "../../shared/api"
+} from "@shared/api"
 import { ApiStream } from "../transform/stream"
 
 type AskSageRequest = {

src/api/providers/bedrock.ts

@@ -3,7 +3,7 @@ import { Anthropic } from "@anthropic-ai/sdk"
 import { withRetry } from "../retry"
 import { ApiHandler } from "../"
 import { convertToR1Format } from "../transform/r1-format"
-import { ApiHandlerOptions, bedrockDefaultModelId, BedrockModelId, bedrockModels, ModelInfo } from "../../shared/api"
+import { ApiHandlerOptions, bedrockDefaultModelId, BedrockModelId, bedrockModels, ModelInfo } from "@shared/api"
 import { calculateApiCostOpenAI } from "../../utils/cost"
 import { ApiStream } from "../transform/stream"
 import { fromNodeProviderChain } from "@aws-sdk/credential-providers"

src/api/providers/cline.ts

@@ -1,7 +1,7 @@
 import { Anthropic } from "@anthropic-ai/sdk"
 import OpenAI from "openai"
 import { ApiHandler } from "../"
-import { ApiHandlerOptions, ModelInfo, openRouterDefaultModelId, openRouterDefaultModelInfo } from "../../shared/api"
+import { ApiHandlerOptions, ModelInfo, openRouterDefaultModelId, openRouterDefaultModelInfo } from "@shared/api"
 import { createOpenRouterStream } from "../transform/openrouter-stream"
 import { ApiStream, ApiStreamUsageChunk } from "../transform/stream"
 import axios from "axios"
@@ -74,6 +74,8 @@ export class ClineHandler implements ApiHandler {
 			if (!didOutputUsage && chunk.usage) {
 				yield {
 					type: "usage",
+					cacheWriteTokens: 0,
+					cacheReadTokens: chunk.usage.prompt_tokens_details?.cached_tokens || 0,
 					inputTokens: chunk.usage.prompt_tokens || 0,
 					outputTokens: chunk.usage.completion_tokens || 0,
 					// @ts-ignore-next-line
@@ -105,6 +107,9 @@ export class ClineHandler implements ApiHandler {
 				const generation = response.data
 				return {
 					type: "usage",
+					// at this time there's no support for gatting cached_tokens from generation endpoint
+					cacheWriteTokens: 0,
+					cacheReadTokens: 0,
 					inputTokens: generation?.native_tokens_prompt || 0,
 					outputTokens: generation?.native_tokens_completion || 0,
 					totalCost: generation?.total_cost || 0,

src/api/providers/deepseek.ts

@@ -2,7 +2,7 @@ import { Anthropic } from "@anthropic-ai/sdk"
 import OpenAI from "openai"
 import { withRetry } from "../retry"
 import { ApiHandler } from "../"
-import { ApiHandlerOptions, DeepSeekModelId, ModelInfo, deepSeekDefaultModelId, deepSeekModels } from "../../shared/api"
+import { ApiHandlerOptions, DeepSeekModelId, ModelInfo, deepSeekDefaultModelId, deepSeekModels } from "@shared/api"
 import { calculateApiCostOpenAI } from "../../utils/cost"
 import { convertToOpenAiMessages } from "../transform/openai-format"
 import { ApiStream } from "../transform/stream"

src/api/providers/doubao.ts

@@ -1,5 +1,5 @@
 import { ApiHandler } from ".."
-import { ApiHandlerOptions, doubaoDefaultModelId, DoubaoModelId, doubaoModels, ModelInfo } from "../../shared/api"
+import { ApiHandlerOptions, doubaoDefaultModelId, DoubaoModelId, doubaoModels, ModelInfo } from "@shared/api"
 import { Anthropic } from "@anthropic-ai/sdk"
 import OpenAI from "openai"
 import { convertToOpenAiMessages } from "../transform/openai-format"

src/api/providers/gemini.ts

@@ -1,49 +1,154 @@
-import { Anthropic } from "@anthropic-ai/sdk"
-import { GoogleGenerativeAI } from "@google/generative-ai"
+import type { Anthropic } from "@anthropic-ai/sdk"
+// Restore GenerateContentConfig import and add GenerateContentResponseUsageMetadata
+import { GoogleGenAI, type Content, type GenerateContentConfig, type GenerateContentResponseUsageMetadata } from "@google/genai"
 import { withRetry } from "../retry"
 import { ApiHandler } from "../"
-import { ApiHandlerOptions, geminiDefaultModelId, GeminiModelId, geminiModels, ModelInfo } from "../../shared/api"
+import { ApiHandlerOptions, geminiDefaultModelId, GeminiModelId, geminiModels, ModelInfo } from "@shared/api"
 import { convertAnthropicMessageToGemini } from "../transform/gemini-format"
 import { ApiStream } from "../transform/stream"
 
+// Define a default TTL for the cache (e.g., 15 minutes in seconds)
+const DEFAULT_CACHE_TTL_SECONDS = 900
+
 export class GeminiHandler implements ApiHandler {
 	private options: ApiHandlerOptions
-	private client: GoogleGenerativeAI
+	private client: GoogleGenAI // Updated client type
+
+	// Internal state for caching
+	private cacheName: string | null = null
+	private cacheExpireTime: number | null = null
+	private isFirstApiCall = true
 
 	constructor(options: ApiHandlerOptions) {
 		if (!options.geminiApiKey) {
 			throw new Error("API key is required for Google Gemini")
 		}
 		this.options = options
-		this.client = new GoogleGenerativeAI(options.geminiApiKey)
+		// Updated client initialization
+		this.client = new GoogleGenAI({ apiKey: options.geminiApiKey })
 	}
 
 	@withRetry()
 	async *createMessage(systemPrompt: string, messages: Anthropic.Messages.MessageParam[]): ApiStream {
-		const model = this.client.getGenerativeModel({
-			model: this.getModel().id,
-			systemInstruction: systemPrompt,
-		})
-		const result = await model.generateContentStream({
-			contents: messages.map(convertAnthropicMessageToGemini),
-			generationConfig: {
-				// maxOutputTokens: this.getModel().info.maxTokens,
-				temperature: 0,
-			},
-		})
+		const { id: modelId, info: modelInfo } = this.getModel()
 
-		for await (const chunk of result.stream) {
-			yield {
-				type: "text",
-				text: chunk.text(),
+		// --- Cache Handling Logic ---
+		const isCacheValid = this.cacheName && this.cacheExpireTime && Date.now() < this.cacheExpireTime
+		let useCache = !this.isFirstApiCall && isCacheValid
+
+		if (this.isFirstApiCall && !isCacheValid && systemPrompt) {
+			// It's the first call, no valid cache exists, and we have a system prompt. Attempt cache creation.
+			this.isFirstApiCall = false
+
+			// Minimum token check heuristic (simple length check for now, could be improved)
+			// Gemini requires minimum 4096 tokens. A simple length check isn't accurate but avoids complex token counting here.
+			// Let's assume a generous average of 4 chars/token. 4096 tokens * 4 chars/token = 16384 chars.
+			const MIN_SYSTEM_PROMPT_LENGTH_FOR_CACHE = 16384
+			if (systemPrompt.length >= MIN_SYSTEM_PROMPT_LENGTH_FOR_CACHE) {
+				// Start cache creation asynchronously, don't block the main request
+				this.createCacheInBackground(modelId, systemPrompt)
+			}
+			// Proceed with the first request *without* using the cache, as it's being created.
+			useCache = false
+		} else if (!isCacheValid && this.cacheName) {
+			// Cache exists but has expired
+			this.cacheName = null
+			this.cacheExpireTime = null
+			useCache = false
+		}
+		// --- End Cache Handling Logic ---
+
+		// Re-implement thinking budget logic based on new SDK structure
+		const thinkingBudget = this.options.thinkingBudgetTokens ?? 0
+		const maxBudget = modelInfo.thinkingConfig?.maxBudget ?? 0
+
+		// port add baseUrl configuration for gemini api requests (#2843)
+		const httpOptions = this.options.geminiBaseUrl ? { baseUrl: this.options.geminiBaseUrl } : undefined
+
+		// Base generation config - Conditionally include systemInstruction based on cache usage
+		const generationConfig: GenerateContentConfig = {
+			httpOptions,
+			temperature: 0, // Default temperature
+			// Only include systemInstruction if NOT using the cache
+			...(useCache ? {} : { systemInstruction: systemPrompt }),
+		}
+
+		// Convert messages to the format expected by @google/genai
+		// Note: convertAnthropicMessageToGemini might need adjustments
+		const contents: Content[] = messages.map(convertAnthropicMessageToGemini)
+
+		// Construct the main request config - Type as GenerateContentConfig
+		const requestConfig: GenerateContentConfig = {
+			...generationConfig,
+		}
+
+		// Add thinking config if the model supports it
+		if (modelInfo.thinkingConfig?.outputPrice !== undefined && maxBudget > 0) {
+			requestConfig.thinkingConfig = {
+				thinkingBudget: thinkingBudget,
 			}
 		}
 
-		const response = await result.response
-		yield {
-			type: "usage",
-			inputTokens: response.usageMetadata?.promptTokenCount ?? 0,
-			outputTokens: response.usageMetadata?.candidatesTokenCount ?? 0,
+		// Generate content using the new SDK structure via client.models
+		const result = await this.client.models.generateContentStream({
+			model: modelId, // Pass model ID directly
+			contents,
+			// Add cachedContent if using the cache
+			config: {
+				...requestConfig,
+				...(useCache ? { cachedContent: this.cacheName! } : {}),
+			},
+		})
+
+		// Declare variable to hold the last usage metadata found
+		let lastUsageMetadata: GenerateContentResponseUsageMetadata | undefined
+
+		// Iterate directly over the stream
+		for await (const chunk of result) {
+			if (chunk.text) {
+				yield {
+					type: "text",
+					text: chunk.text,
+				}
+			}
+			if (chunk.usageMetadata) {
+				lastUsageMetadata = chunk.usageMetadata
+			}
+		}
+
+		if (lastUsageMetadata) {
+			yield {
+				type: "usage",
+				inputTokens: lastUsageMetadata.promptTokenCount ?? 0,
+				outputTokens: lastUsageMetadata.candidatesTokenCount ?? 0,
+				cacheWriteTokens: lastUsageMetadata.cachedContentTokenCount ?? 0,
+				cacheReadTokens: useCache ? (lastUsageMetadata.promptTokenCount ?? 0) : 0, // If cache used, prompt tokens are read from cache
+			}
+		}
+	}
+
+	private async createCacheInBackground(modelId: string, systemInstruction: string): Promise<void> {
+		try {
+			const cache = await this.client.caches.create({
+				model: modelId,
+				config: {
+					systemInstruction: systemInstruction,
+					ttl: `${DEFAULT_CACHE_TTL_SECONDS}s`,
+				},
+			})
+
+			if (cache?.name) {
+				this.cacheName = cache.name
+				// Calculate expiry timestamp using the default TTL, as the response object might not contain it directly.
+				this.cacheExpireTime = Date.now() + DEFAULT_CACHE_TTL_SECONDS * 1000
+			} else {
+				console.warn("Gemini cache creation call succeeded but returned no cache name.")
+			}
+		} catch (error) {
+			console.error("Failed to create Gemini cache in background:", error)
+			// Reset state if creation failed definitively
+			this.cacheName = null
+			this.cacheExpireTime = null
 		}
 	}
 

src/api/providers/litellm.ts

@@ -1,6 +1,6 @@
 import { Anthropic } from "@anthropic-ai/sdk"
 import OpenAI from "openai"
-import { ApiHandlerOptions, liteLlmDefaultModelId, liteLlmModelInfoSaneDefaults } from "../../shared/api"
+import { ApiHandlerOptions, liteLlmDefaultModelId, liteLlmModelInfoSaneDefaults } from "@shared/api"
 import { ApiHandler } from ".."
 import { ApiStream } from "../transform/stream"
 import { convertToOpenAiMessages } from "../transform/openai-format"

src/api/providers/lmstudio.ts

@@ -1,7 +1,7 @@
 import { Anthropic } from "@anthropic-ai/sdk"
 import OpenAI from "openai"
 import { ApiHandler } from "../"
-import { ApiHandlerOptions, ModelInfo, openAiModelInfoSaneDefaults } from "../../shared/api"
+import { ApiHandlerOptions, ModelInfo, openAiModelInfoSaneDefaults } from "@shared/api"
 import { convertToOpenAiMessages } from "../transform/openai-format"
 import { ApiStream } from "../transform/stream"
 

src/api/providers/mistral.ts

@@ -2,16 +2,7 @@ import { Anthropic } from "@anthropic-ai/sdk"
 import { Mistral } from "@mistralai/mistralai"
 import { withRetry } from "../retry"
 import { ApiHandler } from "../"
-import {
-	ApiHandlerOptions,
-	mistralDefaultModelId,
-	MistralModelId,
-	mistralModels,
-	ModelInfo,
-	openAiNativeDefaultModelId,
-	OpenAiNativeModelId,
-	openAiNativeModels,
-} from "../../shared/api"
+import { ApiHandlerOptions, mistralDefaultModelId, MistralModelId, mistralModels, ModelInfo } from "@shared/api"
 import { convertToMistralMessages } from "../transform/mistral-format"
 import { ApiStream } from "../transform/stream"
 

src/api/providers/ollama.ts

@@ -4,6 +4,7 @@ import { ApiHandler } from "../"
 import { ApiHandlerOptions, ModelInfo, openAiModelInfoSaneDefaults } from "../../shared/api"
 import { convertToOllamaMessages } from "../transform/ollama-format"
 import { ApiStream } from "../transform/stream"
+import { withRetry } from "../retry"
 
 export class OllamaHandler implements ApiHandler {
 	private options: ApiHandlerOptions
@@ -14,24 +15,65 @@ export class OllamaHandler implements ApiHandler {
 		this.client = new Ollama({ host: this.options.ollamaBaseUrl || "http://localhost:11434" })
 	}
 
+	@withRetry({ retryAllErrors: true })
 	async *createMessage(systemPrompt: string, messages: Anthropic.Messages.MessageParam[]): ApiStream {
 		const ollamaMessages: Message[] = [{ role: "system", content: systemPrompt }, ...convertToOllamaMessages(messages)]
 
-		const stream = await this.client.chat({
-			model: this.getModel().id,
-			messages: ollamaMessages,
-			stream: true,
-			options: {
-				num_ctx: Number(this.options.ollamaApiOptionsCtxNum) || 32768,
-			},
-		})
-		for await (const chunk of stream) {
-			if (typeof chunk.message.content === "string") {
-				yield {
-					type: "text",
-					text: chunk.message.content,
+		try {
+			// Create a promise that rejects after timeout
+			const timeoutMs = this.options.requestTimeoutMs || 30000
+			const timeoutPromise = new Promise<never>((_, reject) => {
+				setTimeout(() => reject(new Error(`Ollama request timed out after ${timeoutMs / 1000} seconds`)), timeoutMs)
+			})
+
+			// Create the actual API request promise
+			const apiPromise = this.client.chat({
+				model: this.getModel().id,
+				messages: ollamaMessages,
+				stream: true,
+				options: {
+					num_ctx: Number(this.options.ollamaApiOptionsCtxNum) || 32768,
+				},
+			})
+
+			// Race the API request against the timeout
+			const stream = (await Promise.race([apiPromise, timeoutPromise])) as Awaited<typeof apiPromise>
+
+			try {
+				for await (const chunk of stream) {
+					if (typeof chunk.message.content === "string") {
+						yield {
+							type: "text",
+							text: chunk.message.content,
+						}
+					}
+
+					// Handle token usage if available
+					if (chunk.eval_count !== undefined || chunk.prompt_eval_count !== undefined) {
+						yield {
+							type: "usage",
+							inputTokens: chunk.prompt_eval_count || 0,
+							outputTokens: chunk.eval_count || 0,
+						}
+					}
 				}
+			} catch (streamError: any) {
+				console.error("Error processing Ollama stream:", streamError)
+				throw new Error(`Ollama stream processing error: ${streamError.message || "Unknown error"}`)
 			}
+		} catch (error: any) {
+			// Check if it's a timeout error
+			if (error.message && error.message.includes("timed out")) {
+				const timeoutMs = this.options.requestTimeoutMs || 30000
+				throw new Error(`Ollama request timed out after ${timeoutMs / 1000} seconds`)
+			}
+
+			// Enhance error reporting
+			const statusCode = error.status || error.statusCode
+			const errorMessage = error.message || "Unknown error"
+
+			console.error(`Ollama API error (${statusCode || "unknown"}): ${errorMessage}`)
+			throw error
 		}
 	}
 

src/api/providers/openai-native.ts

@@ -2,17 +2,11 @@ import { Anthropic } from "@anthropic-ai/sdk"
 import OpenAI from "openai"
 import { withRetry } from "../retry"
 import { ApiHandler } from "../"
-import {
-	ApiHandlerOptions,
-	ModelInfo,
-	openAiNativeDefaultModelId,
-	OpenAiNativeModelId,
-	openAiNativeModels,
-} from "../../shared/api"
+import { ApiHandlerOptions, ModelInfo, openAiNativeDefaultModelId, OpenAiNativeModelId, openAiNativeModels } from "@shared/api"
 import { convertToOpenAiMessages } from "../transform/openai-format"
 import { calculateApiCostOpenAI } from "../../utils/cost"
 import { ApiStream } from "../transform/stream"
-import { ChatCompletionReasoningEffort } from "openai/resources/chat/completions.mjs"
+import type { ChatCompletionReasoningEffort } from "openai/resources/chat/completions"
 
 export class OpenAiNativeHandler implements ApiHandler {
 	private options: ApiHandlerOptions
@@ -50,7 +44,7 @@ export class OpenAiNativeHandler implements ApiHandler {
 			case "o1":
 			case "o1-preview":
 			case "o1-mini": {
-				// o1 doesnt support streaming, non-1 temp, or system prompt
+				// o1 doesn't support streaming, non-1 temp, or system prompt
 				const response = await this.client.chat.completions.create({
 					model: model.id,
 					messages: [{ role: "user", content: systemPrompt }, ...convertToOpenAiMessages(messages)],
@@ -64,6 +58,8 @@ export class OpenAiNativeHandler implements ApiHandler {
 
 				break
 			}
+			case "o4-mini":
+			case "o3":
 			case "o3-mini": {
 				const stream = await this.client.chat.completions.create({
 					model: model.id,

src/api/providers/openai.ts

@@ -1,12 +1,12 @@
 import { Anthropic } from "@anthropic-ai/sdk"
 import OpenAI, { AzureOpenAI } from "openai"
 import { withRetry } from "../retry"
-import { ApiHandlerOptions, azureOpenAiDefaultApiVersion, ModelInfo, openAiModelInfoSaneDefaults } from "../../shared/api"
+import { ApiHandlerOptions, azureOpenAiDefaultApiVersion, ModelInfo, openAiModelInfoSaneDefaults } from "@shared/api"
 import { ApiHandler } from "../index"
 import { convertToOpenAiMessages } from "../transform/openai-format"
 import { ApiStream } from "../transform/stream"
 import { convertToR1Format } from "../transform/r1-format"
-import { ChatCompletionReasoningEffort } from "openai/resources/chat/completions.mjs"
+import type { ChatCompletionReasoningEffort } from "openai/resources/chat/completions"
 
 export class OpenAiHandler implements ApiHandler {
 	private options: ApiHandlerOptions
@@ -16,16 +16,22 @@ export class OpenAiHandler implements ApiHandler {
 		this.options = options
 		// Azure API shape slightly differs from the core API shape: https://github.com/openai/openai-node?tab=readme-ov-file#microsoft-azure-openai
 		// Use azureApiVersion to determine if this is an Azure endpoint, since the URL may not always contain 'azure.com'
-		if (this.options.azureApiVersion || this.options.openAiBaseUrl?.toLowerCase().includes("azure.com")) {
+		if (
+			this.options.azureApiVersion ||
+			(this.options.openAiBaseUrl?.toLowerCase().includes("azure.com") &&
+				!this.options.openAiModelId?.toLowerCase().includes("deepseek"))
+		) {
 			this.client = new AzureOpenAI({
 				baseURL: this.options.openAiBaseUrl,
 				apiKey: this.options.openAiApiKey,
 				apiVersion: this.options.azureApiVersion || azureOpenAiDefaultApiVersion,
+				defaultHeaders: this.options.openAiHeaders,
 			})
 		} else {
 			this.client = new OpenAI({
 				baseURL: this.options.openAiBaseUrl,
 				apiKey: this.options.openAiApiKey,
+				defaultHeaders: this.options.openAiHeaders,
 			})
 		}
 	}
@@ -35,7 +41,7 @@ export class OpenAiHandler implements ApiHandler {
 		const modelId = this.options.openAiModelId ?? ""
 		const isDeepseekReasoner = modelId.includes("deepseek-reasoner")
 		const isR1FormatRequired = this.options.openAiModelInfo?.isR1FormatRequired ?? false
-		const isO3Mini = modelId.includes("o3-mini")
+		const isReasoningModelFamily = modelId.includes("o1") || modelId.includes("o3") || modelId.includes("o4")
 
 		let openAiMessages: OpenAI.Chat.ChatCompletionMessageParam[] = [
 			{ role: "system", content: systemPrompt },
@@ -55,7 +61,7 @@ export class OpenAiHandler implements ApiHandler {
 			openAiMessages = convertToR1Format([{ role: "user", content: systemPrompt }, ...messages])
 		}
 
-		if (isO3Mini) {
+		if (isReasoningModelFamily) {
 			openAiMessages = [{ role: "developer", content: systemPrompt }, ...convertToOpenAiMessages(messages)]
 			temperature = undefined // does not support temperature
 			reasoningEffort = (this.options.o3MiniReasoningEffort as ChatCompletionReasoningEffort) || "medium"

src/api/providers/openrouter.ts

@@ -3,7 +3,7 @@ import axios from "axios"
 import { setTimeout as setTimeoutPromise } from "node:timers/promises"
 import OpenAI from "openai"
 import { ApiHandler } from "../"
-import { ApiHandlerOptions, ModelInfo, openRouterDefaultModelId, openRouterDefaultModelInfo } from "../../shared/api"
+import { ApiHandlerOptions, ModelInfo, openRouterDefaultModelId, openRouterDefaultModelInfo } from "@shared/api"
 import { withRetry } from "../retry"
 import { createOpenRouterStream } from "../transform/openrouter-stream"
 import { ApiStream, ApiStreamUsageChunk } from "../transform/stream"
@@ -76,6 +76,8 @@ export class OpenRouterHandler implements ApiHandler {
 			if (!didOutputUsage && chunk.usage) {
 				yield {
 					type: "usage",
+					cacheWriteTokens: 0,
+					cacheReadTokens: chunk.usage.prompt_tokens_details?.cached_tokens || 0,
 					inputTokens: chunk.usage.prompt_tokens || 0,
 					outputTokens: chunk.usage.completion_tokens || 0,
 					// @ts-ignore-next-line
@@ -103,8 +105,9 @@ export class OpenRouterHandler implements ApiHandler {
 				// console.log("OpenRouter generation details:", generation)
 				return {
 					type: "usage",
-					// cacheWriteTokens: 0,
-					// cacheReadTokens: 0,
+					// at this time there's no support for gatting cached_tokens from generation endpoint
+					cacheWriteTokens: 0,
+					cacheReadTokens: 0,
 					// openrouter generation endpoint fails often
 					inputTokens: generation?.native_tokens_prompt || 0,
 					outputTokens: generation?.native_tokens_completion || 0,

src/api/providers/qwen.ts

@@ -10,7 +10,7 @@ import {
 	internationalQwenDefaultModelId,
 	MainlandQwenModelId,
 	InternationalQwenModelId,
-} from "../../shared/api"
+} from "@shared/api"
 import { convertToOpenAiMessages } from "../transform/openai-format"
 import { ApiStream } from "../transform/stream"
 import { convertToR1Format } from "../transform/r1-format"

src/api/providers/requesty.ts

@@ -1,10 +1,11 @@
 import { Anthropic } from "@anthropic-ai/sdk"
 import OpenAI from "openai"
-import { ApiHandlerOptions, ModelInfo, openAiModelInfoSaneDefaults } from "../../shared/api"
+import { ApiHandlerOptions, ModelInfo, requestyDefaultModelId, requestyDefaultModelInfo } from "@shared/api"
 import { ApiHandler } from "../index"
 import { withRetry } from "../retry"
-import { convertToOpenAiMessages } from "../transform/openai-format"
-import { ApiStream } from "../transform/stream"
+import { convertToOpenAiMessages } from "@api/transform/openai-format"
+import { calculateApiCostOpenAI } from "@utils/cost"
+import { ApiStream } from "@api/transform/stream"
 
 export class RequestyHandler implements ApiHandler {
 	private options: ApiHandlerOptions
@@ -24,21 +25,34 @@ export class RequestyHandler implements ApiHandler {
 
 	@withRetry()
 	async *createMessage(systemPrompt: string, messages: Anthropic.Messages.MessageParam[]): ApiStream {
-		const modelId = this.options.requestyModelId ?? ""
+		const model = this.getModel()
 
 		const openAiMessages: OpenAI.Chat.ChatCompletionMessageParam[] = [
 			{ role: "system", content: systemPrompt },
 			...convertToOpenAiMessages(messages),
 		]
 
+		const reasoningEffort = this.options.o3MiniReasoningEffort || "medium"
+		const reasoning = { reasoning_effort: reasoningEffort }
+		const reasoningArgs = model.id === "openai/o3-mini" ? reasoning : {}
+
+		const thinkingBudget = this.options.thinkingBudgetTokens || 0
+		const thinking =
+			thinkingBudget > 0
+				? { thinking: { type: "enabled", budget_tokens: thinkingBudget } }
+				: { thinking: { type: "disabled" } }
+		const thinkingArgs = model.id.includes("claude-3-7-sonnet") ? thinking : {}
+
 		// @ts-ignore-next-line
 		const stream = await this.client.chat.completions.create({
-			model: modelId,
+			model: model.id,
+			max_tokens: model.info.maxTokens || undefined,
 			messages: openAiMessages,
 			temperature: 0,
 			stream: true,
 			stream_options: { include_usage: true },
-			...(modelId === "openai/o3-mini" ? { reasoning_effort: this.options.o3MiniReasoningEffort || "medium" } : {}),
+			...reasoningArgs,
+			...thinkingArgs,
 		})
 
 		for await (const chunk of stream) {
@@ -73,7 +87,7 @@ export class RequestyHandler implements ApiHandler {
 				const outputTokens = usage.completion_tokens || 0
 				const cacheWriteTokens = usage.prompt_tokens_details?.caching_tokens || undefined
 				const cacheReadTokens = usage.prompt_tokens_details?.cached_tokens || undefined
-				const totalCost = 0 // TODO: Replace with calculateApiCostOpenAI(model.info, inputTokens, outputTokens, cacheWriteTokens, cacheReadTokens)
+				const totalCost = calculateApiCostOpenAI(model.info, inputTokens, outputTokens, cacheWriteTokens, cacheReadTokens)
 
 				yield {
 					type: "usage",
@@ -88,9 +102,11 @@ export class RequestyHandler implements ApiHandler {
 	}
 
 	getModel(): { id: string; info: ModelInfo } {
-		return {
-			id: this.options.requestyModelId ?? "",
-			info: openAiModelInfoSaneDefaults,
+		const modelId = this.options.requestyModelId
+		const modelInfo = this.options.requestyModelInfo
+		if (modelId && modelInfo) {
+			return { id: modelId, info: modelInfo }
 		}
+		return { id: requestyDefaultModelId, info: requestyDefaultModelInfo }
 	}
 }

src/api/providers/sambanova.ts

@@ -1,11 +1,11 @@
 import { Anthropic } from "@anthropic-ai/sdk"
 import OpenAI from "openai"
 import { withRetry } from "../retry"
-import { ApiHandlerOptions, ModelInfo, SambanovaModelId, sambanovaDefaultModelId, sambanovaModels } from "../../shared/api"
+import { ApiHandlerOptions, ModelInfo, SambanovaModelId, sambanovaDefaultModelId, sambanovaModels } from "@shared/api"
 import { ApiHandler } from "../index"
-import { convertToOpenAiMessages } from "../transform/openai-format"
-import { ApiStream } from "../transform/stream"
-import { convertToR1Format } from "../transform/r1-format"
+import { convertToOpenAiMessages } from "@/api/transform/openai-format"
+import { ApiStream } from "@api/transform/stream"
+import { convertToR1Format } from "@api/transform/r1-format"
 
 export class SambanovaHandler implements ApiHandler {
 	private options: ApiHandlerOptions

src/api/providers/together.ts

@@ -1,11 +1,11 @@
 import { Anthropic } from "@anthropic-ai/sdk"
 import OpenAI from "openai"
 import { withRetry } from "../retry"
-import { ApiHandlerOptions, ModelInfo, openAiModelInfoSaneDefaults } from "../../shared/api"
+import { ApiHandlerOptions, ModelInfo, openAiModelInfoSaneDefaults } from "@shared/api"
 import { ApiHandler } from "../index"
-import { convertToOpenAiMessages } from "../transform/openai-format"
-import { ApiStream } from "../transform/stream"
-import { convertToR1Format } from "../transform/r1-format"
+import { convertToOpenAiMessages } from "@api/transform/openai-format"
+import { ApiStream } from "@api/transform/stream"
+import { convertToR1Format } from "@api/transform/r1-format"
 
 export class TogetherHandler implements ApiHandler {
 	private options: ApiHandlerOptions

src/api/providers/types.ts

@@ -1,6 +1,13 @@
 // For the following openrouter error type sources, see the docs here:
 // https://openrouter.ai/docs/api-reference/errors
 
+export interface LanguageModelChatSelector {
+	vendor?: string
+	family?: string
+	version?: string
+	id?: string
+}
+
 export type OpenRouterErrorResponse = {
 	error: {
 		message: string

src/api/providers/vertex.ts

@@ -2,9 +2,10 @@ import { Anthropic } from "@anthropic-ai/sdk"
 import { AnthropicVertex } from "@anthropic-ai/vertex-sdk"
 import { withRetry } from "../retry"
 import { ApiHandler } from "../"
-import { ApiHandlerOptions, ModelInfo, vertexDefaultModelId, VertexModelId, vertexModels } from "../../shared/api"
-import { ApiStream } from "../transform/stream"
+import { ApiHandlerOptions, ModelInfo, vertexDefaultModelId, VertexModelId, vertexModels } from "@shared/api"
+import { ApiStream } from "@api/transform/stream"
 import { VertexAI } from "@google-cloud/vertexai"
+import { calculateApiCostOpenAI } from "@utils/cost"
 
 // https://docs.anthropic.com/en/api/claude-on-vertex-ai
 export class VertexHandler implements ApiHandler {
@@ -266,6 +267,17 @@ export class VertexHandler implements ApiHandler {
 					}
 				}
 			}
+			// Handle token usage metadata
+			const { usageMetadata } = await streamingResult.response
+			if (usageMetadata) {
+				const { promptTokenCount = 0, candidatesTokenCount = 0 } = usageMetadata
+				yield {
+					type: "usage",
+					inputTokens: promptTokenCount,
+					outputTokens: candidatesTokenCount,
+					totalCost: calculateApiCostOpenAI(model.info, promptTokenCount, candidatesTokenCount, 0, 0),
+				}
+			}
 		}
 	}