activating extension with evals.env

.gitignore

@@ -17,3 +17,5 @@ pnpm-lock.yaml
 coverage
 # But don't ignore the coverage scripts in .github/scripts/
 !.github/scripts/coverage/
+
+*evals.env

evals/README.md

@@ -60,6 +60,17 @@ cline-repo/
 - VSCode with Cline extension installed
 - Git
 
+### Activation Mechanism
+
+The evaluation system uses an `evals.env` file approach to activate test mode in the Cline extension. When an evaluation is run:
+
+1. The CLI creates an `evals.env` file in the workspace directory
+2. The Cline extension activates due to the `workspaceContains:evals.env` activation event
+3. The extension detects this file and automatically enters test mode
+4. After evaluation completes, the file is automatically removed
+
+This approach eliminates the need for environment variables during the build process and allows for targeted activation only when needed for evaluations. The extension remains dormant during normal use, only activating when an evals.env file is present. For more details, see [Evals Env Activation](./docs/evals-env-activation.md).
+
 ### Installation
 
 1. Build the CLI tool:
@@ -106,6 +117,19 @@ Options:
 - `--format`: Report format (json, markdown) (default: markdown)
 - `--output`: Output path for the report
 
+#### Managing Test Mode Activation
+
+The CLI provides a command to manually manage the evals.env file for test mode activation:
+
+```bash
+node dist/index.js evals-env create  # Create evals.env file in current directory
+node dist/index.js evals-env remove  # Remove evals.env file from current directory
+node dist/index.js evals-env check   # Check if evals.env file exists in current directory
+```
+
+Options:
+- `--directory`: Specify a directory other than the current one
+
 ## Benchmarks
 
 ### Exercism

evals/cli/src/commands/evals-env.ts

@@ -0,0 +1,53 @@
+import * as path from "path"
+import chalk from "chalk"
+import { createEvalsEnvFile, removeEvalsEnvFile, checkEvalsEnvFile } from "../utils/evals-env"
+
+interface EvalsEnvOptions {
+	action: "create" | "remove" | "check"
+	directory?: string
+}
+
+/**
+ * Handler for the evals-env command
+ * @param options Command options
+ */
+export async function evalsEnvHandler(options: EvalsEnvOptions): Promise<void> {
+	// Determine the directory to use - default to repository root instead of current directory
+	const currentDir = process.cwd()
+	const repoRoot = path.resolve(currentDir, "..", "..") // Navigate up from evals/cli to root
+	const directory = options.directory || repoRoot
+
+	console.log(chalk.blue(`Working with directory: ${directory}`))
+
+	// Perform the requested action
+	switch (options.action) {
+		case "create":
+			console.log(chalk.blue("Creating evals.env file..."))
+			createEvalsEnvFile(directory)
+			console.log(chalk.green("The Cline extension should now detect this file and enter test mode."))
+			console.log(chalk.yellow("Note: You may need to reload VSCode for the changes to take effect."))
+			break
+
+		case "remove":
+			console.log(chalk.blue("Removing evals.env file..."))
+			removeEvalsEnvFile(directory)
+			console.log(chalk.green("The Cline extension should now exit test mode."))
+			console.log(chalk.yellow("Note: You may need to reload VSCode for the changes to take effect."))
+			break
+
+		case "check":
+			console.log(chalk.blue("Checking for evals.env file..."))
+			const exists = checkEvalsEnvFile(directory)
+			if (exists) {
+				console.log(chalk.green("The Cline extension should be in test mode."))
+			} else {
+				console.log(chalk.yellow("The Cline extension should not be in test mode."))
+			}
+			break
+
+		default:
+			console.error(chalk.red(`Unknown action: ${options.action}`))
+			console.log(chalk.yellow("Valid actions are: create, remove, check"))
+			break
+	}
+}

evals/cli/src/index.ts

@@ -4,6 +4,7 @@ import chalk from "chalk"
 import { setupHandler } from "./commands/setup"
 import { runHandler } from "./commands/run"
 import { reportHandler } from "./commands/report"
+import { evalsEnvHandler } from "./commands/evals-env"
 
 // Create the CLI program
 const program = new Command()
@@ -61,6 +62,21 @@ program
 		}
 	})
 
+// Evals-env command
+program
+	.command("evals-env")
+	.description("Manage evals.env files for test mode activation")
+	.argument("<action>", "Action to perform: create, remove, or check")
+	.option("-d, --directory <directory>", "Directory to create/remove/check evals.env file in (defaults to current directory)")
+	.action(async (action, options) => {
+		try {
+			await evalsEnvHandler({ action, ...options })
+		} catch (error) {
+			console.error(chalk.red(`Error managing evals.env file: ${error instanceof Error ? error.message : String(error)}`))
+			process.exit(1)
+		}
+	})
+
 // Parse command line arguments
 program.parse(process.argv)
 

evals/cli/src/utils/evals-env.ts

@@ -0,0 +1,79 @@
+import * as fs from "fs"
+import * as path from "path"
+import chalk from "chalk"
+
+/**
+ * Creates an evals.env file in the specified directory
+ * @param directory The directory where the evals.env file should be created
+ * @returns True if the file was created, false if it already exists
+ */
+export function createEvalsEnvFile(directory: string): boolean {
+	const evalsEnvPath = path.join(directory, "evals.env")
+
+	// Check if the file already exists
+	if (fs.existsSync(evalsEnvPath)) {
+		console.log(chalk.yellow(`evals.env file already exists at ${evalsEnvPath}`))
+		return false
+	}
+
+	// Create the file
+	try {
+		const content = `# This file activates Cline test mode
+# Created at: ${new Date().toISOString()}
+# 
+# This file is automatically detected by the Cline extension
+# and enables test mode for automated evaluations.
+#
+# Delete this file to deactivate test mode.
+`
+		fs.writeFileSync(evalsEnvPath, content)
+		console.log(chalk.green(`Created evals.env file at ${evalsEnvPath}`))
+		return true
+	} catch (error) {
+		console.error(chalk.red(`Error creating evals.env file: ${error}`))
+		return false
+	}
+}
+
+/**
+ * Removes an evals.env file from the specified directory
+ * @param directory The directory where the evals.env file should be removed
+ * @returns True if the file was removed, false if it doesn't exist
+ */
+export function removeEvalsEnvFile(directory: string): boolean {
+	const evalsEnvPath = path.join(directory, "evals.env")
+
+	// Check if the file exists
+	if (!fs.existsSync(evalsEnvPath)) {
+		console.log(chalk.yellow(`No evals.env file found at ${evalsEnvPath}`))
+		return false
+	}
+
+	// Remove the file
+	try {
+		fs.unlinkSync(evalsEnvPath)
+		console.log(chalk.green(`Removed evals.env file from ${evalsEnvPath}`))
+		return true
+	} catch (error) {
+		console.error(chalk.red(`Error removing evals.env file: ${error}`))
+		return false
+	}
+}
+
+/**
+ * Checks if an evals.env file exists in the specified directory
+ * @param directory The directory to check for an evals.env file
+ * @returns True if the file exists, false otherwise
+ */
+export function checkEvalsEnvFile(directory: string): boolean {
+	const evalsEnvPath = path.join(directory, "evals.env")
+	const exists = fs.existsSync(evalsEnvPath)
+
+	if (exists) {
+		console.log(chalk.green(`evals.env file found at ${evalsEnvPath}`))
+	} else {
+		console.log(chalk.yellow(`No evals.env file found at ${evalsEnvPath}`))
+	}
+
+	return exists
+}

evals/cli/src/utils/vscode.ts

@@ -31,14 +31,11 @@ export async function spawnVSCode(workspacePath: string, vsixPath?: string): Pro
 	// If no VSIX path is provided, build one with IS_TEST=true
 	if (!vsixPath) {
 		try {
-			// Build the VSIX with IS_TEST=true
-			console.log("Building test VSIX...")
+			// Build the VSIX (no longer need to set IS_TEST=true as we'll use evals.env file)
+			console.log("Building VSIX...")
 			const clineRoot = path.resolve(process.cwd(), "..", "..")
 			await execa("npx", ["vsce", "package"], {
 				cwd: clineRoot,
-				env: {
-					IS_TEST: "true",
-				},
 				stdio: "inherit",
 			})
 
@@ -89,6 +86,21 @@ export async function spawnVSCode(workspacePath: string, vsixPath?: string): Pro
 	fs.mkdirSync(tempExtensionsDir, { recursive: true })
 	console.log(`Created temporary extensions directory: ${tempExtensionsDir}`)
 
+	// Create evals.env file in the workspace to trigger test mode
+	console.log(`Creating evals.env file in workspace: ${workspacePath}`)
+	const evalsEnvPath = path.join(workspacePath, "evals.env")
+	fs.writeFileSync(
+		evalsEnvPath,
+		`# This file activates Cline test mode
+# Created at: ${new Date().toISOString()}
+# 
+# This file is automatically detected by the Cline extension
+# and enables test mode for automated evaluations.
+#
+# Delete this file to deactivate test mode.
+`,
+	)
+
 	// Create settings.json in the temporary user data directory to disable workspace trust
 	// and configure Cline to auto-open on startup
 	const settingsDir = path.join(tempUserDataDir, "User")
@@ -571,7 +583,7 @@ export async function cleanupVSCode(workspacePath: string): Promise<void> {
 		console.warn(`Error closing VS Code: ${error}`)
 	}
 
-	// Clean up temporary directories
+	// Clean up temporary directories and evals.env file
 	try {
 		console.log(`Removing temporary user data directory: ${resources.tempUserDataDir}`)
 		fs.rmSync(resources.tempUserDataDir, { recursive: true, force: true })
@@ -586,6 +598,17 @@ export async function cleanupVSCode(workspacePath: string): Promise<void> {
 		console.warn(`Error removing temporary extensions directory: ${error}`)
 	}
 
+	// Remove the evals.env file
+	try {
+		const evalsEnvPath = path.join(workspacePath, "evals.env")
+		if (fs.existsSync(evalsEnvPath)) {
+			console.log(`Removing evals.env file: ${evalsEnvPath}`)
+			fs.unlinkSync(evalsEnvPath)
+		}
+	} catch (error) {
+		console.warn(`Error removing evals.env file: ${error}`)
+	}
+
 	// Remove from the global map
 	workspaceResources.delete(workspacePath)
 

package.json

@@ -39,7 +39,9 @@
 		"ai",
 		"llama"
 	],
-	"activationEvents": [],
+	"activationEvents": [
+		"workspaceContains:evals.env"
+	],
 	"main": "./dist/extension.js",
 	"contributes": {
 		"viewsContainers": {

src/core/task/index.ts

@@ -10,7 +10,6 @@ import * as path from "path"
 import { serializeError } from "serialize-error"
 import * as vscode from "vscode"
 import { Logger } from "../../services/logging/Logger"
-const { IS_TEST } = process.env
 import { ApiHandler, buildApiHandler } from "../../api"
 import { AnthropicHandler } from "../../api/providers/anthropic"
 import { ClineHandler } from "../../api/providers/cline"
@@ -92,6 +91,7 @@ import { getGlobalState } from "../storage/state"
 import { parseSlashCommands } from ".././slash-commands"
 import WorkspaceTracker from "../../integrations/workspace/WorkspaceTracker"
 import { McpHub } from "@services/mcp/McpHub"
+import { isInTestMode } from "../../services/test/TestMode"
 
 export const cwd =
 	vscode.workspace.workspaceFolders?.map((folder) => folder.uri.fsPath).at(0) ?? path.join(os.homedir(), "Desktop") // may or may not exist but fs checking existence would immediately ask for permission which would be bad UX, need to come up with a better solution
@@ -1211,8 +1211,10 @@ export class Task {
 	}
 
 	async executeCommandTool(command: string): Promise<[boolean, ToolResponse]> {
+		Logger.info("IS_TEST: " + isInTestMode())
+
 		// Check if we're in test mode
-		if (IS_TEST === "true") {
+		if (isInTestMode()) {
 			// In test mode, execute the command directly in Node
 			Logger.info("Executing command in Node: " + command)
 			return this.executeCommandInNode(command)

src/extension.ts

@@ -2,6 +2,8 @@
 // Import the module and reference it with the alias vscode in your code below
 import { setTimeout as setTimeoutPromise } from "node:timers/promises"
 import * as vscode from "vscode"
+import * as fs from "fs"
+import * as path from "path"
 import { Logger } from "./services/logging/Logger"
 import { createClineAPI } from "./exports"
 import "./utils/path" // necessary to have access to String.prototype.toPosix
@@ -11,6 +13,7 @@ import { telemetryService } from "./services/telemetry/TelemetryService"
 import { WebviewProvider } from "./core/webview"
 import { createTestServer, shutdownTestServer } from "./services/test/TestServer"
 import { ErrorService } from "./services/error/ErrorService"
+import { setTestMode, isInTestMode } from "./services/test/TestMode"
 
 /*
 Built using https://github.com/microsoft/vscode-webview-ui-toolkit
@@ -23,6 +26,29 @@ https://github.com/microsoft/vscode-webview-ui-toolkit-samples/tree/main/framewo
 
 let outputChannel: vscode.OutputChannel
 
+// Check if we're in test mode by looking for evals.env file in workspace folders
+function checkForTestMode(): boolean {
+	// Get all workspace folders
+	const workspaceFolders = vscode.workspace.workspaceFolders || []
+
+	// Check each workspace folder for an evals.env file
+	for (const folder of workspaceFolders) {
+		const evalsEnvPath = path.join(folder.uri.fsPath, "evals.env")
+		if (fs.existsSync(evalsEnvPath)) {
+			Logger.log(`Found evals.env file at ${evalsEnvPath}, activating test mode`)
+			return true
+		}
+	}
+
+	// Check if IS_TEST environment variable is set (for backward compatibility)
+	if (process.env.IS_TEST === "true") {
+		Logger.log("IS_TEST environment variable is set (legacy), activating test mode")
+		return true
+	}
+
+	return false
+}
+
 // This method is called when your extension is activated
 // Your extension is activated the very first time the command is executed
 export function activate(context: vscode.ExtensionContext) {
@@ -33,10 +59,18 @@ export function activate(context: vscode.ExtensionContext) {
 	Logger.initialize(outputChannel)
 	Logger.log("Cline extension activated")
 
+	// Check if we're in test mode
+	const IS_TEST = checkForTestMode()
+	// Set test mode state for other parts of the code
+	if (IS_TEST) {
+		Logger.log("Test mode detected: Setting test mode state to true")
+		setTestMode(true)
+	}
+
 	const sidebarWebview = new WebviewProvider(context, outputChannel)
 
 	vscode.commands.executeCommand("setContext", "cline.isDevMode", IS_DEV && IS_DEV === "true")
-	vscode.commands.executeCommand("setContext", "cline.isTestMode", IS_TEST && IS_TEST === "true")
+	vscode.commands.executeCommand("setContext", "cline.isTestMode", IS_TEST)
 
 	context.subscriptions.push(
 		vscode.window.registerWebviewViewProvider(WebviewProvider.sideBarId, sidebarWebview, {
@@ -416,10 +450,36 @@ export function activate(context: vscode.ExtensionContext) {
 	)
 
 	// Set up test server if in test mode
-	if (IS_TEST === "true") {
+	if (IS_TEST) {
 		createTestServer(sidebarWebview)
 	}
 
+	// Watch for evals.env files being added or removed
+	const evalsEnvWatcher = vscode.workspace.createFileSystemWatcher("**/evals.env")
+
+	// When an evals.env file is created, activate test mode if not already active
+	evalsEnvWatcher.onDidCreate(async (uri) => {
+		Logger.log(`evals.env file created at ${uri.fsPath}`)
+		if (!isInTestMode()) {
+			setTestMode(true)
+			vscode.commands.executeCommand("setContext", "cline.isTestMode", true)
+			createTestServer(sidebarWebview)
+		}
+	})
+
+	// When an evals.env file is deleted, deactivate test mode if no other evals.env files exist
+	evalsEnvWatcher.onDidDelete(async (uri) => {
+		Logger.log(`evals.env file deleted at ${uri.fsPath}`)
+		// Only deactivate if this was the last evals.env file
+		if (!checkForTestMode()) {
+			setTestMode(false)
+			vscode.commands.executeCommand("setContext", "cline.isTestMode", false)
+			shutdownTestServer()
+		}
+	})
+
+	context.subscriptions.push(evalsEnvWatcher)
+
 	return createClineAPI(outputChannel, sidebarWebview.controller)
 }
 
@@ -429,7 +489,7 @@ export function activate(context: vscode.ExtensionContext) {
 //
 // This is a workaround to reload the extension when the source code changes
 // since vscode doesn't support hot reload for extensions
-const { IS_DEV, DEV_WORKSPACE_FOLDER, IS_TEST } = process.env
+const { IS_DEV, DEV_WORKSPACE_FOLDER } = process.env
 
 // This method is called when your extension is deactivated
 export function deactivate() {

src/services/test/TestMode.ts

@@ -0,0 +1,23 @@
+/**
+ * Module for managing test mode state across the extension
+ * This provides a centralized way to check if the extension is running in test mode
+ * instead of relying on process.env which may not be consistent across different parts of the extension
+ */
+
+let isTestMode = false
+
+/**
+ * Sets the test mode state
+ * @param value Whether test mode is enabled
+ */
+export function setTestMode(value: boolean): void {
+	isTestMode = value
+}
+
+/**
+ * Checks if the extension is running in test mode
+ * @returns True if in test mode, false otherwise
+ */
+export function isInTestMode(): boolean {
+	return isTestMode
+}