feat(telemetry): Add MCP tool usage tracking (#5698)

* feat(telemetry): Add MCP tool usage tracking

This commit introduces telemetry for MCP tool calls to monitor usage, success rates, and errors.

- Adds a new telemetry event 'task.mcp_tool_called'.
- Captures the server name, tool name, and status (started, success, error).
- Integrates telemetry calls into the McpHub to track tool execution lifecycle.

* chore: Add changeset for MCP telemetry

* refactor(telemetry): Clean up MCP tool usage tracking

This commit refactors the MCP tool usage tracking to be cleaner and more efficient.

- Removes null checks for 'ulid' in the 'callTool' method.
- Passes argument keys to the telemetry service for better monitoring without compromising user privacy.

.changeset/shaggy-olives-smell.md

@@ -0,0 +1,5 @@
+---
+"claude-dev": patch
+---
+
+Added MCP telemetry around when a MCP tool is used

src/core/controller/index.ts

@@ -91,6 +91,7 @@ export class Controller {
 			() => ensureMcpServersDirectoryExists(),
 			() => ensureSettingsDirectoryExists(this.context),
 			this.context.extension?.packageJSON?.version ?? "1.0.0",
+			telemetryService,
 		)
 
 		// Clean up legacy checkpoints

src/core/task/ToolExecutor.ts

@@ -1401,7 +1401,7 @@ export class ToolExecutor {
 							await this.say("mcp_notification", `[${notification.serverName}] ${notification.message}`)
 						}
 
-						const toolResult = await this.mcpHub.callTool(server_name, tool_name, parsedArguments)
+						const toolResult = await this.mcpHub.callTool(server_name, tool_name, parsedArguments, this.ulid)
 
 						// Check for any pending notifications after the tool call
 						const notificationsAfter = this.mcpHub.getPendingNotifications()

src/services/mcp/McpHub.ts

@@ -33,6 +33,7 @@ import ReconnectingEventSource from "reconnecting-eventsource"
 import * as vscode from "vscode"
 import { z } from "zod"
 import { HostProvider } from "@/hosts/host-provider"
+import { TelemetryService } from "@/services/posthog/telemetry/TelemetryService"
 import { ShowMessageType } from "@/shared/proto/host/window"
 import { DEFAULT_REQUEST_TIMEOUT_MS } from "./constants"
 import { BaseConfigSchema, McpSettingsSchema, ServerConfigSchema } from "./schemas"
@@ -41,6 +42,7 @@ export class McpHub {
 	getMcpServersPath: () => Promise<string>
 	private getSettingsDirectoryPath: () => Promise<string>
 	private clientVersion: string
+	private telemetryService: TelemetryService
 
 	private disposables: vscode.Disposable[] = []
 	private settingsWatcher?: FSWatcher
@@ -63,10 +65,12 @@ export class McpHub {
 		getMcpServersPath: () => Promise<string>,
 		getSettingsDirectoryPath: () => Promise<string>,
 		clientVersion: string,
+		telemetryService: TelemetryService,
 	) {
 		this.getMcpServersPath = getMcpServersPath
 		this.getSettingsDirectoryPath = getSettingsDirectoryPath
 		this.clientVersion = clientVersion
+		this.telemetryService = telemetryService
 		this.watchMcpSettingsFile()
 		this.initializeMcpServers()
 	}
@@ -808,7 +812,12 @@ export class McpHub {
 		)
 	}
 
-	async callTool(serverName: string, toolName: string, toolArguments?: Record<string, unknown>): Promise<McpToolCallResponse> {
+	async callTool(
+		serverName: string,
+		toolName: string,
+		toolArguments: Record<string, unknown> | undefined,
+		ulid: string,
+	): Promise<McpToolCallResponse> {
 		const connection = this.connections.find((conn) => conn.server.name === serverName)
 		if (!connection) {
 			throw new Error(
@@ -830,23 +839,53 @@ export class McpHub {
 			console.error(`Failed to parse timeout configuration for server ${serverName}: ${error}`)
 		}
 
-		const result = await connection.client.request(
-			{
-				method: "tools/call",
-				params: {
-					name: toolName,
-					arguments: toolArguments,
-				},
-			},
-			CallToolResultSchema,
-			{
-				timeout,
-			},
+		this.telemetryService.captureMcpToolCall(
+			ulid,
+			serverName,
+			toolName,
+			"started",
+			undefined,
+			toolArguments ? Object.keys(toolArguments) : undefined,
 		)
 
-		return {
-			...result,
-			content: result.content ?? [],
+		try {
+			const result = await connection.client.request(
+				{
+					method: "tools/call",
+					params: {
+						name: toolName,
+						arguments: toolArguments,
+					},
+				},
+				CallToolResultSchema,
+				{
+					timeout,
+				},
+			)
+
+			this.telemetryService.captureMcpToolCall(
+				ulid,
+				serverName,
+				toolName,
+				"success",
+				undefined,
+				toolArguments ? Object.keys(toolArguments) : undefined,
+			)
+
+			return {
+				...result,
+				content: result.content ?? [],
+			}
+		} catch (error) {
+			this.telemetryService.captureMcpToolCall(
+				ulid,
+				serverName,
+				toolName,
+				"error",
+				error instanceof Error ? error.message : String(error),
+				toolArguments ? Object.keys(toolArguments) : undefined,
+			)
+			throw error
 		}
 	}
 

src/services/posthog/telemetry/TelemetryService.ts

@@ -66,6 +66,8 @@ export class TelemetryService {
 			CHECKPOINT_USED: "task.checkpoint_used",
 			// Tracks when tools (like file operations, commands) are used
 			TOOL_USED: "task.tool_used",
+			// Tracks when MCP tools are used
+			MCP_TOOL_CALLED: "task.mcp_tool_called",
 			// Tracks when a historical task is loaded from storage
 			HISTORICAL_LOADED: "task.historical_loaded",
 			// Tracks when the retry button is clicked for failed operations
@@ -376,6 +378,40 @@ export class TelemetryService {
 		})
 	}
 
+	/**
+	 * Records when an MCP tool is called.
+	 * This telemetry event is designed to monitor the usage and performance of MCP tools
+	 * without compromising user privacy. It captures the tool's metadata (server, name, and arguments)
+	 * but explicitly avoids logging the values of the arguments.
+	 *
+	 * @param ulid Unique identifier for the task.
+	 * @param serverName The name of the MCP server.
+	 * @param toolName The name of the tool being called.
+	 * @param status The status of the tool call.
+	 * @param errorMessage Optional error message if the call failed.
+	 * @param argumentKeys Optional array of argument keys for the tool.
+	 */
+	public captureMcpToolCall(
+		ulid: string,
+		serverName: string,
+		toolName: string,
+		status: "started" | "success" | "error",
+		errorMessage?: string,
+		argumentKeys?: string[],
+	) {
+		this.capture({
+			event: TelemetryService.EVENTS.TASK.MCP_TOOL_CALLED,
+			properties: {
+				ulid,
+				serverName,
+				toolName,
+				status,
+				errorMessage,
+				argumentKeys,
+			},
+		})
+	}
+
 	/**
 	 * Records interactions with the git-based checkpoint system
 	 * @param ulid Unique identifier for the task