From 89ec9c4277c15e0af2c2c39bb19a934c2eae4fe4 Mon Sep 17 00:00:00 2001 From: Daniel Steigman <35793213+NightTrek@users.noreply.github.com> Date: Wed, 20 Aug 2025 00:05:59 -0700 Subject: [PATCH] feat(telemetry): Add MCP tool usage tracking (#5698) * feat(telemetry): Add MCP tool usage tracking This commit introduces telemetry for MCP tool calls to monitor usage, success rates, and errors. - Adds a new telemetry event 'task.mcp_tool_called'. - Captures the server name, tool name, and status (started, success, error). - Integrates telemetry calls into the McpHub to track tool execution lifecycle. * chore: Add changeset for MCP telemetry * refactor(telemetry): Clean up MCP tool usage tracking This commit refactors the MCP tool usage tracking to be cleaner and more efficient. - Removes null checks for 'ulid' in the 'callTool' method. - Passes argument keys to the telemetry service for better monitoring without compromising user privacy. --- .changeset/shaggy-olives-smell.md | 5 ++ src/core/controller/index.ts | 1 + src/core/task/ToolExecutor.ts | 2 +- src/services/mcp/McpHub.ts | 71 ++++++++++++++----- .../posthog/telemetry/TelemetryService.ts | 36 ++++++++++ 5 files changed, 98 insertions(+), 17 deletions(-) create mode 100644 .changeset/shaggy-olives-smell.md diff --git a/.changeset/shaggy-olives-smell.md b/.changeset/shaggy-olives-smell.md new file mode 100644 index 0000000000..83781ba714 --- /dev/null +++ b/.changeset/shaggy-olives-smell.md @@ -0,0 +1,5 @@ +--- +"claude-dev": patch +--- + +Added MCP telemetry around when a MCP tool is used diff --git a/src/core/controller/index.ts b/src/core/controller/index.ts index 5e158ac51c..fecbeec641 100644 --- a/src/core/controller/index.ts +++ b/src/core/controller/index.ts @@ -91,6 +91,7 @@ export class Controller { () => ensureMcpServersDirectoryExists(), () => ensureSettingsDirectoryExists(this.context), this.context.extension?.packageJSON?.version ?? "1.0.0", + telemetryService, ) // Clean up legacy checkpoints diff --git a/src/core/task/ToolExecutor.ts b/src/core/task/ToolExecutor.ts index bbb98c6ed4..39ea1bddda 100644 --- a/src/core/task/ToolExecutor.ts +++ b/src/core/task/ToolExecutor.ts @@ -1401,7 +1401,7 @@ export class ToolExecutor { await this.say("mcp_notification", `[${notification.serverName}] ${notification.message}`) } - const toolResult = await this.mcpHub.callTool(server_name, tool_name, parsedArguments) + const toolResult = await this.mcpHub.callTool(server_name, tool_name, parsedArguments, this.ulid) // Check for any pending notifications after the tool call const notificationsAfter = this.mcpHub.getPendingNotifications() diff --git a/src/services/mcp/McpHub.ts b/src/services/mcp/McpHub.ts index fdf75fd2d1..479a275a30 100644 --- a/src/services/mcp/McpHub.ts +++ b/src/services/mcp/McpHub.ts @@ -33,6 +33,7 @@ import ReconnectingEventSource from "reconnecting-eventsource" import * as vscode from "vscode" import { z } from "zod" import { HostProvider } from "@/hosts/host-provider" +import { TelemetryService } from "@/services/posthog/telemetry/TelemetryService" import { ShowMessageType } from "@/shared/proto/host/window" import { DEFAULT_REQUEST_TIMEOUT_MS } from "./constants" import { BaseConfigSchema, McpSettingsSchema, ServerConfigSchema } from "./schemas" @@ -41,6 +42,7 @@ export class McpHub { getMcpServersPath: () => Promise private getSettingsDirectoryPath: () => Promise private clientVersion: string + private telemetryService: TelemetryService private disposables: vscode.Disposable[] = [] private settingsWatcher?: FSWatcher @@ -63,10 +65,12 @@ export class McpHub { getMcpServersPath: () => Promise, getSettingsDirectoryPath: () => Promise, clientVersion: string, + telemetryService: TelemetryService, ) { this.getMcpServersPath = getMcpServersPath this.getSettingsDirectoryPath = getSettingsDirectoryPath this.clientVersion = clientVersion + this.telemetryService = telemetryService this.watchMcpSettingsFile() this.initializeMcpServers() } @@ -808,7 +812,12 @@ export class McpHub { ) } - async callTool(serverName: string, toolName: string, toolArguments?: Record): Promise { + async callTool( + serverName: string, + toolName: string, + toolArguments: Record | undefined, + ulid: string, + ): Promise { const connection = this.connections.find((conn) => conn.server.name === serverName) if (!connection) { throw new Error( @@ -830,23 +839,53 @@ export class McpHub { console.error(`Failed to parse timeout configuration for server ${serverName}: ${error}`) } - const result = await connection.client.request( - { - method: "tools/call", - params: { - name: toolName, - arguments: toolArguments, - }, - }, - CallToolResultSchema, - { - timeout, - }, + this.telemetryService.captureMcpToolCall( + ulid, + serverName, + toolName, + "started", + undefined, + toolArguments ? Object.keys(toolArguments) : undefined, ) - return { - ...result, - content: result.content ?? [], + try { + const result = await connection.client.request( + { + method: "tools/call", + params: { + name: toolName, + arguments: toolArguments, + }, + }, + CallToolResultSchema, + { + timeout, + }, + ) + + this.telemetryService.captureMcpToolCall( + ulid, + serverName, + toolName, + "success", + undefined, + toolArguments ? Object.keys(toolArguments) : undefined, + ) + + return { + ...result, + content: result.content ?? [], + } + } catch (error) { + this.telemetryService.captureMcpToolCall( + ulid, + serverName, + toolName, + "error", + error instanceof Error ? error.message : String(error), + toolArguments ? Object.keys(toolArguments) : undefined, + ) + throw error } } diff --git a/src/services/posthog/telemetry/TelemetryService.ts b/src/services/posthog/telemetry/TelemetryService.ts index fc3fad93d7..9af6d11eeb 100644 --- a/src/services/posthog/telemetry/TelemetryService.ts +++ b/src/services/posthog/telemetry/TelemetryService.ts @@ -66,6 +66,8 @@ export class TelemetryService { CHECKPOINT_USED: "task.checkpoint_used", // Tracks when tools (like file operations, commands) are used TOOL_USED: "task.tool_used", + // Tracks when MCP tools are used + MCP_TOOL_CALLED: "task.mcp_tool_called", // Tracks when a historical task is loaded from storage HISTORICAL_LOADED: "task.historical_loaded", // Tracks when the retry button is clicked for failed operations @@ -376,6 +378,40 @@ export class TelemetryService { }) } + /** + * Records when an MCP tool is called. + * This telemetry event is designed to monitor the usage and performance of MCP tools + * without compromising user privacy. It captures the tool's metadata (server, name, and arguments) + * but explicitly avoids logging the values of the arguments. + * + * @param ulid Unique identifier for the task. + * @param serverName The name of the MCP server. + * @param toolName The name of the tool being called. + * @param status The status of the tool call. + * @param errorMessage Optional error message if the call failed. + * @param argumentKeys Optional array of argument keys for the tool. + */ + public captureMcpToolCall( + ulid: string, + serverName: string, + toolName: string, + status: "started" | "success" | "error", + errorMessage?: string, + argumentKeys?: string[], + ) { + this.capture({ + event: TelemetryService.EVENTS.TASK.MCP_TOOL_CALLED, + properties: { + ulid, + serverName, + toolName, + status, + errorMessage, + argumentKeys, + }, + }) + } + /** * Records interactions with the git-based checkpoint system * @param ulid Unique identifier for the task