Update lock

.changeset/happy-taxis-heal.md

@@ -0,0 +1,5 @@
+---
+"claude-dev": minor
+---
+
+opt-in llm observability

package.json

@@ -211,6 +211,11 @@
 					"type": "boolean",
 					"default": true,
 					"description": "Controls whether the MCP Marketplace is enabled."
+				},
+				"cline.conversationTelemetry": {
+					"type": "boolean",
+					"default": false,
+					"description": "Share message data,  code, and more extensive telemetry. This data may be used to improve prompts used in Cline, train models, and understand failure states more accurately."
 				}
 			}
 		}
@@ -272,6 +277,12 @@
 		"@google/generative-ai": "^0.18.0",
 		"@mistralai/mistralai": "^1.5.0",
 		"@modelcontextprotocol/sdk": "^1.7.0",
+		"@opentelemetry/api": "^1.4.1",
+		"@opentelemetry/exporter-trace-otlp-http": "^0.39.1",
+		"@opentelemetry/resources": "^1.30.1",
+		"@opentelemetry/sdk-node": "^0.39.1",
+		"@opentelemetry/sdk-trace-node": "^1.30.1",
+		"@opentelemetry/semantic-conventions": "^1.30.0",
 		"@types/clone-deep": "^4.0.4",
 		"@types/get-folder-size": "^3.0.4",
 		"@types/pdf-parse": "^1.1.4",

src/core/Cline.ts

@@ -64,6 +64,7 @@ import { ClineHandler } from "../api/providers/cline"
 import { ClineProvider } from "./webview/ClineProvider"
 import { DEFAULT_LANGUAGE_SETTINGS, getLanguageKey, LanguageDisplay, LanguageKey } from "../shared/Languages"
 import { telemetryService } from "../services/telemetry/TelemetryService"
+import { ConversationTelemetryService, TelemetryChatMessage } from "../services/telemetry/ConversationTelemetryService"
 import pTimeout from "p-timeout"
 import { GlobalFileNames } from "../global-constants"
 
@@ -1349,6 +1350,24 @@ export class Cline {
 			)
 		}
 
+		// Capture system prompt for telemetry,
+		// ONLY if user is opted in, in advanced settings
+		if (this.providerRef.deref()?.conversationTelemetryService.isOptedInToConversationTelemetry()) {
+			const systemMessage: TelemetryChatMessage = {
+				role: "system",
+				content: systemPrompt,
+				ts: Date.now(), // we dont uniquely identify system messages, so we use the timestamp as the id
+			}
+
+			// no need for timeout here, as there's no timestamp to compare to
+			this.providerRef.deref()?.conversationTelemetryService.captureMessage(this.taskId, systemMessage, {
+				apiProvider: this.apiProvider,
+				model: this.api.getModel().id,
+				tokensIn: 0,
+				tokensOut: 0,
+			})
+		}
+
 		// If the previous API request's total token usage is close to the context window, truncate the conversation history to free up space for the new request
 		if (previousApiReqIndex >= 0) {
 			const previousRequest = this.clineMessages[previousApiReqIndex]
@@ -3165,6 +3184,39 @@ export class Cline {
 
 		telemetryService.captureConversationTurnEvent(this.taskId, this.apiProvider, this.api.getModel().id, "user")
 
+		// Capture message data for telemetry,
+		// ONLY if user is opted in, in advanced settings
+		if (this.providerRef.deref()?.conversationTelemetryService.isOptedInToConversationTelemetry()) {
+			// Get the last message from apiConversationHistory
+			const lastMessage = this.apiConversationHistory[this.apiConversationHistory.length - 1]
+
+			// Get the corresponding timestamp from clineMessages
+			// The last message in clineMessages should be the one we just added
+
+			const lastClineMessage = this.clineMessages[this.clineMessages.length - 1]
+			const ts = lastClineMessage.ts
+
+			// Send individual message to telemetry
+			this.providerRef.deref()?.conversationTelemetryService.captureMessage(
+				this.taskId,
+				// Add the timestamp to the message object for telemetry
+				{
+					...lastMessage,
+					ts,
+				},
+				{
+					apiProvider: this.apiProvider,
+					model: this.api.getModel().id,
+					tokensIn: 0,
+					tokensOut: 0,
+				},
+			)
+
+			// Send entire conversation history to cleanup endpoint
+			// This ensures deleted messages are properly handled in telemetry
+			this.providerRef.deref()?.conversationTelemetryService.cleanupTask(this.taskId, this.clineMessages)
+		}
+
 		// since we sent off a placeholder api_req_started message to update the webview while waiting to actually start the API request (to load potential details for example), we need to update the text of that message
 		const lastApiReqIndex = findLastIndex(this.clineMessages, (m) => m.say === "api_req_started")
 		this.clineMessages[lastApiReqIndex].text = JSON.stringify({
@@ -3242,6 +3294,36 @@ export class Cline {
 
 				telemetryService.captureConversationTurnEvent(this.taskId, this.apiProvider, this.api.getModel().id, "assistant")
 
+				// Capture message data for telemetry after assistant response
+				// ONLY if user is opted in, in advanced settings
+				if (this.providerRef.deref()?.conversationTelemetryService.isOptedInToConversationTelemetry()) {
+					// Get the last message from apiConversationHistory
+					const lastMessage = this.apiConversationHistory[this.apiConversationHistory.length - 1]
+
+					// Find the corresponding timestamp from clineMessages
+					// For assistant messages, we need to find the most recent "text" message
+					const lastTextMessage = findLast(this.clineMessages, (m) => m.say === "text")
+
+					// Add the timestamp to the message object for telemetry
+					if (!lastTextMessage) {
+						console.error("No text message found in clineMessages")
+					} else {
+						this.providerRef.deref()?.conversationTelemetryService.captureMessage(
+							this.taskId,
+							{
+								...lastMessage,
+								ts: lastTextMessage.ts,
+							},
+							{
+								apiProvider: this.apiProvider,
+								model: this.api.getModel().id,
+								tokensIn: inputTokens,
+								tokensOut: outputTokens,
+							},
+						)
+					}
+				}
+
 				// signals to provider that it can retrieve the saved messages from disk, as abortTask can not be awaited on in nature
 				this.didFinishAbortingStream = true
 			}
@@ -3391,6 +3473,32 @@ export class Cline {
 					content: [{ type: "text", text: assistantMessage }],
 				})
 
+				// Capture message data for telemetry after assistant response,
+				// ONLY if user is opted in, in advanced settings
+				if (this.providerRef.deref()?.conversationTelemetryService.isOptedInToConversationTelemetry()) {
+					// Get the last message from apiConversationHistory
+					const lastMessage = this.apiConversationHistory[this.apiConversationHistory.length - 1]
+
+					// Find the corresponding timestamp from clineMessages
+					const lastClineMessage = this.clineMessages[this.clineMessages.length - 1]
+
+					if (lastClineMessage) {
+						this.providerRef.deref()?.conversationTelemetryService.captureMessage(
+							this.taskId,
+							{
+								...lastMessage,
+								ts: lastClineMessage.ts,
+							},
+							{
+								apiProvider: this.apiProvider,
+								model: this.api.getModel().id,
+								tokensIn: inputTokens,
+								tokensOut: outputTokens,
+							},
+						)
+					}
+				}
+
 				// NOTE: this comment is here for future reference - this was a workaround for userMessageContent not getting set to true. It was due to it not recursively calling for partial blocks when didRejectTool, so it would get stuck waiting for a partial block to complete before it could continue.
 				// in case the content blocks finished
 				// it may be the api stream finished after the last parsed content block was executed, so  we are able to detect out of bounds and set userMessageContentReady to true (note you should not call presentAssistantMessage since if the last block is completed it will be presented again)

src/core/webview/ClineProvider.ts

@@ -37,6 +37,7 @@ import { TelemetrySetting } from "../../shared/TelemetrySetting"
 import { cleanupLegacyCheckpoints } from "../../integrations/checkpoints/CheckpointMigration"
 import CheckpointTracker from "../../integrations/checkpoints/CheckpointTracker"
 import { getTotalTasksSize } from "../../utils/storage"
+import { ConversationTelemetryService } from "../../services/telemetry/ConversationTelemetryService"
 import { GlobalFileNames } from "../../global-constants"
 
 /*
@@ -121,6 +122,7 @@ export class ClineProvider implements vscode.WebviewViewProvider {
 	workspaceTracker?: WorkspaceTracker
 	mcpHub?: McpHub
 	private latestAnnouncementId = "feb-19-2025" // update to some unique identifier when we add a new announcement
+	conversationTelemetryService: ConversationTelemetryService
 
 	constructor(
 		readonly context: vscode.ExtensionContext,
@@ -130,6 +132,7 @@ export class ClineProvider implements vscode.WebviewViewProvider {
 		ClineProvider.activeInstances.add(this)
 		this.workspaceTracker = new WorkspaceTracker(this)
 		this.mcpHub = new McpHub(this)
+		this.conversationTelemetryService = new ConversationTelemetryService(this)
 
 		// Clean up legacy checkpoints
 		cleanupLegacyCheckpoints(this.context.globalStorageUri.fsPath, this.outputChannel).catch((error) => {
@@ -160,6 +163,7 @@ export class ClineProvider implements vscode.WebviewViewProvider {
 		this.workspaceTracker = undefined
 		this.mcpHub?.dispose()
 		this.mcpHub = undefined
+		this.conversationTelemetryService.shutdown()
 		this.outputChannel.appendLine("Disposed all disposables")
 		ClineProvider.activeInstances.delete(this)
 	}

src/services/telemetry/ConversationTelemetryService.ts

@@ -0,0 +1,304 @@
+import { context, SpanKind, trace } from "@opentelemetry/api"
+import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http"
+import { Resource } from "@opentelemetry/resources"
+import { SimpleSpanProcessor } from "@opentelemetry/sdk-trace-base"
+import { NodeTracerProvider } from "@opentelemetry/sdk-trace-node"
+import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from "@opentelemetry/semantic-conventions"
+import { Anthropic } from "@anthropic-ai/sdk"
+import * as vscode from "vscode"
+import { ClineProvider } from "../../core/webview/ClineProvider"
+
+export type TelemetryChatMessage = {
+	role: "user" | "assistant" | "system"
+	ts: number
+	content: Anthropic.Messages.MessageParam["content"]
+}
+
+const { IS_DEV } = process.env
+
+interface ConversationMetadata {
+	apiProvider?: string
+	model?: string
+	tokensIn: number
+	tokensOut: number
+}
+
+/**
+ * Service for collecting conversation data using OpenTelemetry
+ */
+export class ConversationTelemetryService {
+	private providerRef: WeakRef<ClineProvider>
+	private distinctId: string = vscode.env.machineId
+	private apiEndpoint: string = "https://api.cline.bot/v1/traces"
+	private tracerProvider: NodeTracerProvider | undefined
+	private tracer: any
+	private messageIndices: Map<string, number> = new Map()
+
+	constructor(provider: ClineProvider) {
+		this.providerRef = new WeakRef(provider)
+		this.initializeTracer()
+	}
+
+	private async getClineApiKey(): Promise<string | undefined> {
+		const provider = this.providerRef.deref()
+		if (!provider) {
+			return undefined
+		}
+		const { apiConfiguration } = await provider.getStateToPostToWebview()
+		return apiConfiguration?.clineApiKey
+	}
+
+	public isOptedInToConversationTelemetry(): boolean {
+		// First check global telemetry level - telemetry should only be enabled when level is "all"
+		const telemetryLevel = vscode.workspace.getConfiguration("telemetry").get<string>("telemetryLevel", "all")
+		const isGlobalTelemetryEnabled = telemetryLevel === "all"
+
+		// User has to manually opt in to conversation telemetry in Advanced Settings
+		const isConversationTelemetryEnabled =
+			vscode.workspace.getConfiguration("cline").get<boolean>("conversationTelemetry") ?? false
+
+		// Currently only enabled in dev environment
+		const isDevEnvironment = !!IS_DEV
+
+		return isDevEnvironment && isGlobalTelemetryEnabled && isConversationTelemetryEnabled
+	}
+
+	private async initializeTracer() {
+		try {
+			// Create a resource that identifies our service
+			const resource = new Resource({
+				[ATTR_SERVICE_NAME]: "cline-extension",
+				[ATTR_SERVICE_VERSION]: "1.0.0",
+			})
+
+			const clineApiKey = await this.getClineApiKey()
+
+			console.log("[ConversationTelemetry] Initializing OpenTelemetry tracer...")
+
+			// Configure the OTLP exporter
+			const headers: Record<string, string> = {
+				"Content-Type": "application/json",
+			}
+
+			// Add API key to headers if available
+			if (clineApiKey) {
+				headers["Authorization"] = `Bearer ${clineApiKey}`
+			}
+
+			const exporter = new OTLPTraceExporter({
+				url: this.apiEndpoint,
+				headers,
+			})
+
+			// Create the span processor
+			const spanProcessor = new SimpleSpanProcessor(exporter as any)
+
+			// Create the trace provider with the span processor in the config
+			this.tracerProvider = new NodeTracerProvider({
+				resource,
+				spanProcessors: [spanProcessor as any],
+			})
+
+			// Register the provider
+			this.tracerProvider.register()
+
+			// Get a tracer
+			this.tracer = trace.getTracer("cline-conversation-tracer")
+
+			console.log("[ConversationTelemetry] OpenTelemetry tracer initialized successfully")
+		} catch (error) {
+			console.error("[ConversationTelemetry] Failed to initialize OpenTelemetry tracer:", error)
+		}
+	}
+
+	/**
+	 * Captures a message in the conversation as an OpenTelemetry span
+	 * ONLY HAPPENS IF USER IS OPTED INTO CONVERSATION TELEMETRY IN ADVANCED SETTINGS
+	 */
+	public async captureMessage(taskId: string, message: TelemetryChatMessage, metadata: ConversationMetadata) {
+		// Do NOT capture message if user has not explicitly opted in
+		if (!this.isOptedInToConversationTelemetry()) {
+			return
+		}
+
+		if (!this.tracer) {
+			return
+		}
+
+		try {
+			// Convert taskId to a valid trace ID (must be 32 hex chars)
+			const traceId = this.generateTraceIdFromTimestamp(taskId)
+
+			// Convert message timestamp to a valid span ID (must be 16 hex chars)
+			if (!message.ts && message.ts !== 0) {
+				throw new Error("Message timestamp is required")
+			}
+
+			const timestamp = message.ts
+			const spanId = this.generateSpanIdFromTimestamp(timestamp)
+
+			// Create a span context with our IDs
+			const spanContext = trace.setSpanContext(context.active(), {
+				traceId,
+				spanId,
+				isRemote: false,
+				traceFlags: 1, // Sampled
+			})
+
+			// Start a new span with the context
+			const span = this.tracer.startSpan(
+				`message.${message.role}`,
+				{
+					kind: SpanKind.CLIENT,
+					startTime: this.millisecondsToHrTime(timestamp), // Convert to nanoseconds
+				},
+				spanContext,
+			)
+
+			// Get the message index for this task
+			const messageIndex = this.getNextMessageIndex(taskId)
+
+			// Add attributes to the span
+			span.setAttribute("task.id", taskId)
+			span.setAttribute("user.id", this.distinctId)
+			span.setAttribute("message.role", message.role)
+			span.setAttribute("message.timestamp", timestamp)
+			span.setAttribute("message.index", messageIndex)
+
+			const c = message.content
+
+			// Add Braintrust-compatible attributes
+			span.setAttribute("gen_ai.request.model", metadata.model)
+
+			if (message.role === "user") {
+				span.setAttribute("gen_ai.prompt", this.extractContent(message))
+			} else if (message.role === "assistant") {
+				span.setAttribute("gen_ai.completion", this.extractContent(message))
+				span.setAttribute("gen_ai.usage.prompt_tokens", metadata.tokensIn)
+				span.setAttribute("gen_ai.usage.completion_tokens", metadata.tokensOut)
+			} else if (message.role === "system") {
+				span.setAttribute("gen_ai.system_prompt", this.extractContent(message))
+			}
+
+			// Add custom metadata in Braintrust format
+			span.setAttribute("braintrust.metadata.api_provider", metadata.apiProvider)
+			span.setAttribute("braintrust.metadata.ts", message.ts)
+
+			// End the span immediately since messages are discrete events
+			span.end(this.millisecondsToHrTime(timestamp)) // Convert to nanoseconds
+
+			console.log(`[ConversationTelemetry] Captured ${message.role} message for task ${taskId}`, { span })
+		} catch (error) {
+			console.error("[ConversationTelemetry] Error capturing message:", error)
+		}
+	}
+
+	/**
+	 * Convert a decimal timestamp to a valid trace ID (32 hex chars)
+	 */
+	private generateTraceIdFromTimestamp(timestamp: string): string {
+		// Pad with zeros and convert to hex
+		const hex = BigInt(timestamp).toString(16).padStart(32, "0")
+		return hex.substring(0, 32) // Ensure it's exactly 32 chars
+	}
+
+	/**
+	 * Converts milliseconds to high-resolution time format expected by OpenTelemetry
+	 * Returns [seconds, nanoseconds]
+	 */
+	private millisecondsToHrTime(milliseconds: number): [number, number] {
+		return [
+			Math.floor(milliseconds / 1000), // seconds
+			(milliseconds % 1000) * 1000000, // nanoseconds (remainder in ms * 10^6)
+		]
+	}
+
+	/**
+	 * Convert a decimal timestamp to a valid span ID (16 hex chars)
+	 */
+	private generateSpanIdFromTimestamp(timestamp: number): string {
+		// Pad with zeros and convert to hex
+		const hex = BigInt(timestamp).toString(16).padStart(16, "0")
+		return hex.substring(0, 16) // Ensure it's exactly 16 chars
+	}
+
+	/**
+	 * Helper to extract content from different message formats
+	 */
+	private extractContent(message: TelemetryChatMessage): string {
+		if (typeof message.content === "string") {
+			return message.content
+		}
+
+		return message.content
+			.map((block) => (block.type === "text" ? block.text : null))
+			.filter(Boolean)
+			.join("\n")
+	}
+
+	/**
+	 * Track message indices per task
+	 */
+	private getNextMessageIndex(taskId: string): number {
+		const currentIndex = this.messageIndices.get(taskId) || 0
+		this.messageIndices.set(taskId, currentIndex + 1)
+		return currentIndex
+	}
+
+	/**
+	 * Sends conversation data to cleanup endpoint to remove deleted messages from telemetry
+	 * ONLY HAPPENS IF USER IS OPTED INTO CONVERSATION TELEMETRY IN ADVANCED SETTINGS
+	 */
+	public async cleanupTask(taskId: string, conversationData: any): Promise<void> {
+		// Do NOT send data if user has not explicitly opted in
+		if (!this.isOptedInToConversationTelemetry()) {
+			return
+		}
+
+		const clineApiKey = await this.getClineApiKey()
+		if (!clineApiKey) {
+			return
+		}
+
+		try {
+			// Configure the headers with API key
+			const headers: Record<string, string> = {
+				"Content-Type": "application/json",
+			}
+
+			// Add API key to headers
+			headers["Authorization"] = `Bearer ${clineApiKey}`
+
+			// Send the data to the cleanup endpoint
+			const cleanupEndpoint = `${this.apiEndpoint.replace("/traces", "/traces/cleanup")}`
+
+			// Use fetch API to send the data
+			const response = await fetch(cleanupEndpoint, {
+				method: "POST",
+				headers,
+				body: JSON.stringify({
+					taskId: taskId,
+					conversationData,
+					userId: this.distinctId,
+				}),
+			})
+
+			if (!response.ok) {
+				throw new Error(`Failed to send cleanup data: ${response.status} ${response.statusText}`)
+			}
+
+			console.log(`[ConversationTelemetry] Cleanup data sent for task ${taskId}`)
+		} catch (error) {
+			console.error("[ConversationTelemetry] Error sending cleanup data:", error)
+		}
+	}
+
+	/**
+	 * Shutdown the tracer provider
+	 */
+	public async shutdown(): Promise<void> {
+		if (this.tracerProvider) {
+			await this.tracerProvider.shutdown()
+		}
+	}
+}