capture telemetry when the extension is closed

.changeset/early-oranges-study.md

@@ -0,0 +1,5 @@
+---
+"claude-dev": patch
+---
+
+Move to conversation tracking to the task close event

src/core/controller/index.ts

@@ -93,6 +93,20 @@ export class Controller {
 		cleanupLegacyCheckpoints(this.context.globalStorageUri.fsPath, this.outputChannel).catch((error) => {
 			console.error("Failed to cleanup legacy checkpoints:", error)
 		})
+
+		// Capture telemetry when the extension is disposed
+		this.context.subscriptions.push({
+			dispose: () => {
+				if (this.task) {
+					telemetryService.captureClosedEvent(
+						this.task.taskId,
+						this.task.apiConversationHistory.length,
+						this.task.apiConversationHistory.filter((msg) => msg.role === "user").length,
+						this.task.apiConversationHistory.filter((msg) => msg.role === "assistant").length,
+					)
+				}
+			},
+		})
 	}
 
 	/*
@@ -1827,7 +1841,15 @@ Here is the project's README to help you get started:\n\n${mcpDetails.readmeCont
 		}
 	}
 
-	async clearTask() {
+	async clearTask(isCloseTaskEvent: boolean = false) {
+		if (isCloseTaskEvent && this.task) {
+			await telemetryService.captureClosedEvent(
+				this.task.taskId,
+				this.task.apiConversationHistory.length,
+				this.task.apiConversationHistory.filter((msg) => msg.role === "user").length,
+				this.task.apiConversationHistory.filter((msg) => msg.role === "assistant").length,
+			)
+		}
 		this.task?.abortTask()
 		this.task = undefined // removes reference to it, so once promises end it will be garbage collected
 	}

src/core/controller/task/clearTask.ts

@@ -1,3 +1,4 @@
+import { telemetryService } from "@/services/telemetry/TelemetryService"
 import { Controller } from ".."
 import { Empty, EmptyRequest } from "../../../shared/proto/common"
 
@@ -8,7 +9,8 @@ import { Empty, EmptyRequest } from "../../../shared/proto/common"
  * @returns Empty response
  */
 export async function clearTask(controller: Controller, _request: EmptyRequest): Promise<Empty> {
-	await controller.clearTask()
+	// passing in true because this is only called when the user closes the task
+	await controller.clearTask(true)
 	await controller.postStateToWebview()
 	return Empty.create()
 }

src/core/task/index.ts

@@ -3492,8 +3492,6 @@ export class Task {
 			content: userContent,
 		})
 
-		telemetryService.captureConversationTurnEvent(this.taskId, currentProviderId, this.api.getModel().id, "user")
-
 		// since we sent off a placeholder api_req_started message to update the webview while waiting to actually start the API request (to load potential details for example), we need to update the text of that message
 		const lastApiReqIndex = findLastIndex(this.clineMessages, (m) => m.say === "api_req_started")
 		this.clineMessages[lastApiReqIndex].text = JSON.stringify({
@@ -3569,8 +3567,6 @@ export class Task {
 				updateApiReqMsg(cancelReason, streamingFailedMessage)
 				await this.saveClineMessagesAndUpdateHistory()
 
-				telemetryService.captureConversationTurnEvent(this.taskId, currentProviderId, this.api.getModel().id, "assistant")
-
 				// signals to provider that it can retrieve the saved messages from disk, as abortTask can not be awaited on in nature
 				this.didFinishAbortingStream = true
 			}
@@ -3712,8 +3708,6 @@ export class Task {
 			// need to save assistant responses to file before proceeding to tool use since user can exit at any moment and we wouldn't be able to save the assistant's response
 			let didEndLoop = false
 			if (assistantMessage.length > 0) {
-				telemetryService.captureConversationTurnEvent(this.taskId, currentProviderId, this.api.getModel().id, "assistant")
-
 				await this.addToApiConversationHistory({
 					role: "assistant",
 					content: [{ type: "text", text: assistantMessage }],

src/services/telemetry/TelemetryService.ts

@@ -23,8 +23,8 @@ class PostHogClient {
 			COMPLETED: "task.completed",
 			// Tracks user feedback on completed tasks
 			FEEDBACK: "task.feedback",
-			// Tracks when a message is sent in a conversation
-			CONVERSATION_TURN: "task.conversation_turn",
+			// Tracks when a task is closed
+			CLOSED: "task.closed",
 			// Tracks token consumption for cost and usage analysis
 			TOKEN_USAGE: "task.tokens",
 			// Tracks switches between plan and act modes
@@ -87,6 +87,8 @@ class PostHogClient {
 	private telemetryEnabled: boolean = false
 	/** Current version of the extension */
 	private readonly version: string = extensionVersion
+	/** Whether the extension is running in development mode */
+	private readonly isDev = process.env.IS_DEV
 
 	/**
 	 * Private constructor to enforce singleton pattern
@@ -146,6 +148,7 @@ class PostHogClient {
 			const propertiesWithVersion = {
 				...event.properties,
 				extension_version: this.version,
+				id_dev: this.isDev,
 			}
 			this.client.capture({ distinctId: this.distinctId, event: event.event, properties: propertiesWithVersion })
 		}
@@ -186,34 +189,34 @@ class PostHogClient {
 	}
 
 	/**
-	 * Captures that a message was sent, and includes the API provider and model used
+	 * Captures that a task was closed, and includes the number of conversation turns, user conversation turns, and assistant conversation turns
 	 * @param taskId Unique identifier for the task
-	 * @param provider The API provider (e.g., OpenAI, Anthropic)
-	 * @param model The specific model used (e.g., GPT-4, Claude)
-	 * @param source The source of the message ("user" | "model"). Used to track message patterns and identify when users need to correct the model's responses.
+	 * @param conversationTurns The total number of conversation turns
+	 * @param userConversationTurns The number of user conversation turns
+	 * @param assistantConversationTurns The number of assistant conversation turns
 	 */
-	public captureConversationTurnEvent(
+	public captureClosedEvent(
 		taskId: string,
-		provider: string = "unknown",
-		model: string = "unknown",
-		source: "user" | "assistant",
+		conversationTurns: number,
+		userConversationTurns: number,
+		assistantConversationTurns: number,
 	) {
 		// Ensure required parameters are provided
-		if (!taskId || !provider || !model || !source) {
+		if (!taskId) {
 			console.warn("TelemetryService: Missing required parameters for message capture")
 			return
 		}
 
 		const properties: Record<string, any> = {
 			taskId,
-			provider,
-			model,
-			source,
+			conversationTurns,
+			userConversationTurns,
+			assistantConversationTurns,
 			timestamp: new Date().toISOString(), // Add timestamp for message sequencing
 		}
 
 		this.capture({
-			event: PostHogClient.EVENTS.TASK.CONVERSATION_TURN,
+			event: PostHogClient.EVENTS.TASK.CLOSED,
 			properties,
 		})
 	}