Compare commits

...
5 changed files with 188 additions and 345 deletions
+12 -17
View File
@@ -256,23 +256,18 @@ export class GeminiHandler implements ApiHandler {
totalDurationSdkMs > 0 && outputTokens > 0 ? outputTokens / (totalDurationSdkMs / 1000) : undefined
if (this.options.taskId) {
telemetryService.captureGeminiApiPerformance(
this.options.taskId,
modelId,
{
ttftSec: ttftSdkMs !== undefined ? ttftSdkMs / 1000 : undefined,
totalDurationSec: totalDurationSdkMs / 1000,
promptTokens,
outputTokens,
cacheReadTokens,
cacheHit,
cacheHitPercentage,
apiSuccess,
apiError,
throughputTokensPerSec: throughputTokensPerSecSdk,
},
true,
)
telemetryService.captureGeminiApiPerformance(this.options.taskId, modelId, {
ttftSec: ttftSdkMs !== undefined ? ttftSdkMs / 1000 : undefined,
totalDurationSec: totalDurationSdkMs / 1000,
promptTokens,
outputTokens,
cacheReadTokens,
cacheHit,
cacheHitPercentage,
apiSuccess,
apiError,
throughputTokensPerSec: throughputTokensPerSecSdk,
})
} else {
console.warn("GeminiHandler: taskId not available for telemetry in createMessage.")
}
-1
View File
@@ -905,7 +905,6 @@ export class Controller {
async clearTask() {
if (this.task) {
await telemetryService.sendCollectedEvents(this.task.taskId)
}
await this.task?.abortTask()
this.task = undefined // removes reference to it, so once promises end it will be garbage collected
+1 -3
View File
@@ -2066,7 +2066,7 @@ export class Task {
content: userContent,
})
telemetryService.captureConversationTurnEvent(this.taskId, currentProviderId, this.api.getModel().id, "user", true)
telemetryService.captureConversationTurnEvent(this.taskId, currentProviderId, this.api.getModel().id, "user")
// since we sent off a placeholder api_req_started message to update the webview while waiting to actually start the API request (to load potential details for example), we need to update the text of that message
const lastApiReqIndex = findLastIndex(this.messageStateHandler.getClineMessages(), (m) => m.say === "api_req_started")
@@ -2136,7 +2136,6 @@ export class Task {
currentProviderId,
this.api.getModel().id,
"assistant",
true,
{
tokensIn: inputTokens,
tokensOut: outputTokens,
@@ -2316,7 +2315,6 @@ export class Task {
currentProviderId,
this.api.getModel().id,
"assistant",
true,
{
tokensIn: inputTokens,
tokensOut: outputTokens,
+6 -8
View File
@@ -363,7 +363,7 @@ export async function activate(context: vscode.ExtensionContext) {
languageId,
Array.isArray(diagnostics) ? diagnostics : undefined,
)
telemetryService.captureButtonClick("codeAction_addToChat", visibleWebview?.controller.task?.taskId, true)
telemetryService.captureButtonClick("codeAction_addToChat", visibleWebview?.controller.task?.taskId)
}),
)
@@ -540,7 +540,7 @@ export async function activate(context: vscode.ExtensionContext) {
// Send to sidebar provider with diagnostics
const visibleWebview = WebviewProvider.getVisibleInstance()
await visibleWebview?.controller.fixWithCline(selectedText, filePath, languageId, diagnostics)
telemetryService.captureButtonClick("codeAction_fixWithCline", visibleWebview?.controller.task?.taskId, true)
telemetryService.captureButtonClick("codeAction_fixWithCline", visibleWebview?.controller.task?.taskId)
}),
)
@@ -567,7 +567,7 @@ export async function activate(context: vscode.ExtensionContext) {
const fileMention = visibleWebview?.controller.getFileMentionFromPath(filePath) || filePath
const prompt = `Explain the following code from ${fileMention}:\n\`\`\`${editor.document.languageId}\n${selectedText}\n\`\`\``
await visibleWebview?.controller.initTask(prompt)
telemetryService.captureButtonClick("codeAction_explainCode", visibleWebview?.controller.task?.taskId, true)
telemetryService.captureButtonClick("codeAction_explainCode", visibleWebview?.controller.task?.taskId)
}),
)
@@ -594,7 +594,7 @@ export async function activate(context: vscode.ExtensionContext) {
const fileMention = visibleWebview?.controller.getFileMentionFromPath(filePath) || filePath
const prompt = `Improve the following code from ${fileMention} (e.g., suggest refactorings, optimizations, or better practices):\n\`\`\`${editor.document.languageId}\n${selectedText}\n\`\`\``
await visibleWebview?.controller.initTask(prompt)
telemetryService.captureButtonClick("codeAction_improveCode", visibleWebview?.controller.task?.taskId, true)
telemetryService.captureButtonClick("codeAction_improveCode", visibleWebview?.controller.task?.taskId)
}),
)
@@ -658,7 +658,7 @@ export async function activate(context: vscode.ExtensionContext) {
}),
)
}
telemetryService.captureButtonClick("command_focusChatInput", activeWebviewProvider?.controller.task?.taskId, true)
telemetryService.captureButtonClick("command_focusChatInput", activeWebviewProvider?.controller.task?.taskId)
}),
)
@@ -666,7 +666,7 @@ export async function activate(context: vscode.ExtensionContext) {
context.subscriptions.push(
vscode.commands.registerCommand("cline.openWalkthrough", async () => {
await vscode.commands.executeCommand("workbench.action.openWalkthrough", "saoudrizwan.claude-dev#ClineWalkthrough")
telemetryService.captureButtonClick("command_openWalkthrough", undefined, true)
telemetryService.captureButtonClick("command_openWalkthrough")
}),
)
@@ -728,8 +728,6 @@ export async function deactivate() {
// Dispose all webview instances
await WebviewProvider.disposeAllInstances()
await telemetryService.sendCollectedEvents()
// Clean up test mode
cleanupTestMode()
await posthogClientProvider.shutdown()
+169 -316
View File
@@ -12,16 +12,6 @@ import { posthogClientProvider } from "../PostHogClientProvider"
* Respects user privacy settings and VSCode's global telemetry configuration
*/
interface CollectedTasks {
taskId: string
collection: Collection[]
}
interface Collection {
event: string
properties: any
}
/**
* Represents telemetry event categories that can be individually enabled or disabled
* When adding a new category, add it both here and to the initial values in telemetryCategoryEnabled
@@ -41,8 +31,6 @@ class TelemetryService {
["browser", true], // Browser telemetry enabled
])
// Stores events when collect=true
private collectedTasks: CollectedTasks[] = []
// Event constants for tracking user interactions and system events
private static readonly EVENTS = {
// Task-related events for tracking conversation and execution flow
@@ -90,8 +78,6 @@ class TelemetryService {
GEMINI_API_PERFORMANCE: "task.gemini_api_performance",
// Tracks when API providers return errors
PROVIDER_API_ERROR: "task.provider_api_error",
// Collection of all task events
TASK_COLLECTION: "task.collection",
},
// UI interaction events for tracking user engagement
UI: {
@@ -197,15 +183,13 @@ class TelemetryService {
}
/**
* Captures a telemetry event if telemetry is enabled or collects if collect=true
* Captures a telemetry event if telemetry is enabled
* @param event The event to capture with its properties
* @param collect If true, store the event in collectedEvents instead of sending to PostHog
*/
public capture(event: { event: string; properties?: any }, collect: boolean = false): void {
public capture(event: { event: string; properties?: any }): void {
if (!this.telemetryEnabled) {
return
}
const taskId = event.properties.taskId
const propertiesWithVersion = this.addProperties(event.properties)
@@ -214,19 +198,7 @@ class TelemetryService {
properties: propertiesWithVersion,
}
if (collect && taskId) {
const existingTask = this.collectedTasks.find((task) => task.taskId === taskId)
if (existingTask) {
existingTask.collection.push(capturedEvent)
} else {
this.collectedTasks.push({
taskId,
collection: [capturedEvent],
})
}
} else {
this.client.capture({ ...capturedEvent, distinctId: this.distinctId })
}
this.client.capture({ ...capturedEvent, distinctId: this.distinctId })
}
public captureExtensionActivated(installId: string) {
@@ -243,47 +215,35 @@ class TelemetryService {
* Records when a new task/conversation is started
* @param taskId Unique identifier for the new task
* @param apiProvider Optional API provider
* @param collect If true, collect event instead of sending
*/
public captureTaskCreated(taskId: string, apiProvider?: string, collect: boolean = false) {
this.capture(
{
event: TelemetryService.EVENTS.TASK.CREATED,
properties: { taskId, apiProvider },
},
collect,
)
public captureTaskCreated(taskId: string, apiProvider?: string) {
this.capture({
event: TelemetryService.EVENTS.TASK.CREATED,
properties: { taskId, apiProvider },
})
}
/**
* Records when a task/conversation is restarted
* @param taskId Unique identifier for the new task
* @param apiProvider Optional API provider
* @param collect If true, collect event instead of sending
*/
public captureTaskRestarted(taskId: string, apiProvider?: string, collect: boolean = false) {
this.capture(
{
event: TelemetryService.EVENTS.TASK.RESTARTED,
properties: { taskId, apiProvider },
},
collect,
)
public captureTaskRestarted(taskId: string, apiProvider?: string) {
this.capture({
event: TelemetryService.EVENTS.TASK.RESTARTED,
properties: { taskId, apiProvider },
})
}
/**
* Records when cline calls the task completion_result tool signifying that cline is done with the task
* @param taskId Unique identifier for the task
* @param collect If true, collect event instead of sending
*/
public captureTaskCompleted(taskId: string, collect: boolean = false) {
this.capture(
{
event: TelemetryService.EVENTS.TASK.COMPLETED,
properties: { taskId },
},
collect,
)
public captureTaskCompleted(taskId: string) {
this.capture({
event: TelemetryService.EVENTS.TASK.COMPLETED,
properties: { taskId },
})
}
/**
@@ -292,7 +252,6 @@ class TelemetryService {
* @param provider The API provider (e.g., OpenAI, Anthropic)
* @param model The specific model used (e.g., GPT-4, Claude)
* @param source The source of the message ("user" | "model"). Used to track message patterns and identify when users need to correct the model's responses.
* @param collect If true, collect event instead of sending
* @param tokenUsage Optional token usage data
*/
public captureConversationTurnEvent(
@@ -300,7 +259,6 @@ class TelemetryService {
provider: string = "unknown",
model: string = "unknown",
source: "user" | "assistant",
collect: boolean = false,
tokenUsage: {
tokensIn?: number
tokensOut?: number
@@ -324,13 +282,10 @@ class TelemetryService {
...tokenUsage,
}
this.capture(
{
event: TelemetryService.EVENTS.TASK.CONVERSATION_TURN,
properties,
},
collect,
)
this.capture({
event: TelemetryService.EVENTS.TASK.CONVERSATION_TURN,
properties,
})
}
/**
@@ -340,19 +295,16 @@ class TelemetryService {
* @param tokensOut Number of output tokens generated
* @param model The model used for token calculation
*/
public captureTokenUsage(taskId: string, tokensIn: number, tokensOut: number, model: string, collect: boolean = false) {
this.capture(
{
event: TelemetryService.EVENTS.TASK.TOKEN_USAGE,
properties: {
taskId,
tokensIn,
tokensOut,
model,
},
public captureTokenUsage(taskId: string, tokensIn: number, tokensOut: number, model: string) {
this.capture({
event: TelemetryService.EVENTS.TASK.TOKEN_USAGE,
properties: {
taskId,
tokensIn,
tokensOut,
model,
},
collect,
)
})
}
/**
@@ -360,17 +312,14 @@ class TelemetryService {
* @param taskId Unique identifier for the task
* @param mode The mode being switched to (plan or act)
*/
public captureModeSwitch(taskId: string, mode: "plan" | "act", collect: boolean = false) {
this.capture(
{
event: TelemetryService.EVENTS.TASK.MODE_SWITCH,
properties: {
taskId,
mode,
},
public captureModeSwitch(taskId: string, mode: "plan" | "act") {
this.capture({
event: TelemetryService.EVENTS.TASK.MODE_SWITCH,
properties: {
taskId,
mode,
},
collect,
)
})
}
/**
@@ -378,18 +327,15 @@ class TelemetryService {
* @param taskId Unique identifier for the task
* @param feedbackType The type of feedback ("thumbs_up" or "thumbs_down")
*/
public captureTaskFeedback(taskId: string, feedbackType: TaskFeedbackType, collect: boolean = false) {
public captureTaskFeedback(taskId: string, feedbackType: TaskFeedbackType) {
console.info("TelemetryService: Capturing task feedback", { taskId, feedbackType })
this.capture(
{
event: TelemetryService.EVENTS.TASK.FEEDBACK,
properties: {
taskId,
feedbackType,
},
this.capture({
event: TelemetryService.EVENTS.TASK.FEEDBACK,
properties: {
taskId,
feedbackType,
},
collect,
)
})
}
// Tool events
@@ -400,27 +346,17 @@ class TelemetryService {
* @param autoApproved Whether the tool was auto-approved based on settings
* @param success Whether the tool execution was successful
*/
public captureToolUsage(
taskId: string,
tool: string,
modelId: string,
autoApproved: boolean,
success: boolean,
collect: boolean = false,
) {
this.capture(
{
event: TelemetryService.EVENTS.TASK.TOOL_USED,
properties: {
taskId,
tool,
autoApproved,
success,
modelId,
},
public captureToolUsage(taskId: string, tool: string, modelId: string, autoApproved: boolean, success: boolean) {
this.capture({
event: TelemetryService.EVENTS.TASK.TOOL_USED,
properties: {
taskId,
tool,
autoApproved,
success,
modelId,
},
collect,
)
})
}
/**
@@ -433,23 +369,19 @@ class TelemetryService {
taskId: string,
action: "shadow_git_initialized" | "commit_created" | "restored" | "diff_generated",
durationMs?: number,
collect: boolean = false,
) {
if (!this.isCategoryEnabled("checkpoints")) {
return
}
this.capture(
{
event: TelemetryService.EVENTS.TASK.CHECKPOINT_USED,
properties: {
taskId,
action,
durationMs,
},
this.capture({
event: TelemetryService.EVENTS.TASK.CHECKPOINT_USED,
properties: {
taskId,
action,
durationMs,
},
collect,
)
})
}
/**
@@ -457,18 +389,15 @@ class TelemetryService {
* @param taskId Unique identifier for the task
* @param errorType Type of error that occurred (e.g., "search_not_found", "invalid_format")
*/
public captureDiffEditFailure(taskId: string, modelId: string, errorType?: string, collect: boolean = false) {
this.capture(
{
event: TelemetryService.EVENTS.TASK.DIFF_EDIT_FAILED,
properties: {
taskId,
errorType,
modelId,
},
public captureDiffEditFailure(taskId: string, modelId: string, errorType?: string) {
this.capture({
event: TelemetryService.EVENTS.TASK.DIFF_EDIT_FAILED,
properties: {
taskId,
errorType,
modelId,
},
collect,
)
})
}
/**
@@ -477,50 +406,41 @@ class TelemetryService {
* @param provider Provider of the selected model
* @param taskId Optional task identifier if model was selected during a task
*/
public captureModelSelected(model: string, provider: string, taskId?: string, collect: boolean = false) {
this.capture(
{
event: TelemetryService.EVENTS.UI.MODEL_SELECTED,
properties: {
model,
provider,
taskId,
},
public captureModelSelected(model: string, provider: string, taskId?: string) {
this.capture({
event: TelemetryService.EVENTS.UI.MODEL_SELECTED,
properties: {
model,
provider,
taskId,
},
collect,
)
})
}
/**
* Records when a historical task is loaded from storage
* @param taskId Unique identifier for the historical task
*/
public captureHistoricalTaskLoaded(taskId: string, collect: boolean = false) {
this.capture(
{
event: TelemetryService.EVENTS.TASK.HISTORICAL_LOADED,
properties: {
taskId,
},
public captureHistoricalTaskLoaded(taskId: string) {
this.capture({
event: TelemetryService.EVENTS.TASK.HISTORICAL_LOADED,
properties: {
taskId,
},
collect,
)
})
}
/**
* Records when the retry button is clicked for failed operations
* @param taskId Unique identifier for the task being retried
*/
public captureRetryClicked(taskId: string, collect: boolean = false) {
this.capture(
{
event: TelemetryService.EVENTS.TASK.RETRY_CLICKED,
properties: {
taskId,
},
public captureRetryClicked(taskId: string) {
this.capture({
event: TelemetryService.EVENTS.TASK.RETRY_CLICKED,
properties: {
taskId,
},
collect,
)
})
}
/**
@@ -528,24 +448,21 @@ class TelemetryService {
* @param taskId Unique identifier for the task
* @param browserSettings The browser settings being used
*/
public captureBrowserToolStart(taskId: string, browserSettings: BrowserSettings, collect: boolean = false) {
public captureBrowserToolStart(taskId: string, browserSettings: BrowserSettings) {
if (!this.isCategoryEnabled("browser")) {
return
}
this.capture(
{
event: TelemetryService.EVENTS.TASK.BROWSER_TOOL_START,
properties: {
taskId,
viewport: browserSettings.viewport,
isRemote: !!browserSettings.remoteBrowserEnabled,
remoteBrowserHost: browserSettings.remoteBrowserHost,
timestamp: new Date().toISOString(),
},
this.capture({
event: TelemetryService.EVENTS.TASK.BROWSER_TOOL_START,
properties: {
taskId,
viewport: browserSettings.viewport,
isRemote: !!browserSettings.remoteBrowserEnabled,
remoteBrowserHost: browserSettings.remoteBrowserHost,
timestamp: new Date().toISOString(),
},
collect,
)
})
}
/**
@@ -560,25 +477,21 @@ class TelemetryService {
duration: number
actions?: string[]
},
collect: boolean = false,
) {
if (!this.isCategoryEnabled("browser")) {
return
}
this.capture(
{
event: TelemetryService.EVENTS.TASK.BROWSER_TOOL_END,
properties: {
taskId,
actionCount: stats.actionCount,
duration: stats.duration,
actions: stats.actions,
timestamp: new Date().toISOString(),
},
this.capture({
event: TelemetryService.EVENTS.TASK.BROWSER_TOOL_END,
properties: {
taskId,
actionCount: stats.actionCount,
duration: stats.duration,
actions: stats.actions,
timestamp: new Date().toISOString(),
},
collect,
)
})
}
/**
@@ -598,25 +511,21 @@ class TelemetryService {
isRemote?: boolean
[key: string]: any
},
collect: boolean = false,
) {
if (!this.isCategoryEnabled("browser")) {
return
}
this.capture(
{
event: TelemetryService.EVENTS.TASK.BROWSER_ERROR,
properties: {
taskId,
errorType,
errorMessage,
context,
timestamp: new Date().toISOString(),
},
this.capture({
event: TelemetryService.EVENTS.TASK.BROWSER_ERROR,
properties: {
taskId,
errorType,
errorMessage,
context,
timestamp: new Date().toISOString(),
},
collect,
)
})
}
/**
@@ -625,18 +534,15 @@ class TelemetryService {
* @param qty The quantity of options that were presented
* @param mode The mode in which the option was selected ("plan" or "act")
*/
public captureOptionSelected(taskId: string, qty: number, mode: "plan" | "act", collect: boolean = false) {
this.capture(
{
event: TelemetryService.EVENTS.TASK.OPTION_SELECTED,
properties: {
taskId,
qty,
mode,
},
public captureOptionSelected(taskId: string, qty: number, mode: "plan" | "act") {
this.capture({
event: TelemetryService.EVENTS.TASK.OPTION_SELECTED,
properties: {
taskId,
qty,
mode,
},
collect,
)
})
}
/**
@@ -645,18 +551,15 @@ class TelemetryService {
* @param qty The quantity of options that were presented
* @param mode The mode in which the custom response was provided ("plan" or "act")
*/
public captureOptionsIgnored(taskId: string, qty: number, mode: "plan" | "act", collect: boolean = false) {
this.capture(
{
event: TelemetryService.EVENTS.TASK.OPTIONS_IGNORED,
properties: {
taskId,
qty,
mode,
},
public captureOptionsIgnored(taskId: string, qty: number, mode: "plan" | "act") {
this.capture({
event: TelemetryService.EVENTS.TASK.OPTIONS_IGNORED,
properties: {
taskId,
qty,
mode,
},
collect,
)
})
}
/**
@@ -664,7 +567,6 @@ class TelemetryService {
* @param taskId Unique identifier for the task
* @param modelId Specific Gemini model ID
* @param data Performance data including TTFT, durations, token counts, cache stats, and API success status
* @param collect If true, collect event instead of sending
*/
public captureGeminiApiPerformance(
taskId: string,
@@ -681,19 +583,15 @@ class TelemetryService {
apiError?: string
throughputTokensPerSec?: number
},
collect: boolean = false,
) {
this.capture(
{
event: TelemetryService.EVENTS.TASK.GEMINI_API_PERFORMANCE,
properties: {
taskId,
modelId,
...data,
},
this.capture({
event: TelemetryService.EVENTS.TASK.GEMINI_API_PERFORMANCE,
properties: {
taskId,
modelId,
...data,
},
collect,
)
})
}
/**
@@ -701,30 +599,24 @@ class TelemetryService {
* @param model The name of the model the user has interacted with
* @param isFavorited Whether the model is being favorited (true) or unfavorited (false)
*/
public captureModelFavoritesUsage(model: string, isFavorited: boolean, collect: boolean = false) {
this.capture(
{
event: TelemetryService.EVENTS.UI.MODEL_FAVORITE_TOGGLED,
properties: {
model,
isFavorited,
},
public captureModelFavoritesUsage(model: string, isFavorited: boolean) {
this.capture({
event: TelemetryService.EVENTS.UI.MODEL_FAVORITE_TOGGLED,
properties: {
model,
isFavorited,
},
collect,
)
})
}
public captureButtonClick(button: string, taskId?: string, collect: boolean = false) {
this.capture(
{
event: TelemetryService.EVENTS.UI.BUTTON_CLICKED,
properties: {
button,
taskId,
},
public captureButtonClick(button: string, taskId?: string) {
this.capture({
event: TelemetryService.EVENTS.UI.BUTTON_CLICKED,
properties: {
button,
taskId,
},
collect,
)
})
}
/**
@@ -736,27 +628,21 @@ class TelemetryService {
* @param errorStatus HTTP status code of the error response, if available
* @param collect Optional flag to determine if the event should be collected for batch sending
*/
public captureProviderApiError(
args: {
taskId: string
model: string
errorMessage: string
errorStatus?: number | undefined
requestId?: string | undefined
},
collect: boolean = true,
) {
this.capture(
{
event: TelemetryService.EVENTS.TASK.PROVIDER_API_ERROR,
properties: {
...args,
errorMessage: args.errorMessage.substring(0, MAX_ERROR_MESSAGE_LENGTH), // Truncate long error messages
timestamp: new Date().toISOString(),
},
public captureProviderApiError(args: {
taskId: string
model: string
errorMessage: string
errorStatus?: number | undefined
requestId?: string | undefined
}) {
this.capture({
event: TelemetryService.EVENTS.TASK.PROVIDER_API_ERROR,
properties: {
...args,
errorMessage: args.errorMessage.substring(0, MAX_ERROR_MESSAGE_LENGTH), // Truncate long error messages
timestamp: new Date().toISOString(),
},
collect,
)
})
}
/**
@@ -777,39 +663,6 @@ class TelemetryService {
return this.telemetryCategoryEnabled.get(category) ?? true
}
public async sendCollectedEvents(taskId?: string): Promise<void> {
if (!this.telemetryEnabled) {
return
}
if (this.collectedTasks.length > 0) {
if (taskId) {
const task = this.collectedTasks.find((t) => t.taskId === taskId)
if (task) {
this.capture(
{
event: TelemetryService.EVENTS.TASK.TASK_COLLECTION,
properties: { taskId, events: task.collection },
},
false,
)
this.collectedTasks = this.collectedTasks.filter((t) => t.taskId !== taskId)
}
} else {
for (const task of this.collectedTasks) {
this.capture(
{
event: TelemetryService.EVENTS.TASK.TASK_COLLECTION,
properties: { taskId: task.taskId, events: task.collection },
},
false,
)
this.collectedTasks = this.collectedTasks.filter((t) => t.taskId !== task.taskId)
}
}
}
}
public async shutdown(): Promise<void> {
await this.client.shutdown()
}