Compare commits

...

1 Commits

Author SHA1 Message Date
Max Paulus 🥪 3e4b1b2431 update telemetry to match old telemetry schema
- maintain parity with old telemetry to make analysis on an even playing
field
2026-05-26 12:48:26 -07:00
12 changed files with 300 additions and 12 deletions
+18 -9
View File
@@ -195,6 +195,7 @@ interface PreparedToolExecution {
tool?: AgentTool;
input: unknown;
skipReason?: string;
autoApproved?: boolean;
}
interface HookBag {
@@ -1132,6 +1133,7 @@ export class AgentRuntime {
}
}
let autoApproved: boolean | undefined;
if (tool && !skipReason) {
const policy = resolveToolPolicy(
toolCall.toolName,
@@ -1139,15 +1141,18 @@ export class AgentRuntime {
);
if (policy.enabled === false) {
skipReason = `Tool "${toolCall.toolName}" is disabled by policy`;
} else if (policy.autoApprove === false) {
const approval = await this.requestToolApproval(
toolCall,
input,
policy,
);
if (!approval.approved) {
skipReason =
approval.reason ?? `Tool "${toolCall.toolName}" was not approved`;
} else {
autoApproved = policy.autoApprove;
if (policy.autoApprove === false) {
const approval = await this.requestToolApproval(
toolCall,
input,
policy,
);
if (!approval.approved) {
skipReason =
approval.reason ?? `Tool "${toolCall.toolName}" was not approved`;
}
}
}
}
@@ -1157,6 +1162,7 @@ export class AgentRuntime {
tool,
input,
skipReason,
autoApproved,
};
}
@@ -1293,6 +1299,9 @@ export class AgentRuntime {
iteration: this.state.iteration,
toolCall: prepared.toolCall,
message,
...(prepared.autoApproved !== undefined
? { autoApproved: prepared.autoApproved }
: {}),
});
return message;
@@ -298,6 +298,7 @@ export class RuntimeEventAdapter {
private translateToolFinished(event: {
toolCall: { toolCallId: string; toolName: string };
message: AgentMessage;
autoApproved?: boolean;
}): AgentEvent[] {
const startedAt = this.toolStartedAt.get(event.toolCall.toolCallId);
const durationMs =
@@ -315,6 +316,9 @@ export class RuntimeEventAdapter {
output,
error,
durationMs,
...(event.autoApproved !== undefined
? { autoApproved: event.autoApproved }
: {}),
},
];
}
@@ -0,0 +1,113 @@
import type { ITelemetryService } from "@cline/shared";
import { describe, expect, test, vi } from "vitest";
import { type AgentEventContext, handleAgentEvent } from "./agent-events";
import { createInitialAccumulatedUsage } from "./usage";
function createTelemetryStub() {
const capture = vi.fn();
const telemetry = {
capture,
captureRequired: vi.fn(),
setDistinctId: vi.fn(),
setMetadata: vi.fn(),
updateMetadata: vi.fn(),
setCommonProperties: vi.fn(),
updateCommonProperties: vi.fn(),
isEnabled: vi.fn(() => true),
recordCounter: vi.fn(),
recordHistogram: vi.fn(),
recordGauge: vi.fn(),
flush: vi.fn(async () => {}),
dispose: vi.fn(async () => {}),
} satisfies ITelemetryService;
return { telemetry, capture };
}
function createContext(telemetry: ITelemetryService): AgentEventContext {
return {
sessionId: "task-1",
config: {
providerId: "cline",
modelId: "anthropic/claude-sonnet-4.6",
mode: "act",
telemetry,
} as AgentEventContext["config"],
liveSession: undefined,
usageBySession: new Map(),
aggregateUsageBySession: new Map(),
persistMessages: vi.fn(),
emit: vi.fn(),
};
}
function captureEvents(capture: ReturnType<typeof vi.fn>) {
return capture.mock.calls.map(
([arg]) => arg as { event: string; properties?: Record<string, unknown> },
);
}
describe("handleAgentEvent telemetry compatibility", () => {
test("emits task.tool_used with provider, model, and known autoApproved state", () => {
const stub = createTelemetryStub();
const ctx = createContext(stub.telemetry);
handleAgentEvent(ctx, {
type: "content_end",
contentType: "tool",
toolName: "read_files",
toolCallId: "call-1",
autoApproved: true,
});
expect(captureEvents(stub.capture)).toContainEqual({
event: "task.tool_used",
properties: expect.objectContaining({
ulid: "task-1",
tool: "read_files",
modelId: "anthropic/claude-sonnet-4.6",
provider: "cline",
autoApproved: true,
success: true,
}),
});
});
test("emits task.tokens with provider and model attribution", () => {
const stub = createTelemetryStub();
const ctx = createContext(stub.telemetry);
ctx.liveSession = {
runtime: {},
turnUsageBaseline: createInitialAccumulatedUsage(),
turnAggregateUsageBaseline: createInitialAccumulatedUsage(),
turnPrimaryUsage: createInitialAccumulatedUsage(),
} as AgentEventContext["liveSession"];
handleAgentEvent(ctx, {
type: "usage",
inputTokens: 100,
outputTokens: 25,
cacheReadTokens: 10,
cacheWriteTokens: 0,
cost: 0.01,
totalInputTokens: 100,
totalOutputTokens: 25,
totalCacheReadTokens: 10,
totalCacheWriteTokens: 0,
totalCost: 0.01,
});
expect(captureEvents(stub.capture)).toContainEqual({
event: "task.tokens",
properties: expect.objectContaining({
ulid: "task-1",
provider: "cline",
model: "anthropic/claude-sonnet-4.6",
tokensIn: 100,
tokensOut: 25,
cacheReadTokens: 10,
cacheWriteTokens: 0,
totalCost: 0.01,
}),
});
});
});
@@ -191,10 +191,11 @@ export function handleAgentEvent(
captureToolUsage(telemetry, {
ulid: sessionId,
tool: toolName,
autoApproved: undefined,
autoApproved: event.autoApproved,
success,
modelId: config.modelId,
provider: config.providerId,
isNativeToolCall: event.isNativeToolCall,
...agentIdentity,
});
if (!success && (toolName === "editor" || toolName === "apply_patch")) {
@@ -250,6 +251,7 @@ export function handleAgentEvent(
});
captureTokenUsage(telemetry, {
ulid: sessionId,
provider: config.providerId,
tokensIn: event.inputTokens,
tokensOut: event.outputTokens,
cacheWriteTokens: event.cacheWriteTokens,
@@ -324,6 +324,7 @@ export async function prepareLocalRuntimeBootstrap(
durationMs,
initError,
featureFlagEnabled: true,
isRemoteWorkspace: extensionContext.workspace?.isRemoteWorkspace,
});
const fileHookExtension = createHookConfigFileExtension({
@@ -24,6 +24,7 @@ describe("OpenTelemetryAdapter", () => {
},
items: ["a", "b"],
nullable: null,
optional: undefined,
});
expect(emit).toHaveBeenCalledWith({
@@ -33,7 +34,6 @@ describe("OpenTelemetryAdapter", () => {
ulid: "01HXYZ",
"nested.mode": "act",
items: JSON.stringify(["a", "b"]),
nullable: "null",
distinct_id: "user-123",
organization_id: "org-1",
extension_version: "1.2.3",
@@ -41,6 +41,44 @@ describe("OpenTelemetryAdapter", () => {
platform: "terminal",
}),
});
const attributes = emit.mock.calls[0]?.[0].attributes as Record<
string,
unknown
>;
expect(attributes).not.toHaveProperty("nullable");
expect(attributes).not.toHaveProperty("optional");
expect(Object.values(attributes)).not.toContain("undefined");
});
it("omits unset optional telemetry fields instead of serializing undefined", () => {
const emit = vi.fn();
const adapter = new OpenTelemetryAdapter({
metadata: makeMetadata(),
loggerProvider: {
getLogger: () => ({ emit }),
} as unknown as LoggerProvider,
});
adapter.emit("task.tool_used", {
ulid: "01HXYZ",
tool: "read_files",
autoApproved: undefined,
teamId: undefined,
parentAgentId: undefined,
});
const attributes = emit.mock.calls[0]?.[0].attributes as Record<
string,
unknown
>;
expect(attributes).toMatchObject({
ulid: "01HXYZ",
tool: "read_files",
});
expect(attributes).not.toHaveProperty("autoApproved");
expect(attributes).not.toHaveProperty("teamId");
expect(attributes).not.toHaveProperty("parentAgentId");
expect(Object.values(attributes)).not.toContain("undefined");
});
it("marks required events with the expected flag", () => {
@@ -271,7 +271,6 @@ export class OpenTelemetryAdapter implements ITelemetryAdapter {
const fullKey = prefix ? `${prefix}.${key}` : key;
if (value === null || value === undefined) {
flattened[fullKey] = String(value);
continue;
}
@@ -7,9 +7,12 @@ import {
captureExtensionActivated,
captureProviderConfigured,
captureTelemetryOptOut,
captureTokenUsage,
captureToolUsage,
captureWorkspaceInitError,
captureWorkspaceInitialized,
captureWorkspacePathResolved,
identifyAccount,
} from "./core-events";
import type { ITelemetryAdapter } from "./ITelemetryAdapter";
import { TelemetryService } from "./TelemetryService";
@@ -219,6 +222,113 @@ describe("captureWorkspaceInitError", () => {
});
});
describe("task telemetry compatibility", () => {
test("captureTokenUsage includes provider and model attribution", () => {
const stub = createTelemetryStub();
captureTokenUsage(stub.telemetry, {
ulid: "task-1",
provider: "openrouter",
model: "anthropic/claude-sonnet-4.6",
tokensIn: 123,
tokensOut: 45,
cacheReadTokens: 10,
cacheWriteTokens: 0,
totalCost: 0.0123,
});
const { event, properties } = captureCallAt(stub, 0);
expect(event).toBe("task.tokens");
expect(properties).toMatchObject({
ulid: "task-1",
provider: "openrouter",
model: "anthropic/claude-sonnet-4.6",
tokensIn: 123,
tokensOut: 45,
cacheReadTokens: 10,
cacheWriteTokens: 0,
totalCost: 0.0123,
});
});
test("captureToolUsage emits boolean autoApproved when known", () => {
const stub = createTelemetryStub();
captureToolUsage(stub.telemetry, {
ulid: "task-1",
tool: "read_files",
modelId: "anthropic/claude-sonnet-4.6",
provider: "cline",
autoApproved: true,
success: true,
});
const { event, properties } = captureCallAt(stub, 0);
expect(event).toBe("task.tool_used");
expect(properties).toMatchObject({
tool: "read_files",
provider: "cline",
modelId: "anthropic/claude-sonnet-4.6",
autoApproved: true,
success: true,
});
});
test("captureToolUsage omits autoApproved when unknown", () => {
const stub = createTelemetryStub();
captureToolUsage(stub.telemetry, {
ulid: "task-1",
tool: "read_files",
provider: "cline",
success: true,
});
const { properties } = captureCallAt(stub, 0);
expect(properties).not.toHaveProperty("autoApproved");
});
});
describe("identifyAccount", () => {
test("sets legacy user fields alongside SDK account fields", () => {
const stub = createTelemetryStub();
identifyAccount(stub.telemetry, {
id: "usr-1",
email: "user@example.com",
displayName: "User One",
provider: "cline",
organizationId: "org-1",
organizationName: "Acme",
memberId: "mem-1",
});
expect(stub.telemetry.setDistinctId).toHaveBeenCalledWith("usr-1");
expect(stub.telemetry.updateCommonProperties).toHaveBeenCalledWith({
user_id: "usr-1",
user_name: "User One",
account_id: "usr-1",
account_email: "user@example.com",
provider: "cline",
organization_id: "org-1",
organization_name: "Acme",
member_id: "mem-1",
});
});
test("defaults legacy user_name to an empty string when displayName is unavailable", () => {
const stub = createTelemetryStub();
identifyAccount(stub.telemetry, {
id: "usr-1",
provider: "cline",
});
expect(stub.telemetry.updateCommonProperties).toHaveBeenCalledWith(
expect.objectContaining({
user_id: "usr-1",
user_name: "",
account_id: "usr-1",
}),
);
});
});
describe("captureWorkspacePathResolved", () => {
test("emits workspace.path_resolved with snake_case fields", () => {
const stub = createTelemetryStub();
@@ -254,6 +254,7 @@ export function identifyAccount(
account: {
id?: string;
email?: string;
displayName?: string;
provider?: string;
organizationId?: string;
organizationName?: string;
@@ -265,6 +266,8 @@ export function identifyAccount(
telemetry?.setDistinctId(distinctId);
}
telemetry?.updateCommonProperties({
user_id: account.id,
user_name: account.displayName ?? "",
account_id: account.id,
account_email: account.email,
provider: account.provider,
@@ -343,6 +346,7 @@ export function captureTokenUsage(
telemetry: ITelemetryService | undefined,
properties: {
ulid: string;
provider?: string;
tokensIn: number;
tokensOut: number;
cacheWriteTokens?: number;
@@ -371,6 +375,7 @@ export function captureToolUsage(
provider?: string;
autoApproved?: boolean;
success: boolean;
isNativeToolCall?: boolean;
} & Partial<TelemetryAgentIdentityProperties>,
): void {
emit(telemetry, CORE_TELEMETRY_EVENTS.TASK.TOOL_USED, properties);
+1
View File
@@ -515,6 +515,7 @@ export type AgentRuntimeEvent =
iteration: number;
toolCall: AgentToolCallPart;
message: AgentMessage;
autoApproved?: boolean;
}
| {
type: "usage-updated";
+4
View File
@@ -112,6 +112,10 @@ export interface AgentContentEndEvent extends AgentEventMetadata {
error?: string;
/** Time taken in milliseconds for tool content */
durationMs?: number;
/** Whether the completed tool call ran without a user approval prompt */
autoApproved?: boolean;
/** Whether this event came from a provider-native tool call */
isNativeToolCall?: boolean;
}
export interface AgentIterationStartEvent extends AgentEventMetadata {
@@ -67,6 +67,8 @@ export interface WorkspaceContext extends WorkspaceInfo {
ide?: string;
/** Node process.platform string, e.g. "darwin", "win32", "linux" */
platform?: string;
/** Whether the workspace is hosted through a remote IDE connection */
isRemoteWorkspace?: boolean;
}
/**