From 0bf14eb2ff5ef59f9dc98342218addc670a87481 Mon Sep 17 00:00:00 2001 From: "kiloconnect[bot]" <240665456+kiloconnect[bot]@users.noreply.github.com> Date: Wed, 29 Apr 2026 08:52:51 +0000 Subject: [PATCH] chore(telemetry): remove ai.* and gen_ai.* span emission Drops the PostHogSpanExporter that converted AI SDK OpenTelemetry spans into PostHog ai.* / gen_ai.* events, and disables AI SDK experimental_telemetry in the CLI so those spans are no longer emitted. Tool/session/indexing telemetry is unchanged. --- .changeset/remove-ai-gen_ai-telemetry.md | 6 + bun.lock | 6 - packages/kilo-telemetry/package.json | 8 +- .../src/__tests__/telemetry.test.ts | 127 ---------- packages/kilo-telemetry/src/index.ts | 2 - packages/kilo-telemetry/src/otel-exporter.ts | 231 ------------------ packages/kilo-telemetry/src/telemetry.ts | 22 -- packages/kilo-telemetry/src/tracer.ts | 71 ------ packages/opencode/src/kilocode/agent/index.ts | 14 +- packages/opencode/src/session/llm.ts | 11 +- 10 files changed, 12 insertions(+), 486 deletions(-) create mode 100644 .changeset/remove-ai-gen_ai-telemetry.md delete mode 100644 packages/kilo-telemetry/src/otel-exporter.ts delete mode 100644 packages/kilo-telemetry/src/tracer.ts diff --git a/.changeset/remove-ai-gen_ai-telemetry.md b/.changeset/remove-ai-gen_ai-telemetry.md new file mode 100644 index 00000000000..5479a8bab1c --- /dev/null +++ b/.changeset/remove-ai-gen_ai-telemetry.md @@ -0,0 +1,6 @@ +--- +"kilo-code": patch +"@kilocode/cli": patch +--- + +Stop emitting `ai.*` and `gen_ai.*` OpenTelemetry spans from AI SDK calls, and remove the PostHog bridge that forwarded them. Tool/session/indexing telemetry is unchanged. diff --git a/bun.lock b/bun.lock index cf6fa323bd1..9f017ccf39f 100644 --- a/bun.lock +++ b/bun.lock @@ -285,12 +285,6 @@ "version": "7.2.26", "dependencies": { "@kilocode/kilo-gateway": "workspace:*", - "@opentelemetry/api": "1.9.0", - "@opentelemetry/core": "2.6.1", - "@opentelemetry/resources": "2.6.1", - "@opentelemetry/sdk-trace-base": "2.6.1", - "@opentelemetry/sdk-trace-node": "2.6.1", - "@opentelemetry/semantic-conventions": "1.40.0", "posthog-node": "4.4.0", }, "devDependencies": { diff --git a/packages/kilo-telemetry/package.json b/packages/kilo-telemetry/package.json index 69aa0259dc9..46ec4c39d81 100644 --- a/packages/kilo-telemetry/package.json +++ b/packages/kilo-telemetry/package.json @@ -18,13 +18,7 @@ }, "dependencies": { "posthog-node": "4.4.0", - "@kilocode/kilo-gateway": "workspace:*", - "@opentelemetry/api": "1.9.0", - "@opentelemetry/core": "2.6.1", - "@opentelemetry/sdk-trace-base": "2.6.1", - "@opentelemetry/sdk-trace-node": "2.6.1", - "@opentelemetry/resources": "2.6.1", - "@opentelemetry/semantic-conventions": "1.40.0" + "@kilocode/kilo-gateway": "workspace:*" }, "devDependencies": { "@tsconfig/bun": "catalog:", diff --git a/packages/kilo-telemetry/src/__tests__/telemetry.test.ts b/packages/kilo-telemetry/src/__tests__/telemetry.test.ts index 38470f03eb7..4c3c62714ad 100644 --- a/packages/kilo-telemetry/src/__tests__/telemetry.test.ts +++ b/packages/kilo-telemetry/src/__tests__/telemetry.test.ts @@ -2,21 +2,6 @@ import { describe, test, expect, beforeEach } from "bun:test" import { Identity } from "../identity.js" import { TelemetryEvent } from "../events.js" import { Telemetry } from "../telemetry.js" -import { PostHogSpanExporter } from "../otel-exporter.js" -import { ExportResultCode } from "@opentelemetry/core" -import type { ReadableSpan } from "@opentelemetry/sdk-trace-base" -import type { PostHog } from "posthog-node" - -function createMockPostHogClient(): PostHog { - return { - capture: () => {}, - alias: () => {}, - flush: async () => {}, - shutdown: async () => {}, - optIn: () => {}, - optOut: () => {}, - } as unknown as PostHog -} describe("Identity", () => { beforeEach(() => { @@ -104,115 +89,3 @@ describe("Telemetry", () => { expect(typeof Telemetry.trackIndexingError).toBe("function") }) }) - -describe("PostHogSpanExporter", () => { - function createMockSpan(name: string, attributes: Record): ReadableSpan { - return { - name, - attributes, - spanContext: () => ({ - traceId: "trace-123", - spanId: "span-456", - traceFlags: 1, - }), - parentSpanContext: undefined, - startTime: [1000, 0], - endTime: [1001, 0], - status: { code: 0 }, - kind: 0, - resource: { attributes: {} }, - instrumentationScope: { name: "test" }, - events: [], - links: [], - ended: true, - duration: [1, 0], - droppedAttributesCount: 0, - droppedEventsCount: 0, - droppedLinksCount: 0, - } as unknown as ReadableSpan - } - - test("export returns success when disabled", () => { - const exporter = new PostHogSpanExporter(createMockPostHogClient(), { - appName: "test", - appVersion: "1.0.0", - platform: "test", - }) - exporter.setEnabled(false) - - const span = createMockSpan("ai.generateText", { "ai.model.id": "gpt-4" }) - let result: { code: number } | null = null - - exporter.export([span], (r) => { - result = r - }) - - expect(result).not.toBeNull() - expect(result!.code).toBe(ExportResultCode.SUCCESS) - }) - - test("sensitive attributes are not included in exported properties", () => { - // This test verifies the filtering logic by checking the SENSITIVE_ATTRIBUTES set - // and the mapAttributes method behavior through the export function - const exporter = new PostHogSpanExporter(createMockPostHogClient(), { - appName: "test", - appVersion: "1.0.0", - platform: "test", - }) - - // Create a span with both safe and sensitive attributes - const span = createMockSpan("ai.generateText", { - // Safe attributes (should be passed through) - "ai.model.id": "gpt-4", - "ai.model.provider": "openai", - "gen_ai.request.model": "gpt-4", - "gen_ai.usage.input_tokens": 100, - "gen_ai.usage.output_tokens": 50, - // Sensitive attributes (should be filtered) - "ai.prompt": '{"messages": [{"role": "user", "content": "secret data"}]}', - "ai.prompt.messages": '[{"role": "user", "content": "secret"}]', - "ai.response.text": "This is a secret response", - "ai.toolCall.args": '{"secret": "value"}', - "ai.toolCall.result": '{"result": "secret"}', - "gen_ai.prompt": "secret prompt", - "gen_ai.completion": "secret completion", - }) - - // The exporter should complete successfully - let result: { code: number } | null = null - exporter.export([span], (r) => { - result = r - }) - - expect(result).not.toBeNull() - expect(result!.code).toBe(ExportResultCode.SUCCESS) - }) - - test("SENSITIVE_ATTRIBUTES blocklist contains all required patterns", () => { - // Verify all sensitive attribute patterns are in the blocklist - const sensitivePatterns = [ - "ai.prompt", - "ai.prompt.messages", - "ai.response.text", - "ai.response.toolCalls", - "ai.toolCall.args", - "ai.toolCall.result", - "ai.value", - "ai.values", - "ai.embedding", - "ai.embeddings", - "ai.prompt.tools", - "gen_ai.prompt", - "gen_ai.completion", - "gen_ai.input.messages", - "gen_ai.output.messages", - "gen_ai.system_instructions", - "gen_ai.tool.definitions", - ] - - // Import the module to check the exported constant exists - // Since SENSITIVE_ATTRIBUTES is not exported, we verify through behavior - // by ensuring the exporter handles these attributes correctly - expect(sensitivePatterns.length).toBe(17) - }) -}) diff --git a/packages/kilo-telemetry/src/index.ts b/packages/kilo-telemetry/src/index.ts index efd9c211115..10f657fc46e 100644 --- a/packages/kilo-telemetry/src/index.ts +++ b/packages/kilo-telemetry/src/index.ts @@ -1,6 +1,4 @@ export { Telemetry } from "./telemetry.js" export { TelemetryEvent } from "./events.js" export { Identity } from "./identity.js" -export { TracerSetup } from "./tracer.js" -export { PostHogSpanExporter } from "./otel-exporter.js" export type { TelemetryProperties } from "./telemetry.js" diff --git a/packages/kilo-telemetry/src/otel-exporter.ts b/packages/kilo-telemetry/src/otel-exporter.ts deleted file mode 100644 index a672e76fcf0..00000000000 --- a/packages/kilo-telemetry/src/otel-exporter.ts +++ /dev/null @@ -1,231 +0,0 @@ -import type { SpanExporter, ReadableSpan } from "@opentelemetry/sdk-trace-base" -import type { ExportResult } from "@opentelemetry/core" -import { ExportResultCode } from "@opentelemetry/core" -import type { PostHog } from "posthog-node" -import { Identity } from "./identity.js" - -/** - * Sensitive attributes that should never be sent to PostHog. - * This is a defense-in-depth measure - the AI SDK should already filter these - * when recordInputs/recordOutputs are set to false. - */ -const SENSITIVE_ATTRIBUTES = new Set([ - // AI SDK prompt/message content - "ai.prompt", - "ai.prompt.messages", - "ai.response.text", - "ai.response.toolCalls", - "ai.toolCall.args", - "ai.toolCall.result", - "ai.value", - "ai.values", - "ai.embedding", - "ai.embeddings", - "ai.prompt.tools", - // OpenTelemetry GenAI semantic conventions - "gen_ai.prompt", - "gen_ai.completion", - "gen_ai.input.messages", - "gen_ai.output.messages", - "gen_ai.system_instructions", - "gen_ai.tool.definitions", -]) - -/** - * PostHogSpanExporter converts OpenTelemetry spans to PostHog AI events. - * Maps OTel spans to PostHog's $ai_span, $ai_generation, and $ai_trace events. - */ -export class PostHogSpanExporter implements SpanExporter { - private client: PostHog - private enabled = true - private appName: string - private appVersion: string - private platform: string - private editorName?: string - private vscodeVersion?: string - - constructor( - client: PostHog, - options: { appName: string; appVersion: string; platform: string; editorName?: string; vscodeVersion?: string }, - ) { - this.client = client - this.appName = options.appName - this.appVersion = options.appVersion - this.platform = options.platform - this.editorName = options.editorName - this.vscodeVersion = options.vscodeVersion - } - - setEnabled(value: boolean) { - this.enabled = value - if (value) this.client.optIn() - else this.client.optOut() - } - - export(spans: ReadableSpan[], resultCallback: (result: ExportResult) => void): void { - if (!this.enabled) { - resultCallback({ code: ExportResultCode.SUCCESS }) - return - } - - for (const span of spans) { - this.exportSpan(span) - } - - resultCallback({ code: ExportResultCode.SUCCESS }) - } - - private exportSpan(span: ReadableSpan) { - const distinctId = Identity.getDistinctId() - const orgId = Identity.getOrganizationId() - - const name = span.name - const attrs = span.attributes - const duration = span.endTime[0] - span.startTime[0] + (span.endTime[1] - span.startTime[1]) / 1e9 - - // Determine event type based on span name/attributes - const eventType = this.determineEventType(name, attrs) - - // Build PostHog AI event properties - const properties: Record = { - appName: this.appName, - appVersion: this.appVersion, - platform: this.platform, - ...(this.editorName && { editorName: this.editorName }), - ...(this.vscodeVersion && { vscodeVersion: this.vscodeVersion }), - $ai_trace_id: span.spanContext().traceId, - $ai_span_id: span.spanContext().spanId, - $ai_span_name: name, - $ai_latency: duration, - ...(orgId && { kilocodeOrganizationId: orgId }), - } - - // Add parent ID if present - const parentSpanId = span.parentSpanContext?.spanId - if (parentSpanId) { - properties.$ai_parent_id = parentSpanId - } - - // Map span attributes to PostHog properties - this.mapAttributes(attrs, properties) - - // Handle errors - if (span.status.code === 2) { - // SpanStatusCode.ERROR - properties.$ai_is_error = true - properties.$ai_error = span.status.message || "Unknown error" - } - - // Capture the event - this.client.capture({ - distinctId, - event: eventType, - properties, - }) - } - - private determineEventType(name: string, attrs: Record): string { - // AI SDK spans typically have these patterns: - // - "ai.generateText" / "ai.streamText" -> $ai_generation - // - "ai.toolCall" -> $ai_span (tool call) - // - "ai.embed" -> $ai_span (embedding) - - const lowerName = name.toLowerCase() - - if ( - lowerName.includes("generatetext") || - lowerName.includes("streamtext") || - lowerName.includes("generateobject") || - lowerName.includes("streamobject") - ) { - return "$ai_generation" - } - - if (lowerName.includes("toolcall") || lowerName.includes("tool")) { - return "$ai_span" - } - - if (lowerName.includes("embed")) { - return "$ai_span" - } - - // Check for gen_ai semantic conventions - if (attrs["gen_ai.operation.name"]) { - const op = String(attrs["gen_ai.operation.name"]) - if (op === "chat" || op === "text_completion") { - return "$ai_generation" - } - } - - // Default to span for other AI operations - return "$ai_span" - } - - private mapAttributes(attrs: Record, props: Record) { - // Map OpenTelemetry semantic conventions to PostHog AI properties - - // Model info - if (attrs["gen_ai.request.model"]) { - props.$ai_model = attrs["gen_ai.request.model"] - } - if (attrs["gen_ai.system"]) { - props.$ai_provider = attrs["gen_ai.system"] - } - - // Token usage - if (attrs["gen_ai.usage.input_tokens"]) { - props.$ai_input_tokens = attrs["gen_ai.usage.input_tokens"] - } - if (attrs["gen_ai.usage.output_tokens"]) { - props.$ai_output_tokens = attrs["gen_ai.usage.output_tokens"] - } - if (attrs["gen_ai.usage.total_tokens"]) { - props.$ai_total_tokens = attrs["gen_ai.usage.total_tokens"] - } - - // Track presence of prompt/completion without content (for analytics) - if (attrs["gen_ai.prompt"] || attrs["ai.prompt"]) { - props.$ai_has_prompt = true - } - if (attrs["gen_ai.completion"] || attrs["ai.response.text"]) { - props.$ai_has_completion = true - } - - // Tool calls - only track name, not args/result - if (attrs["ai.toolCall.name"]) { - props.$ai_tool_name = attrs["ai.toolCall.name"] - } - - // Temperature and other settings - if (attrs["gen_ai.request.temperature"]) { - props.$ai_temperature = attrs["gen_ai.request.temperature"] - } - if (attrs["gen_ai.request.max_tokens"]) { - props.$ai_max_tokens = attrs["gen_ai.request.max_tokens"] - } - - // Finish reason - if (attrs["gen_ai.response.finish_reasons"]) { - props.$ai_finish_reason = attrs["gen_ai.response.finish_reasons"] - } - - // Copy safe ai.* prefixed attributes (filter out sensitive ones) - for (const [key, value] of Object.entries(attrs)) { - if (!key.startsWith("ai.")) continue - if (SENSITIVE_ATTRIBUTES.has(key)) continue - // Additional pattern-based filtering for safety - if (key.includes("prompt") || key.includes("messages") || key.includes("response.text")) continue - const propKey = key.replace(/\./g, "_") - props[propKey] = value - } - } - - async shutdown(): Promise { - // Only flush, don't shutdown - the shared client is managed by Client namespace - await this.client.flush() - } - - async forceFlush(): Promise { - await this.client.flush() - } -} diff --git a/packages/kilo-telemetry/src/telemetry.ts b/packages/kilo-telemetry/src/telemetry.ts index 7a80c9e39c0..94a8d946ce8 100644 --- a/packages/kilo-telemetry/src/telemetry.ts +++ b/packages/kilo-telemetry/src/telemetry.ts @@ -1,8 +1,6 @@ import { Client } from "./client.js" import { Identity } from "./identity.js" import { TelemetryEvent } from "./events.js" -import { TracerSetup } from "./tracer.js" -import type { Tracer } from "@opentelemetry/api" export interface TelemetryProperties { appName: string @@ -83,16 +81,6 @@ export namespace Telemetry { const enabled = level ? level === "all" : options.enabled Client.setEnabled(enabled) - // Initialize OpenTelemetry tracer for AI SDK spans - TracerSetup.init({ - version: props.appVersion, - enabled, - appName: props.appName, - platform: props.platform, - editorName: props.editorName, - vscodeVersion: props.vscodeVersion, - }) - await Identity.getMachineId() initialized = true @@ -101,15 +89,6 @@ export namespace Telemetry { export function setEnabled(value: boolean) { Client.setEnabled(value) - TracerSetup.setEnabled(value) - } - - /** - * Get the OpenTelemetry tracer for use with AI SDK's experimental_telemetry. - * Returns null if telemetry is not initialized. - */ - export function getTracer(): Tracer | null { - return TracerSetup.getTracer() } export function isEnabled(): boolean { @@ -263,7 +242,6 @@ export namespace Telemetry { } export async function shutdown(): Promise { - await TracerSetup.shutdown() await Client.shutdown() } } diff --git a/packages/kilo-telemetry/src/tracer.ts b/packages/kilo-telemetry/src/tracer.ts deleted file mode 100644 index b49ef7e5375..00000000000 --- a/packages/kilo-telemetry/src/tracer.ts +++ /dev/null @@ -1,71 +0,0 @@ -import { NodeTracerProvider } from "@opentelemetry/sdk-trace-node" -import { SimpleSpanProcessor } from "@opentelemetry/sdk-trace-base" -import { resourceFromAttributes } from "@opentelemetry/resources" -import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from "@opentelemetry/semantic-conventions" -import type { Tracer } from "@opentelemetry/api" -import { PostHogSpanExporter } from "./otel-exporter.js" -import { Client } from "./client.js" - -let provider: NodeTracerProvider | null = null -let exporter: PostHogSpanExporter | null = null -let tracer: Tracer | null = null - -export namespace TracerSetup { - export function init(options: { - version: string - enabled: boolean - appName: string - platform: string - editorName?: string - vscodeVersion?: string - }): Tracer { - if (tracer) return tracer - - const client = Client.getClient() - if (!client) { - throw new Error("PostHog client not initialized. Call Client.init() first.") - } - - exporter = new PostHogSpanExporter(client, { - appName: options.appName, - appVersion: options.version, - platform: options.platform, - editorName: options.editorName, - vscodeVersion: options.vscodeVersion, - }) - exporter.setEnabled(options.enabled) - - provider = new NodeTracerProvider({ - resource: resourceFromAttributes({ - [ATTR_SERVICE_NAME]: options.appName, - [ATTR_SERVICE_VERSION]: options.version, - }), - spanProcessors: [new SimpleSpanProcessor(exporter)], - }) - - // Register the provider globally so all tracers use our exporter - provider.register() - - // Get tracer from our provider - tracer = provider.getTracer(options.appName, options.version) - - return tracer - } - - export function getTracer(): Tracer | null { - return tracer - } - - export function setEnabled(value: boolean) { - exporter?.setEnabled(value) - } - - export async function shutdown(): Promise { - if (provider) { - await provider.shutdown() - provider = null - tracer = null - exporter = null - } - } -} diff --git a/packages/opencode/src/kilocode/agent/index.ts b/packages/opencode/src/kilocode/agent/index.ts index c5a4894d67d..39fb9ad264a 100644 --- a/packages/opencode/src/kilocode/agent/index.ts +++ b/packages/opencode/src/kilocode/agent/index.ts @@ -7,7 +7,6 @@ import { Config } from "../../config" import { Instance } from "../../project/instance" import { makeRuntime } from "@/effect/run-service" import { Global } from "@/global" -import { Telemetry } from "@kilocode/kilo-telemetry" import z from "zod" import path from "path" @@ -173,16 +172,9 @@ export function processConfigItem(item: { } // Returns experimental_telemetry config for generate calls. -export function telemetryOptions(cfg: Config.Info) { - return { - isEnabled: cfg.experimental?.openTelemetry !== false, - recordInputs: false, - recordOutputs: false, - tracer: Telemetry.getTracer() ?? undefined, - metadata: { - userId: cfg.username ?? "unknown", - }, - } +// AI SDK span recording (ai.* / gen_ai.*) is disabled. +export function telemetryOptions(_cfg: Config.Info) { + return { isEnabled: false as const } } // Patch the base agents map in-place with all kilo-specific changes: diff --git a/packages/opencode/src/session/llm.ts b/packages/opencode/src/session/llm.ts index 4e97daefaf4..82d5503c0a1 100644 --- a/packages/opencode/src/session/llm.ts +++ b/packages/opencode/src/session/llm.ts @@ -20,7 +20,6 @@ import { Wildcard } from "@/util" import { SessionID } from "@/session/schema" import { Auth } from "@/auth" // kilocode_change start -import { Telemetry } from "@kilocode/kilo-telemetry" import { DEFAULT_HEADERS } from "@/kilocode/const" import { getKiloProjectId } from "@/kilocode/project-id" import { HEADER_PROJECTID, HEADER_MACHINEID, HEADER_TASKID } from "@kilocode/kilo-gateway" @@ -430,14 +429,8 @@ const live: Layer.Layer< }, ], }), - // kilocode_change start - enable telemetry by default with custom PostHog tracer - experimental_telemetry: { - isEnabled: cfg.experimental?.openTelemetry !== false, - recordInputs: false, - recordOutputs: false, - tracer: Telemetry.getTracer() ?? undefined, - }, - // kilocode_change end + // kilocode_change - disable AI SDK span recording (ai.* / gen_ai.*) + experimental_telemetry: { isEnabled: false }, }) })