From c791ccf440fbc0c9152b9846b4aa8640c45751fd Mon Sep 17 00:00:00 2001 From: Alex Alecu Date: Fri, 22 May 2026 11:08:22 +0300 Subject: [PATCH] feat: track accepted review suggestions --- .../src/__tests__/telemetry.test.ts | 5 + packages/kilo-telemetry/src/events.ts | 1 + packages/kilo-telemetry/src/telemetry.ts | 11 +++ .../src/kilocode/session/processor.ts | 28 ++++++ .../opencode/src/kilocode/suggestion/index.ts | 18 ++++ packages/opencode/src/session/processor.ts | 5 + packages/opencode/src/session/prompt.ts | 13 +-- ...session-processor-review-telemetry.test.ts | 42 ++++++++ .../kilocode/suggestion/suggestion.test.ts | 98 ++++++++++++++++++- packages/opencode/test/session/prompt.test.ts | 54 ++++++++++ 10 files changed, 268 insertions(+), 7 deletions(-) diff --git a/packages/kilo-telemetry/src/__tests__/telemetry.test.ts b/packages/kilo-telemetry/src/__tests__/telemetry.test.ts index 4c3c62714ad..deda61621f5 100644 --- a/packages/kilo-telemetry/src/__tests__/telemetry.test.ts +++ b/packages/kilo-telemetry/src/__tests__/telemetry.test.ts @@ -50,6 +50,7 @@ describe("TelemetryEvent", () => { expect(TelemetryEvent.COMMAND_USED).toBeDefined() expect(TelemetryEvent.TOOL_USED).toBeDefined() expect(TelemetryEvent.AGENT_USED).toBeDefined() + expect(TelemetryEvent.SUGGESTION_ACCEPTED).toBeDefined() }) test("indexing events are defined", () => { @@ -88,4 +89,8 @@ describe("Telemetry", () => { expect(typeof Telemetry.trackIndexingBatchRetry).toBe("function") expect(typeof Telemetry.trackIndexingError).toBe("function") }) + + test("suggestion helper is exposed", () => { + expect(typeof Telemetry.trackSuggestionAccepted).toBe("function") + }) }) diff --git a/packages/kilo-telemetry/src/events.ts b/packages/kilo-telemetry/src/events.ts index 59b489a34ac..42646ade46c 100644 --- a/packages/kilo-telemetry/src/events.ts +++ b/packages/kilo-telemetry/src/events.ts @@ -16,6 +16,7 @@ export enum TelemetryEvent { TOOL_USED = "Tool Used", AGENT_USED = "Agent Used", PLAN_FOLLOWUP = "Plan Followup", + SUGGESTION_ACCEPTED = "Suggestion Accepted", // Code Indexing INDEXING_STARTED = "Indexing Started", diff --git a/packages/kilo-telemetry/src/telemetry.ts b/packages/kilo-telemetry/src/telemetry.ts index 4f8837a2579..39b25976f3b 100644 --- a/packages/kilo-telemetry/src/telemetry.ts +++ b/packages/kilo-telemetry/src/telemetry.ts @@ -157,6 +157,7 @@ export namespace Telemetry { mode?: "review" feature?: "code_reviews" command?: "review" | "local-review" | "local-review-uncommitted" + tool?: "suggest" apiProvider: string modelId: string inputTokens?: number @@ -187,6 +188,16 @@ export namespace Telemetry { track(TelemetryEvent.PLAN_FOLLOWUP, { sessionId, choice }) } + export function trackSuggestionAccepted(properties: { + sessionId: string + requestId: string + index: number + tool: "suggest" + command: "review" | "local-review" | "local-review-uncommitted" + }) { + track(TelemetryEvent.SUGGESTION_ACCEPTED, properties) + } + export function trackIndexingStarted(properties: IndexingTelemetryProperties) { track(TelemetryEvent.INDEXING_STARTED, properties) } diff --git a/packages/opencode/src/kilocode/session/processor.ts b/packages/opencode/src/kilocode/session/processor.ts index 098c1acbfa3..6d1d379b8cb 100644 --- a/packages/opencode/src/kilocode/session/processor.ts +++ b/packages/opencode/src/kilocode/session/processor.ts @@ -4,6 +4,7 @@ import { SessionNetwork } from "@/session/network" import type { SessionID } from "@/session/schema" import type { SessionStatus } from "@/session/status" import { MessageV2 } from "@/session/message-v2" +import { isRecord } from "@/util/record" import * as Log from "@opencode-ai/core/util/log" import { Effect } from "effect" import { Flag } from "@opencode-ai/core/flag/flag" @@ -12,6 +13,7 @@ export type ReviewTelemetry = { mode: "review" feature: "code_reviews" command: "review" | "local-review" | "local-review-uncommitted" + tool?: "suggest" } export namespace KiloSessionProcessor { @@ -28,6 +30,13 @@ export namespace KiloSessionProcessor { } } + function command(prompt: string | undefined) { + if (!prompt?.startsWith("/")) return + const name = prompt.slice(1).split(/\s/, 1)[0] + if (!name) return + return name + } + /** * Tag the text parts of a prompt with review telemetry metadata so that * downstream LLM completions in the same turn (including child sessions @@ -59,6 +68,25 @@ export namespace KiloSessionProcessor { } } + export function suggestionReviewTelemetry(metadata: unknown): ReviewTelemetry | undefined { + if (!isRecord(metadata)) return + if (!isRecord(metadata.accepted)) return + const prompt = typeof metadata.accepted.prompt === "string" ? metadata.accepted.prompt : undefined + const tel = reviewTelemetry(command(prompt)) + if (!tel) return + return { ...tel, tool: "suggest" } + } + + export function extractSuggestionReviewTelemetry(parts: MessageV2.Part[]): ReviewTelemetry | undefined { + for (const part of parts) { + if (part.type !== "tool") continue + if (part.tool !== "suggest") continue + if (part.state.status !== "completed") continue + const tel = suggestionReviewTelemetry(part.state.metadata) + if (tel) return tel + } + } + /** * Track LLM completion telemetry for a finished step. * Only fires if at least one token bucket is non-zero. diff --git a/packages/opencode/src/kilocode/suggestion/index.ts b/packages/opencode/src/kilocode/suggestion/index.ts index 1d319400110..f6b4a5a4351 100644 --- a/packages/opencode/src/kilocode/suggestion/index.ts +++ b/packages/opencode/src/kilocode/suggestion/index.ts @@ -4,6 +4,7 @@ import { Identifier } from "../../id/id" import { SessionID } from "../../session/schema" import { ZodOverride } from "../../util/effect-zod" import * as Log from "@opencode-ai/core/util/log" +import { Telemetry } from "@kilocode/kilo-telemetry" import z from "zod" import { Schema } from "effect" import { KiloSessionPromptQueue } from "../session/prompt-queue" @@ -11,6 +12,12 @@ import { KiloSessionPromptQueue } from "../session/prompt-queue" export namespace Suggestion { const log = Log.create({ service: "suggestion" }) + function command(prompt: string): "review" | "local-review" | "local-review-uncommitted" | undefined { + if (!prompt.startsWith("/")) return + const name = prompt.slice(1).split(/\s/, 1)[0] + if (name === "review" || name === "local-review" || name === "local-review-uncommitted") return name + } + export const Action = z .object({ label: z.string().describe("Button or option label (1-5 words)"), @@ -176,6 +183,17 @@ export namespace Suggestion { log.info("accepted", { requestID: input.requestID, index: input.index, label: action.label }) + const cmd = command(action.prompt) + if (cmd) { + Telemetry.trackSuggestionAccepted({ + sessionId: existing.info.sessionID, + requestId: existing.info.id, + index: input.index, + tool: "suggest", + command: cmd, + }) + } + Bus.publish(Event.Accepted, { sessionID: SessionID.make(existing.info.sessionID), requestID: existing.info.id, diff --git a/packages/opencode/src/session/processor.ts b/packages/opencode/src/session/processor.ts index f29d6b3d16b..f5b58d6e24a 100644 --- a/packages/opencode/src/session/processor.ts +++ b/packages/opencode/src/session/processor.ts @@ -228,6 +228,11 @@ export const layer: Layer.Layer< attachments: output.attachments, }, }) + // kilocode_change start - accepted suggest review actions tag following LLM completion telemetry + if (match.part.tool === "suggest") { + ctx.telemetry = KiloSessionProcessor.suggestionReviewTelemetry(output.metadata) ?? ctx.telemetry + } + // kilocode_change end yield* settleToolCall(toolCallID) }) diff --git a/packages/opencode/src/session/prompt.ts b/packages/opencode/src/session/prompt.ts index 88c3f2b2af6..dff73fcb631 100644 --- a/packages/opencode/src/session/prompt.ts +++ b/packages/opencode/src/session/prompt.ts @@ -1505,15 +1505,16 @@ NOTE: At any point in time through this workflow you should feel free to ask the if (!lastUser) throw new Error("No user message found in stream. This should never happen.") - // kilocode_change start - carry local review command marker into LLM telemetry - const telemetry = KiloSessionProcessor.extractReviewTelemetry( - msgs.findLast((m) => m.info.role === "user" && m.info.id === lastUser.id)?.parts ?? [], - ) - // kilocode_change end - const lastAssistantMsg = msgs.findLast( (msg) => msg.info.role === "assistant" && msg.info.id === lastAssistant?.id, ) + // kilocode_change start - carry local review command marker into LLM telemetry + const telemetry = + KiloSessionProcessor.extractReviewTelemetry( + msgs.findLast((m) => m.info.role === "user" && m.info.id === lastUser.id)?.parts ?? [], + ) ?? KiloSessionProcessor.extractSuggestionReviewTelemetry(lastAssistantMsg?.parts ?? []) + // kilocode_change end + // kilocode_change start - keep provider-executed tools from forcing a re-loop // Some providers return "stop" even when the assistant message contains tool calls. // Keep the loop running so tool results can be sent back to the model. diff --git a/packages/opencode/test/kilocode/session-processor-review-telemetry.test.ts b/packages/opencode/test/kilocode/session-processor-review-telemetry.test.ts index 1a235c7b134..fc19ce12595 100644 --- a/packages/opencode/test/kilocode/session-processor-review-telemetry.test.ts +++ b/packages/opencode/test/kilocode/session-processor-review-telemetry.test.ts @@ -81,3 +81,45 @@ describe("KiloSessionProcessor.extractReviewTelemetry", () => { expect(KiloSessionProcessor.extractReviewTelemetry(parts as unknown as MessageV2.Part[])).toBeUndefined() }) }) + +describe("KiloSessionProcessor.suggestionReviewTelemetry", () => { + test("returns suggest-sourced telemetry for accepted review commands", () => { + expect( + KiloSessionProcessor.suggestionReviewTelemetry({ + accepted: { prompt: "/local-review-uncommitted --focus telemetry" }, + }), + ).toEqual({ ...expected("local-review-uncommitted"), tool: "suggest" }) + }) + + test("returns undefined for accepted non-review commands", () => { + expect(KiloSessionProcessor.suggestionReviewTelemetry({ accepted: { prompt: "/test" } })).toBeUndefined() + }) + + test("returns undefined when accepted prompt is not a slash command", () => { + expect(KiloSessionProcessor.suggestionReviewTelemetry({ accepted: { prompt: "Run tests" } })).toBeUndefined() + }) + + test("returns undefined when accepted metadata is missing", () => { + expect(KiloSessionProcessor.suggestionReviewTelemetry({ dismissed: true })).toBeUndefined() + }) +}) + +describe("KiloSessionProcessor.extractSuggestionReviewTelemetry", () => { + test("recovers review telemetry from completed suggest tool metadata", () => { + const parts = [ + { + type: "tool", + tool: "suggest", + state: { + status: "completed", + metadata: { accepted: { prompt: "/local-review" } }, + }, + }, + ] + + expect(KiloSessionProcessor.extractSuggestionReviewTelemetry(parts as unknown as MessageV2.Part[])).toEqual({ + ...expected("local-review"), + tool: "suggest", + }) + }) +}) diff --git a/packages/opencode/test/kilocode/suggestion/suggestion.test.ts b/packages/opencode/test/kilocode/suggestion/suggestion.test.ts index 53f635c8b10..16315fc0394 100644 --- a/packages/opencode/test/kilocode/suggestion/suggestion.test.ts +++ b/packages/opencode/test/kilocode/suggestion/suggestion.test.ts @@ -1,8 +1,13 @@ -import { describe, expect, test } from "bun:test" +import { afterEach, describe, expect, mock, spyOn, test } from "bun:test" +import { Telemetry } from "@kilocode/kilo-telemetry" import { WithInstance } from "../../../src/project/with-instance" import { Suggestion } from "../../../src/kilocode/suggestion" import { tmpdir } from "../../fixture/fixture" +afterEach(() => { + mock.restore() +}) + describe("suggestion", () => { test("show adds pending request with blocking flag", async () => { await using tmp = await tmpdir({ git: true }) @@ -54,6 +59,97 @@ describe("suggestion", () => { }) }) + test("accept tracks suggestion telemetry with parsed slash command", async () => { + await using tmp = await tmpdir({ git: true }) + await WithInstance.provide({ + directory: tmp.path, + fn: async () => { + const track = spyOn(Telemetry, "trackSuggestionAccepted") + const ask = Suggestion.show({ + sessionID: "ses_test", + text: "Review changes?", + actions: [{ label: "Review", prompt: "/local-review-uncommitted --focus tests" }], + }) + + const list = await Suggestion.list() + await Suggestion.accept({ requestID: list[0]!.id, index: 0 }) + + expect(track).toHaveBeenCalledTimes(1) + expect(track).toHaveBeenCalledWith({ + sessionId: "ses_test", + requestId: list[0]!.id, + index: 0, + tool: "suggest", + command: "local-review-uncommitted", + }) + await expect(ask).resolves.toEqual({ label: "Review", prompt: "/local-review-uncommitted --focus tests" }) + }, + }) + }) + + test("accept does not track non-review suggestion telemetry", async () => { + await using tmp = await tmpdir({ git: true }) + await WithInstance.provide({ + directory: tmp.path, + fn: async () => { + const track = spyOn(Telemetry, "trackSuggestionAccepted") + const ask = Suggestion.show({ + sessionID: "ses_test", + text: "Run tests?", + actions: [{ label: "Test", prompt: "/custom-project-command" }], + }) + + const list = await Suggestion.list() + await Suggestion.accept({ requestID: list[0]!.id, index: 0 }) + + expect(track).toHaveBeenCalledTimes(0) + await expect(ask).resolves.toEqual({ label: "Test", prompt: "/custom-project-command" }) + }, + }) + }) + + test("dismiss does not track suggestion telemetry", async () => { + await using tmp = await tmpdir({ git: true }) + await WithInstance.provide({ + directory: tmp.path, + fn: async () => { + const track = spyOn(Telemetry, "trackSuggestionAccepted") + const ask = Suggestion.show({ + sessionID: "ses_test", + text: "Review changes?", + actions: [{ label: "Review", prompt: "/local-review" }], + }) + + const list = await Suggestion.list() + await Suggestion.dismiss(list[0]!.id) + + expect(track).toHaveBeenCalledTimes(0) + await expect(ask).rejects.toBeInstanceOf(Suggestion.DismissedError) + }, + }) + }) + + test("invalid action index does not track suggestion telemetry", async () => { + await using tmp = await tmpdir({ git: true }) + await WithInstance.provide({ + directory: tmp.path, + fn: async () => { + const track = spyOn(Telemetry, "trackSuggestionAccepted") + const ask = Suggestion.show({ + sessionID: "ses_test", + text: "Review changes?", + actions: [{ label: "Review", prompt: "/local-review" }], + }) + + const list = await Suggestion.list() + await expect(Suggestion.accept({ requestID: list[0]!.id, index: 1 })).resolves.toBe(false) + + expect(track).toHaveBeenCalledTimes(0) + await expect(ask).rejects.toThrow("Invalid action index: 1") + }, + }) + }) + test("dismiss rejects pending request and removes it", async () => { await using tmp = await tmpdir({ git: true }) await WithInstance.provide({ diff --git a/packages/opencode/test/session/prompt.test.ts b/packages/opencode/test/session/prompt.test.ts index 8af306ef258..04e5c7aecdd 100644 --- a/packages/opencode/test/session/prompt.test.ts +++ b/packages/opencode/test/session/prompt.test.ts @@ -31,6 +31,7 @@ import { SessionProcessor } from "../../src/session/processor" import { SessionPrompt } from "../../src/session/prompt" import { SessionRevert } from "../../src/session/revert" import { SessionRunState } from "../../src/session/run-state" +import { Suggestion } from "../../src/kilocode/suggestion" // kilocode_change - accept suggestion in telemetry test import { MessageID, PartID, SessionID } from "../../src/session/schema" import { SessionStatus } from "../../src/session/status" import { SessionV2 } from "../../src/v2/session" @@ -2106,6 +2107,59 @@ it.live( ), 30_000, ) + +it.live( + "accepted suggest tool marks following completion with review telemetry", + () => + provideTmpdirServer( + Effect.fnUntraced(function* ({ llm }) { + const trackSpy = spyOn(Telemetry, "trackLlmCompletion") + const prompt = yield* SessionPrompt.Service + const sessions = yield* Session.Service + const chat = yield* sessions.create({ + title: "Suggest telemetry", + permission: [{ permission: "*", pattern: "*", action: "allow" }], + }) + + yield* llm.tool("suggest", { + suggest: "Run a local review?", + actions: [{ label: "Review", prompt: "/local-review-uncommitted --focus telemetry" }], + }) + yield* llm.text("review done", { usage: { input: 100, output: 50 } }) + + const fiber = yield* prompt + .prompt({ + sessionID: chat.id, + agent: "build", + model: ref, + parts: [{ type: "text", text: "Suggest a review action." }], + }) + .pipe(Effect.forkChild) + const request = yield* waitFor( + "suggestion request", + Effect.promise(() => Suggestion.list()).pipe( + Effect.map((items) => items.find((item) => item.sessionID === chat.id)), + ), + ) + + yield* Effect.promise(() => Suggestion.accept({ requestID: request.id, index: 0 })) + yield* Fiber.join(fiber) + + const tagged = trackSpy.mock.calls + .map((args) => args[0] as Parameters[0]) + .find( + (p) => + p.mode === "review" && + p.feature === "code_reviews" && + p.command === "local-review-uncommitted" && + p.tool === "suggest", + ) + expect(tagged).toBeDefined() + }), + { git: true, config: providerCfg }, + ), + 30_000, +) // kilocode_change end // Agent / command resolution errors