feat: track accepted review suggestions

This commit is contained in:
Alex Alecu
2026-05-22 11:08:22 +03:00
parent e27b1bfd22
commit c791ccf440
10 changed files with 268 additions and 7 deletions
@@ -50,6 +50,7 @@ describe("TelemetryEvent", () => {
expect(TelemetryEvent.COMMAND_USED).toBeDefined()
expect(TelemetryEvent.TOOL_USED).toBeDefined()
expect(TelemetryEvent.AGENT_USED).toBeDefined()
expect(TelemetryEvent.SUGGESTION_ACCEPTED).toBeDefined()
})
test("indexing events are defined", () => {
@@ -88,4 +89,8 @@ describe("Telemetry", () => {
expect(typeof Telemetry.trackIndexingBatchRetry).toBe("function")
expect(typeof Telemetry.trackIndexingError).toBe("function")
})
test("suggestion helper is exposed", () => {
expect(typeof Telemetry.trackSuggestionAccepted).toBe("function")
})
})
+1
View File
@@ -16,6 +16,7 @@ export enum TelemetryEvent {
TOOL_USED = "Tool Used",
AGENT_USED = "Agent Used",
PLAN_FOLLOWUP = "Plan Followup",
SUGGESTION_ACCEPTED = "Suggestion Accepted",
// Code Indexing
INDEXING_STARTED = "Indexing Started",
+11
View File
@@ -157,6 +157,7 @@ export namespace Telemetry {
mode?: "review"
feature?: "code_reviews"
command?: "review" | "local-review" | "local-review-uncommitted"
tool?: "suggest"
apiProvider: string
modelId: string
inputTokens?: number
@@ -187,6 +188,16 @@ export namespace Telemetry {
track(TelemetryEvent.PLAN_FOLLOWUP, { sessionId, choice })
}
export function trackSuggestionAccepted(properties: {
sessionId: string
requestId: string
index: number
tool: "suggest"
command: "review" | "local-review" | "local-review-uncommitted"
}) {
track(TelemetryEvent.SUGGESTION_ACCEPTED, properties)
}
export function trackIndexingStarted(properties: IndexingTelemetryProperties) {
track(TelemetryEvent.INDEXING_STARTED, properties)
}
@@ -4,6 +4,7 @@ import { SessionNetwork } from "@/session/network"
import type { SessionID } from "@/session/schema"
import type { SessionStatus } from "@/session/status"
import { MessageV2 } from "@/session/message-v2"
import { isRecord } from "@/util/record"
import * as Log from "@opencode-ai/core/util/log"
import { Effect } from "effect"
import { Flag } from "@opencode-ai/core/flag/flag"
@@ -12,6 +13,7 @@ export type ReviewTelemetry = {
mode: "review"
feature: "code_reviews"
command: "review" | "local-review" | "local-review-uncommitted"
tool?: "suggest"
}
export namespace KiloSessionProcessor {
@@ -28,6 +30,13 @@ export namespace KiloSessionProcessor {
}
}
function command(prompt: string | undefined) {
if (!prompt?.startsWith("/")) return
const name = prompt.slice(1).split(/\s/, 1)[0]
if (!name) return
return name
}
/**
* Tag the text parts of a prompt with review telemetry metadata so that
* downstream LLM completions in the same turn (including child sessions
@@ -59,6 +68,25 @@ export namespace KiloSessionProcessor {
}
}
export function suggestionReviewTelemetry(metadata: unknown): ReviewTelemetry | undefined {
if (!isRecord(metadata)) return
if (!isRecord(metadata.accepted)) return
const prompt = typeof metadata.accepted.prompt === "string" ? metadata.accepted.prompt : undefined
const tel = reviewTelemetry(command(prompt))
if (!tel) return
return { ...tel, tool: "suggest" }
}
export function extractSuggestionReviewTelemetry(parts: MessageV2.Part[]): ReviewTelemetry | undefined {
for (const part of parts) {
if (part.type !== "tool") continue
if (part.tool !== "suggest") continue
if (part.state.status !== "completed") continue
const tel = suggestionReviewTelemetry(part.state.metadata)
if (tel) return tel
}
}
/**
* Track LLM completion telemetry for a finished step.
* Only fires if at least one token bucket is non-zero.
@@ -4,6 +4,7 @@ import { Identifier } from "../../id/id"
import { SessionID } from "../../session/schema"
import { ZodOverride } from "../../util/effect-zod"
import * as Log from "@opencode-ai/core/util/log"
import { Telemetry } from "@kilocode/kilo-telemetry"
import z from "zod"
import { Schema } from "effect"
import { KiloSessionPromptQueue } from "../session/prompt-queue"
@@ -11,6 +12,12 @@ import { KiloSessionPromptQueue } from "../session/prompt-queue"
export namespace Suggestion {
const log = Log.create({ service: "suggestion" })
function command(prompt: string): "review" | "local-review" | "local-review-uncommitted" | undefined {
if (!prompt.startsWith("/")) return
const name = prompt.slice(1).split(/\s/, 1)[0]
if (name === "review" || name === "local-review" || name === "local-review-uncommitted") return name
}
export const Action = z
.object({
label: z.string().describe("Button or option label (1-5 words)"),
@@ -176,6 +183,17 @@ export namespace Suggestion {
log.info("accepted", { requestID: input.requestID, index: input.index, label: action.label })
const cmd = command(action.prompt)
if (cmd) {
Telemetry.trackSuggestionAccepted({
sessionId: existing.info.sessionID,
requestId: existing.info.id,
index: input.index,
tool: "suggest",
command: cmd,
})
}
Bus.publish(Event.Accepted, {
sessionID: SessionID.make(existing.info.sessionID),
requestID: existing.info.id,
@@ -228,6 +228,11 @@ export const layer: Layer.Layer<
attachments: output.attachments,
},
})
// kilocode_change start - accepted suggest review actions tag following LLM completion telemetry
if (match.part.tool === "suggest") {
ctx.telemetry = KiloSessionProcessor.suggestionReviewTelemetry(output.metadata) ?? ctx.telemetry
}
// kilocode_change end
yield* settleToolCall(toolCallID)
})
+7 -6
View File
@@ -1505,15 +1505,16 @@ NOTE: At any point in time through this workflow you should feel free to ask the
if (!lastUser) throw new Error("No user message found in stream. This should never happen.")
// kilocode_change start - carry local review command marker into LLM telemetry
const telemetry = KiloSessionProcessor.extractReviewTelemetry(
msgs.findLast((m) => m.info.role === "user" && m.info.id === lastUser.id)?.parts ?? [],
)
// kilocode_change end
const lastAssistantMsg = msgs.findLast(
(msg) => msg.info.role === "assistant" && msg.info.id === lastAssistant?.id,
)
// kilocode_change start - carry local review command marker into LLM telemetry
const telemetry =
KiloSessionProcessor.extractReviewTelemetry(
msgs.findLast((m) => m.info.role === "user" && m.info.id === lastUser.id)?.parts ?? [],
) ?? KiloSessionProcessor.extractSuggestionReviewTelemetry(lastAssistantMsg?.parts ?? [])
// kilocode_change end
// kilocode_change start - keep provider-executed tools from forcing a re-loop
// Some providers return "stop" even when the assistant message contains tool calls.
// Keep the loop running so tool results can be sent back to the model.
@@ -81,3 +81,45 @@ describe("KiloSessionProcessor.extractReviewTelemetry", () => {
expect(KiloSessionProcessor.extractReviewTelemetry(parts as unknown as MessageV2.Part[])).toBeUndefined()
})
})
describe("KiloSessionProcessor.suggestionReviewTelemetry", () => {
test("returns suggest-sourced telemetry for accepted review commands", () => {
expect(
KiloSessionProcessor.suggestionReviewTelemetry({
accepted: { prompt: "/local-review-uncommitted --focus telemetry" },
}),
).toEqual({ ...expected("local-review-uncommitted"), tool: "suggest" })
})
test("returns undefined for accepted non-review commands", () => {
expect(KiloSessionProcessor.suggestionReviewTelemetry({ accepted: { prompt: "/test" } })).toBeUndefined()
})
test("returns undefined when accepted prompt is not a slash command", () => {
expect(KiloSessionProcessor.suggestionReviewTelemetry({ accepted: { prompt: "Run tests" } })).toBeUndefined()
})
test("returns undefined when accepted metadata is missing", () => {
expect(KiloSessionProcessor.suggestionReviewTelemetry({ dismissed: true })).toBeUndefined()
})
})
describe("KiloSessionProcessor.extractSuggestionReviewTelemetry", () => {
test("recovers review telemetry from completed suggest tool metadata", () => {
const parts = [
{
type: "tool",
tool: "suggest",
state: {
status: "completed",
metadata: { accepted: { prompt: "/local-review" } },
},
},
]
expect(KiloSessionProcessor.extractSuggestionReviewTelemetry(parts as unknown as MessageV2.Part[])).toEqual({
...expected("local-review"),
tool: "suggest",
})
})
})
@@ -1,8 +1,13 @@
import { describe, expect, test } from "bun:test"
import { afterEach, describe, expect, mock, spyOn, test } from "bun:test"
import { Telemetry } from "@kilocode/kilo-telemetry"
import { WithInstance } from "../../../src/project/with-instance"
import { Suggestion } from "../../../src/kilocode/suggestion"
import { tmpdir } from "../../fixture/fixture"
afterEach(() => {
mock.restore()
})
describe("suggestion", () => {
test("show adds pending request with blocking flag", async () => {
await using tmp = await tmpdir({ git: true })
@@ -54,6 +59,97 @@ describe("suggestion", () => {
})
})
test("accept tracks suggestion telemetry with parsed slash command", async () => {
await using tmp = await tmpdir({ git: true })
await WithInstance.provide({
directory: tmp.path,
fn: async () => {
const track = spyOn(Telemetry, "trackSuggestionAccepted")
const ask = Suggestion.show({
sessionID: "ses_test",
text: "Review changes?",
actions: [{ label: "Review", prompt: "/local-review-uncommitted --focus tests" }],
})
const list = await Suggestion.list()
await Suggestion.accept({ requestID: list[0]!.id, index: 0 })
expect(track).toHaveBeenCalledTimes(1)
expect(track).toHaveBeenCalledWith({
sessionId: "ses_test",
requestId: list[0]!.id,
index: 0,
tool: "suggest",
command: "local-review-uncommitted",
})
await expect(ask).resolves.toEqual({ label: "Review", prompt: "/local-review-uncommitted --focus tests" })
},
})
})
test("accept does not track non-review suggestion telemetry", async () => {
await using tmp = await tmpdir({ git: true })
await WithInstance.provide({
directory: tmp.path,
fn: async () => {
const track = spyOn(Telemetry, "trackSuggestionAccepted")
const ask = Suggestion.show({
sessionID: "ses_test",
text: "Run tests?",
actions: [{ label: "Test", prompt: "/custom-project-command" }],
})
const list = await Suggestion.list()
await Suggestion.accept({ requestID: list[0]!.id, index: 0 })
expect(track).toHaveBeenCalledTimes(0)
await expect(ask).resolves.toEqual({ label: "Test", prompt: "/custom-project-command" })
},
})
})
test("dismiss does not track suggestion telemetry", async () => {
await using tmp = await tmpdir({ git: true })
await WithInstance.provide({
directory: tmp.path,
fn: async () => {
const track = spyOn(Telemetry, "trackSuggestionAccepted")
const ask = Suggestion.show({
sessionID: "ses_test",
text: "Review changes?",
actions: [{ label: "Review", prompt: "/local-review" }],
})
const list = await Suggestion.list()
await Suggestion.dismiss(list[0]!.id)
expect(track).toHaveBeenCalledTimes(0)
await expect(ask).rejects.toBeInstanceOf(Suggestion.DismissedError)
},
})
})
test("invalid action index does not track suggestion telemetry", async () => {
await using tmp = await tmpdir({ git: true })
await WithInstance.provide({
directory: tmp.path,
fn: async () => {
const track = spyOn(Telemetry, "trackSuggestionAccepted")
const ask = Suggestion.show({
sessionID: "ses_test",
text: "Review changes?",
actions: [{ label: "Review", prompt: "/local-review" }],
})
const list = await Suggestion.list()
await expect(Suggestion.accept({ requestID: list[0]!.id, index: 1 })).resolves.toBe(false)
expect(track).toHaveBeenCalledTimes(0)
await expect(ask).rejects.toThrow("Invalid action index: 1")
},
})
})
test("dismiss rejects pending request and removes it", async () => {
await using tmp = await tmpdir({ git: true })
await WithInstance.provide({
@@ -31,6 +31,7 @@ import { SessionProcessor } from "../../src/session/processor"
import { SessionPrompt } from "../../src/session/prompt"
import { SessionRevert } from "../../src/session/revert"
import { SessionRunState } from "../../src/session/run-state"
import { Suggestion } from "../../src/kilocode/suggestion" // kilocode_change - accept suggestion in telemetry test
import { MessageID, PartID, SessionID } from "../../src/session/schema"
import { SessionStatus } from "../../src/session/status"
import { SessionV2 } from "../../src/v2/session"
@@ -2106,6 +2107,59 @@ it.live(
),
30_000,
)
it.live(
"accepted suggest tool marks following completion with review telemetry",
() =>
provideTmpdirServer(
Effect.fnUntraced(function* ({ llm }) {
const trackSpy = spyOn(Telemetry, "trackLlmCompletion")
const prompt = yield* SessionPrompt.Service
const sessions = yield* Session.Service
const chat = yield* sessions.create({
title: "Suggest telemetry",
permission: [{ permission: "*", pattern: "*", action: "allow" }],
})
yield* llm.tool("suggest", {
suggest: "Run a local review?",
actions: [{ label: "Review", prompt: "/local-review-uncommitted --focus telemetry" }],
})
yield* llm.text("review done", { usage: { input: 100, output: 50 } })
const fiber = yield* prompt
.prompt({
sessionID: chat.id,
agent: "build",
model: ref,
parts: [{ type: "text", text: "Suggest a review action." }],
})
.pipe(Effect.forkChild)
const request = yield* waitFor(
"suggestion request",
Effect.promise(() => Suggestion.list()).pipe(
Effect.map((items) => items.find((item) => item.sessionID === chat.id)),
),
)
yield* Effect.promise(() => Suggestion.accept({ requestID: request.id, index: 0 }))
yield* Fiber.join(fiber)
const tagged = trackSpy.mock.calls
.map((args) => args[0] as Parameters<typeof Telemetry.trackLlmCompletion>[0])
.find(
(p) =>
p.mode === "review" &&
p.feature === "code_reviews" &&
p.command === "local-review-uncommitted" &&
p.tool === "suggest",
)
expect(tagged).toBeDefined()
}),
{ git: true, config: providerCfg },
),
30_000,
)
// kilocode_change end
// Agent / command resolution errors