mirror of
https://github.com/Kilo-Org/kilocode.git
synced 2026-09-21 14:07:20 +08:00
feat: track accepted review suggestions
This commit is contained in:
@@ -50,6 +50,7 @@ describe("TelemetryEvent", () => {
|
||||
expect(TelemetryEvent.COMMAND_USED).toBeDefined()
|
||||
expect(TelemetryEvent.TOOL_USED).toBeDefined()
|
||||
expect(TelemetryEvent.AGENT_USED).toBeDefined()
|
||||
expect(TelemetryEvent.SUGGESTION_ACCEPTED).toBeDefined()
|
||||
})
|
||||
|
||||
test("indexing events are defined", () => {
|
||||
@@ -88,4 +89,8 @@ describe("Telemetry", () => {
|
||||
expect(typeof Telemetry.trackIndexingBatchRetry).toBe("function")
|
||||
expect(typeof Telemetry.trackIndexingError).toBe("function")
|
||||
})
|
||||
|
||||
test("suggestion helper is exposed", () => {
|
||||
expect(typeof Telemetry.trackSuggestionAccepted).toBe("function")
|
||||
})
|
||||
})
|
||||
|
||||
@@ -16,6 +16,7 @@ export enum TelemetryEvent {
|
||||
TOOL_USED = "Tool Used",
|
||||
AGENT_USED = "Agent Used",
|
||||
PLAN_FOLLOWUP = "Plan Followup",
|
||||
SUGGESTION_ACCEPTED = "Suggestion Accepted",
|
||||
|
||||
// Code Indexing
|
||||
INDEXING_STARTED = "Indexing Started",
|
||||
|
||||
@@ -157,6 +157,7 @@ export namespace Telemetry {
|
||||
mode?: "review"
|
||||
feature?: "code_reviews"
|
||||
command?: "review" | "local-review" | "local-review-uncommitted"
|
||||
tool?: "suggest"
|
||||
apiProvider: string
|
||||
modelId: string
|
||||
inputTokens?: number
|
||||
@@ -187,6 +188,16 @@ export namespace Telemetry {
|
||||
track(TelemetryEvent.PLAN_FOLLOWUP, { sessionId, choice })
|
||||
}
|
||||
|
||||
export function trackSuggestionAccepted(properties: {
|
||||
sessionId: string
|
||||
requestId: string
|
||||
index: number
|
||||
tool: "suggest"
|
||||
command: "review" | "local-review" | "local-review-uncommitted"
|
||||
}) {
|
||||
track(TelemetryEvent.SUGGESTION_ACCEPTED, properties)
|
||||
}
|
||||
|
||||
export function trackIndexingStarted(properties: IndexingTelemetryProperties) {
|
||||
track(TelemetryEvent.INDEXING_STARTED, properties)
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import { SessionNetwork } from "@/session/network"
|
||||
import type { SessionID } from "@/session/schema"
|
||||
import type { SessionStatus } from "@/session/status"
|
||||
import { MessageV2 } from "@/session/message-v2"
|
||||
import { isRecord } from "@/util/record"
|
||||
import * as Log from "@opencode-ai/core/util/log"
|
||||
import { Effect } from "effect"
|
||||
import { Flag } from "@opencode-ai/core/flag/flag"
|
||||
@@ -12,6 +13,7 @@ export type ReviewTelemetry = {
|
||||
mode: "review"
|
||||
feature: "code_reviews"
|
||||
command: "review" | "local-review" | "local-review-uncommitted"
|
||||
tool?: "suggest"
|
||||
}
|
||||
|
||||
export namespace KiloSessionProcessor {
|
||||
@@ -28,6 +30,13 @@ export namespace KiloSessionProcessor {
|
||||
}
|
||||
}
|
||||
|
||||
function command(prompt: string | undefined) {
|
||||
if (!prompt?.startsWith("/")) return
|
||||
const name = prompt.slice(1).split(/\s/, 1)[0]
|
||||
if (!name) return
|
||||
return name
|
||||
}
|
||||
|
||||
/**
|
||||
* Tag the text parts of a prompt with review telemetry metadata so that
|
||||
* downstream LLM completions in the same turn (including child sessions
|
||||
@@ -59,6 +68,25 @@ export namespace KiloSessionProcessor {
|
||||
}
|
||||
}
|
||||
|
||||
export function suggestionReviewTelemetry(metadata: unknown): ReviewTelemetry | undefined {
|
||||
if (!isRecord(metadata)) return
|
||||
if (!isRecord(metadata.accepted)) return
|
||||
const prompt = typeof metadata.accepted.prompt === "string" ? metadata.accepted.prompt : undefined
|
||||
const tel = reviewTelemetry(command(prompt))
|
||||
if (!tel) return
|
||||
return { ...tel, tool: "suggest" }
|
||||
}
|
||||
|
||||
export function extractSuggestionReviewTelemetry(parts: MessageV2.Part[]): ReviewTelemetry | undefined {
|
||||
for (const part of parts) {
|
||||
if (part.type !== "tool") continue
|
||||
if (part.tool !== "suggest") continue
|
||||
if (part.state.status !== "completed") continue
|
||||
const tel = suggestionReviewTelemetry(part.state.metadata)
|
||||
if (tel) return tel
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Track LLM completion telemetry for a finished step.
|
||||
* Only fires if at least one token bucket is non-zero.
|
||||
|
||||
@@ -4,6 +4,7 @@ import { Identifier } from "../../id/id"
|
||||
import { SessionID } from "../../session/schema"
|
||||
import { ZodOverride } from "../../util/effect-zod"
|
||||
import * as Log from "@opencode-ai/core/util/log"
|
||||
import { Telemetry } from "@kilocode/kilo-telemetry"
|
||||
import z from "zod"
|
||||
import { Schema } from "effect"
|
||||
import { KiloSessionPromptQueue } from "../session/prompt-queue"
|
||||
@@ -11,6 +12,12 @@ import { KiloSessionPromptQueue } from "../session/prompt-queue"
|
||||
export namespace Suggestion {
|
||||
const log = Log.create({ service: "suggestion" })
|
||||
|
||||
function command(prompt: string): "review" | "local-review" | "local-review-uncommitted" | undefined {
|
||||
if (!prompt.startsWith("/")) return
|
||||
const name = prompt.slice(1).split(/\s/, 1)[0]
|
||||
if (name === "review" || name === "local-review" || name === "local-review-uncommitted") return name
|
||||
}
|
||||
|
||||
export const Action = z
|
||||
.object({
|
||||
label: z.string().describe("Button or option label (1-5 words)"),
|
||||
@@ -176,6 +183,17 @@ export namespace Suggestion {
|
||||
|
||||
log.info("accepted", { requestID: input.requestID, index: input.index, label: action.label })
|
||||
|
||||
const cmd = command(action.prompt)
|
||||
if (cmd) {
|
||||
Telemetry.trackSuggestionAccepted({
|
||||
sessionId: existing.info.sessionID,
|
||||
requestId: existing.info.id,
|
||||
index: input.index,
|
||||
tool: "suggest",
|
||||
command: cmd,
|
||||
})
|
||||
}
|
||||
|
||||
Bus.publish(Event.Accepted, {
|
||||
sessionID: SessionID.make(existing.info.sessionID),
|
||||
requestID: existing.info.id,
|
||||
|
||||
@@ -228,6 +228,11 @@ export const layer: Layer.Layer<
|
||||
attachments: output.attachments,
|
||||
},
|
||||
})
|
||||
// kilocode_change start - accepted suggest review actions tag following LLM completion telemetry
|
||||
if (match.part.tool === "suggest") {
|
||||
ctx.telemetry = KiloSessionProcessor.suggestionReviewTelemetry(output.metadata) ?? ctx.telemetry
|
||||
}
|
||||
// kilocode_change end
|
||||
yield* settleToolCall(toolCallID)
|
||||
})
|
||||
|
||||
|
||||
@@ -1505,15 +1505,16 @@ NOTE: At any point in time through this workflow you should feel free to ask the
|
||||
|
||||
if (!lastUser) throw new Error("No user message found in stream. This should never happen.")
|
||||
|
||||
// kilocode_change start - carry local review command marker into LLM telemetry
|
||||
const telemetry = KiloSessionProcessor.extractReviewTelemetry(
|
||||
msgs.findLast((m) => m.info.role === "user" && m.info.id === lastUser.id)?.parts ?? [],
|
||||
)
|
||||
// kilocode_change end
|
||||
|
||||
const lastAssistantMsg = msgs.findLast(
|
||||
(msg) => msg.info.role === "assistant" && msg.info.id === lastAssistant?.id,
|
||||
)
|
||||
// kilocode_change start - carry local review command marker into LLM telemetry
|
||||
const telemetry =
|
||||
KiloSessionProcessor.extractReviewTelemetry(
|
||||
msgs.findLast((m) => m.info.role === "user" && m.info.id === lastUser.id)?.parts ?? [],
|
||||
) ?? KiloSessionProcessor.extractSuggestionReviewTelemetry(lastAssistantMsg?.parts ?? [])
|
||||
// kilocode_change end
|
||||
|
||||
// kilocode_change start - keep provider-executed tools from forcing a re-loop
|
||||
// Some providers return "stop" even when the assistant message contains tool calls.
|
||||
// Keep the loop running so tool results can be sent back to the model.
|
||||
|
||||
@@ -81,3 +81,45 @@ describe("KiloSessionProcessor.extractReviewTelemetry", () => {
|
||||
expect(KiloSessionProcessor.extractReviewTelemetry(parts as unknown as MessageV2.Part[])).toBeUndefined()
|
||||
})
|
||||
})
|
||||
|
||||
describe("KiloSessionProcessor.suggestionReviewTelemetry", () => {
|
||||
test("returns suggest-sourced telemetry for accepted review commands", () => {
|
||||
expect(
|
||||
KiloSessionProcessor.suggestionReviewTelemetry({
|
||||
accepted: { prompt: "/local-review-uncommitted --focus telemetry" },
|
||||
}),
|
||||
).toEqual({ ...expected("local-review-uncommitted"), tool: "suggest" })
|
||||
})
|
||||
|
||||
test("returns undefined for accepted non-review commands", () => {
|
||||
expect(KiloSessionProcessor.suggestionReviewTelemetry({ accepted: { prompt: "/test" } })).toBeUndefined()
|
||||
})
|
||||
|
||||
test("returns undefined when accepted prompt is not a slash command", () => {
|
||||
expect(KiloSessionProcessor.suggestionReviewTelemetry({ accepted: { prompt: "Run tests" } })).toBeUndefined()
|
||||
})
|
||||
|
||||
test("returns undefined when accepted metadata is missing", () => {
|
||||
expect(KiloSessionProcessor.suggestionReviewTelemetry({ dismissed: true })).toBeUndefined()
|
||||
})
|
||||
})
|
||||
|
||||
describe("KiloSessionProcessor.extractSuggestionReviewTelemetry", () => {
|
||||
test("recovers review telemetry from completed suggest tool metadata", () => {
|
||||
const parts = [
|
||||
{
|
||||
type: "tool",
|
||||
tool: "suggest",
|
||||
state: {
|
||||
status: "completed",
|
||||
metadata: { accepted: { prompt: "/local-review" } },
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
expect(KiloSessionProcessor.extractSuggestionReviewTelemetry(parts as unknown as MessageV2.Part[])).toEqual({
|
||||
...expected("local-review"),
|
||||
tool: "suggest",
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,8 +1,13 @@
|
||||
import { describe, expect, test } from "bun:test"
|
||||
import { afterEach, describe, expect, mock, spyOn, test } from "bun:test"
|
||||
import { Telemetry } from "@kilocode/kilo-telemetry"
|
||||
import { WithInstance } from "../../../src/project/with-instance"
|
||||
import { Suggestion } from "../../../src/kilocode/suggestion"
|
||||
import { tmpdir } from "../../fixture/fixture"
|
||||
|
||||
afterEach(() => {
|
||||
mock.restore()
|
||||
})
|
||||
|
||||
describe("suggestion", () => {
|
||||
test("show adds pending request with blocking flag", async () => {
|
||||
await using tmp = await tmpdir({ git: true })
|
||||
@@ -54,6 +59,97 @@ describe("suggestion", () => {
|
||||
})
|
||||
})
|
||||
|
||||
test("accept tracks suggestion telemetry with parsed slash command", async () => {
|
||||
await using tmp = await tmpdir({ git: true })
|
||||
await WithInstance.provide({
|
||||
directory: tmp.path,
|
||||
fn: async () => {
|
||||
const track = spyOn(Telemetry, "trackSuggestionAccepted")
|
||||
const ask = Suggestion.show({
|
||||
sessionID: "ses_test",
|
||||
text: "Review changes?",
|
||||
actions: [{ label: "Review", prompt: "/local-review-uncommitted --focus tests" }],
|
||||
})
|
||||
|
||||
const list = await Suggestion.list()
|
||||
await Suggestion.accept({ requestID: list[0]!.id, index: 0 })
|
||||
|
||||
expect(track).toHaveBeenCalledTimes(1)
|
||||
expect(track).toHaveBeenCalledWith({
|
||||
sessionId: "ses_test",
|
||||
requestId: list[0]!.id,
|
||||
index: 0,
|
||||
tool: "suggest",
|
||||
command: "local-review-uncommitted",
|
||||
})
|
||||
await expect(ask).resolves.toEqual({ label: "Review", prompt: "/local-review-uncommitted --focus tests" })
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
test("accept does not track non-review suggestion telemetry", async () => {
|
||||
await using tmp = await tmpdir({ git: true })
|
||||
await WithInstance.provide({
|
||||
directory: tmp.path,
|
||||
fn: async () => {
|
||||
const track = spyOn(Telemetry, "trackSuggestionAccepted")
|
||||
const ask = Suggestion.show({
|
||||
sessionID: "ses_test",
|
||||
text: "Run tests?",
|
||||
actions: [{ label: "Test", prompt: "/custom-project-command" }],
|
||||
})
|
||||
|
||||
const list = await Suggestion.list()
|
||||
await Suggestion.accept({ requestID: list[0]!.id, index: 0 })
|
||||
|
||||
expect(track).toHaveBeenCalledTimes(0)
|
||||
await expect(ask).resolves.toEqual({ label: "Test", prompt: "/custom-project-command" })
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
test("dismiss does not track suggestion telemetry", async () => {
|
||||
await using tmp = await tmpdir({ git: true })
|
||||
await WithInstance.provide({
|
||||
directory: tmp.path,
|
||||
fn: async () => {
|
||||
const track = spyOn(Telemetry, "trackSuggestionAccepted")
|
||||
const ask = Suggestion.show({
|
||||
sessionID: "ses_test",
|
||||
text: "Review changes?",
|
||||
actions: [{ label: "Review", prompt: "/local-review" }],
|
||||
})
|
||||
|
||||
const list = await Suggestion.list()
|
||||
await Suggestion.dismiss(list[0]!.id)
|
||||
|
||||
expect(track).toHaveBeenCalledTimes(0)
|
||||
await expect(ask).rejects.toBeInstanceOf(Suggestion.DismissedError)
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
test("invalid action index does not track suggestion telemetry", async () => {
|
||||
await using tmp = await tmpdir({ git: true })
|
||||
await WithInstance.provide({
|
||||
directory: tmp.path,
|
||||
fn: async () => {
|
||||
const track = spyOn(Telemetry, "trackSuggestionAccepted")
|
||||
const ask = Suggestion.show({
|
||||
sessionID: "ses_test",
|
||||
text: "Review changes?",
|
||||
actions: [{ label: "Review", prompt: "/local-review" }],
|
||||
})
|
||||
|
||||
const list = await Suggestion.list()
|
||||
await expect(Suggestion.accept({ requestID: list[0]!.id, index: 1 })).resolves.toBe(false)
|
||||
|
||||
expect(track).toHaveBeenCalledTimes(0)
|
||||
await expect(ask).rejects.toThrow("Invalid action index: 1")
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
test("dismiss rejects pending request and removes it", async () => {
|
||||
await using tmp = await tmpdir({ git: true })
|
||||
await WithInstance.provide({
|
||||
|
||||
@@ -31,6 +31,7 @@ import { SessionProcessor } from "../../src/session/processor"
|
||||
import { SessionPrompt } from "../../src/session/prompt"
|
||||
import { SessionRevert } from "../../src/session/revert"
|
||||
import { SessionRunState } from "../../src/session/run-state"
|
||||
import { Suggestion } from "../../src/kilocode/suggestion" // kilocode_change - accept suggestion in telemetry test
|
||||
import { MessageID, PartID, SessionID } from "../../src/session/schema"
|
||||
import { SessionStatus } from "../../src/session/status"
|
||||
import { SessionV2 } from "../../src/v2/session"
|
||||
@@ -2106,6 +2107,59 @@ it.live(
|
||||
),
|
||||
30_000,
|
||||
)
|
||||
|
||||
it.live(
|
||||
"accepted suggest tool marks following completion with review telemetry",
|
||||
() =>
|
||||
provideTmpdirServer(
|
||||
Effect.fnUntraced(function* ({ llm }) {
|
||||
const trackSpy = spyOn(Telemetry, "trackLlmCompletion")
|
||||
const prompt = yield* SessionPrompt.Service
|
||||
const sessions = yield* Session.Service
|
||||
const chat = yield* sessions.create({
|
||||
title: "Suggest telemetry",
|
||||
permission: [{ permission: "*", pattern: "*", action: "allow" }],
|
||||
})
|
||||
|
||||
yield* llm.tool("suggest", {
|
||||
suggest: "Run a local review?",
|
||||
actions: [{ label: "Review", prompt: "/local-review-uncommitted --focus telemetry" }],
|
||||
})
|
||||
yield* llm.text("review done", { usage: { input: 100, output: 50 } })
|
||||
|
||||
const fiber = yield* prompt
|
||||
.prompt({
|
||||
sessionID: chat.id,
|
||||
agent: "build",
|
||||
model: ref,
|
||||
parts: [{ type: "text", text: "Suggest a review action." }],
|
||||
})
|
||||
.pipe(Effect.forkChild)
|
||||
const request = yield* waitFor(
|
||||
"suggestion request",
|
||||
Effect.promise(() => Suggestion.list()).pipe(
|
||||
Effect.map((items) => items.find((item) => item.sessionID === chat.id)),
|
||||
),
|
||||
)
|
||||
|
||||
yield* Effect.promise(() => Suggestion.accept({ requestID: request.id, index: 0 }))
|
||||
yield* Fiber.join(fiber)
|
||||
|
||||
const tagged = trackSpy.mock.calls
|
||||
.map((args) => args[0] as Parameters<typeof Telemetry.trackLlmCompletion>[0])
|
||||
.find(
|
||||
(p) =>
|
||||
p.mode === "review" &&
|
||||
p.feature === "code_reviews" &&
|
||||
p.command === "local-review-uncommitted" &&
|
||||
p.tool === "suggest",
|
||||
)
|
||||
expect(tagged).toBeDefined()
|
||||
}),
|
||||
{ git: true, config: providerCfg },
|
||||
),
|
||||
30_000,
|
||||
)
|
||||
// kilocode_change end
|
||||
|
||||
// Agent / command resolution errors
|
||||
|
||||
Reference in New Issue
Block a user