feat(agent-manager): refine browser controls and render element feedback

This commit is contained in:
marius-kilocode
2026-08-27 12:40:45 +02:00
parent c7f59f11ef
commit 71fe99d99d
31 changed files with 1688 additions and 431 deletions
+1 -1
View File
@@ -3,4 +3,4 @@
"kilo-code": minor
---
Open and interact with local applications in Agent Manager through a session-scoped browser.
Inspect local applications in Agent Manager with embedded developer tools and review-style element feedback for precise frontend changes.
+52 -18
View File
@@ -142,7 +142,8 @@ import {
} from "./kilo-provider/handlers/question"
import { fetchAndSendPendingSuggestions } from "./kilo-provider/handlers/suggestion"
import { nativeTitle } from "./kilo-provider/native-tab-title"
import { parseReview, reviewMetadata, type ReviewMessageData } from "./shared/review-comments"
import { type ReviewMessageData } from "./shared/review-comments"
import { feedbackMetadata, parseFeedback, type BrowserFeedbackData } from "./shared/browser-feedback"
import { completesWithoutStatus } from "./kilo-provider/command-completion"
import { KiloProviderMemory } from "./kilo-provider/memory"
@@ -202,6 +203,29 @@ type TypedWebviewMessage = {
type: string
value?: unknown
}
type WebviewMessage = Parameters<Parameters<vscode.Webview["onDidReceiveMessage"]>[0]>[0]
function feedbackMessage(message: { text: string; review?: unknown; browserFeedback?: unknown }) {
return parseFeedback({ review: message.review, browserFeedback: message.browserFeedback }, message.text)
}
type SendWebviewMessage = {
type: "sendMessage"
text: string
messageID?: unknown
sessionID?: string
draftID?: unknown
providerID?: string
modelID?: string
agent?: string
variant?: string
files?: unknown
review?: unknown
browserFeedback?: unknown
agentManagerContext?: unknown
contextDirectory?: unknown
}
type SandboxSupportClient = {
support: (
parameters: { directory?: string },
@@ -1096,21 +1120,7 @@ export class KiloProvider implements vscode.WebviewViewProvider, TelemetryProper
this.readyResolvers.splice(0).forEach((r) => r())
break
case "sendMessage": {
const msg = message as typeof message & ContextMessage
await this.handleSendMessage(
message.text,
typeof message.messageID === "string" ? message.messageID : undefined,
message.sessionID,
typeof message.draftID === "string" ? message.draftID : undefined,
message.providerID,
message.modelID,
message.agent,
message.variant,
parseMessageFiles(message.files),
parseReview(message.review, message.text),
typeof message.agentManagerContext === "string" ? message.agentManagerContext : undefined,
typeof msg.contextDirectory === "string" ? msg.contextDirectory : undefined,
)
await this.sendWebviewMessage(message as SendWebviewMessage)
break
}
case "sendCommand": {
@@ -1462,6 +1472,7 @@ export class KiloProvider implements vscode.WebviewViewProvider, TelemetryProper
break
case "importAndSend": {
const files = parseMessageFiles(message.files)
const feedback = feedbackMessage(message)
void handleImportAndSend(
this.cloudSessionCtx,
message.cloudSessionId,
@@ -1472,9 +1483,10 @@ export class KiloProvider implements vscode.WebviewViewProvider, TelemetryProper
message.agent,
message.variant,
files,
parseReview(message.review, message.text),
feedback?.review,
typeof message.command === "string" ? message.command : undefined,
typeof message.commandArgs === "string" ? message.commandArgs : undefined,
feedback?.browserFeedback,
)
break
}
@@ -1552,6 +1564,25 @@ export class KiloProvider implements vscode.WebviewViewProvider, TelemetryProper
this.webviewMessageDisposable = watchWorkStyleConfig((msg) => this.postMessage(msg), this.webviewMessageDisposable)
}
private async sendWebviewMessage(message: SendWebviewMessage): Promise<void> {
const feedback = feedbackMessage(message)
await this.handleSendMessage(
message.text,
typeof message.messageID === "string" ? message.messageID : undefined,
message.sessionID,
typeof message.draftID === "string" ? message.draftID : undefined,
message.providerID,
message.modelID,
message.agent,
message.variant,
parseMessageFiles(message.files),
feedback?.review,
typeof message.agentManagerContext === "string" ? message.agentManagerContext : undefined,
typeof message.contextDirectory === "string" ? message.contextDirectory : undefined,
feedback?.browserFeedback,
)
}
private async handleProfileDataMessage(message: TypedWebviewMessage): Promise<boolean> {
if (message.type === "refreshProfile") {
await handleRefreshProfile(this.authCtx)
@@ -3987,6 +4018,7 @@ export class KiloProvider implements vscode.WebviewViewProvider, TelemetryProper
review?: ReviewMessageData,
context?: string,
contextDirectory?: string,
browserFeedback?: BrowserFeedbackData,
): Promise<void> {
if (!this.client) {
this.postMessage({
@@ -3998,6 +4030,7 @@ export class KiloProvider implements vscode.WebviewViewProvider, TelemetryProper
messageID,
files,
review,
browserFeedback,
})
return
}
@@ -4019,7 +4052,7 @@ export class KiloProvider implements vscode.WebviewViewProvider, TelemetryProper
parts.push({ type: "file", mime: f.mime, url: f.url, filename: f.filename, source: f.source })
}
}
parts.push({ type: "text", text, metadata: review ? reviewMetadata(review) : undefined })
parts.push({ type: "text", text, metadata: feedbackMetadata(review, browserFeedback) })
const editorContext = await this.gatherEditorContext(dir)
if (draftID && this.closedDrafts.delete(draftID)) {
@@ -4061,6 +4094,7 @@ export class KiloProvider implements vscode.WebviewViewProvider, TelemetryProper
messageID,
files,
review,
browserFeedback,
})
}
}
@@ -114,7 +114,7 @@ function action(
return true
}
void deps.browser
.inspect(m.sessionId, scope.project, point)
.inspect(m.sessionId, scope.project, point, m.hover !== true)
.then((inspection) =>
deps.post({
type: "agentManager.browserInspection",
@@ -20,6 +20,7 @@ import type { ProjectSnapshot } from "./project/contexts"
import type { SidebarTarget } from "./project/route"
import type { TerminalDestination } from "./terminal-destination"
import type { ScriptTerminalView } from "./ScriptTerminalManager"
import type { BrowserFeedbackData } from "../shared/browser-feedback"
export type { TerminalFont }
export type { ProjectSnapshot }
@@ -314,6 +315,7 @@ interface SendInitialMessage {
agent?: string
variant?: string
files?: Array<{ mime: string; url: string }>
browserFeedback?: BrowserFeedbackData
}
interface BranchesMessage {
@@ -469,7 +471,12 @@ interface BrowserInspectionMessage {
sessionId: string
url?: string
title?: string
element?: BrowserElement
element?: BrowserElement & {
hierarchy?: string[]
html?: string
styles?: { color?: string; backgroundColor?: string }
source?: { file: string; line?: number; column?: number }
}
logs: string[]
hover?: boolean
}
@@ -947,6 +954,7 @@ interface SendMessageIn {
files?: Array<{ mime: string; url: string; filename?: string; source?: FileSourceIn }>
agentManagerContext?: string
contextDirectory?: string
browserFeedback?: BrowserFeedbackData
}
interface SendCommandIn {
@@ -9,7 +9,8 @@ import type { KiloClient, Session, TextPartInput, FilePartInput } from "@kilocod
import type { CloudSessionData, EditorContext } from "../../services/cli-backend/types"
import { getErrorMessage, sessionToWebview, mapCloudSessionMessageToWebviewMessage } from "../../kilo-provider-utils"
import type { MessageFile } from "../message-files"
import { reviewMetadata, type ReviewMessageData } from "../../shared/review-comments"
import { type ReviewMessageData } from "../../shared/review-comments"
import { feedbackMetadata, type BrowserFeedbackData } from "../../shared/browser-feedback"
import { completesWithoutStatus } from "../command-completion"
const TIMEOUT = 30_000
@@ -124,6 +125,7 @@ export async function handleImportAndSend(
review?: ReviewMessageData,
command?: string,
commandArgs?: string,
browserFeedback?: BrowserFeedbackData,
): Promise<void> {
if (!ctx.client) {
ctx.postMessage({
@@ -216,7 +218,7 @@ export async function handleImportAndSend(
parts.push({ type: "file", mime: f.mime, url: f.url, filename: f.filename, source: f.source })
}
}
parts.push({ type: "text", text, metadata: review ? reviewMetadata(review) : undefined })
parts.push({ type: "text", text, metadata: feedbackMetadata(review, browserFeedback) })
const editorContext = await ctx.gatherEditorContext()
await client.session.promptAsync(
@@ -247,6 +249,7 @@ export async function handleImportAndSend(
messageID,
files,
review: command ? undefined : review,
browserFeedback: command ? undefined : browserFeedback,
})
}
}
@@ -1,9 +1,11 @@
import { randomBytes, randomUUID, timingSafeEqual } from "node:crypto"
import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http"
import type { Socket } from "node:net"
import { URL } from "node:url"
import { stripVTControlCharacters } from "node:util"
import { chromium, type BrowserContext, type Page } from "playwright-core"
import { BrowserDevtools } from "./browser-devtools"
import { capture as element, locate } from "./browser-element"
export type BrowserStatus = "starting" | "ready" | "loading" | "error" | "closed"
@@ -37,6 +39,10 @@ export interface BrowserElement {
text?: string
selector?: string
rect?: { x: number; y: number; width: number; height: number }
hierarchy?: string[]
html?: string
styles?: { color?: string; backgroundColor?: string }
source?: { file: string; line?: number; column?: number }
}
export interface BrowserInspection {
@@ -183,6 +189,7 @@ export class BrowserBroker {
private readonly token = randomBytes(32).toString("hex")
private owner: ((route: BrowserRoute) => BrowserRoute | undefined) | undefined
private server: Server | undefined
private readonly sockets = new Set<Socket>()
private port: number | undefined
private debugging: number | undefined
private tools: BrowserDevtools | undefined
@@ -201,6 +208,10 @@ export class BrowserBroker {
this.server = createServer((req, res) => {
void this.handle(req, res)
})
this.server.on("connection", (socket) => {
this.sockets.add(socket)
socket.once("close", () => this.sockets.delete(socket))
})
this.server.once("error", reject)
this.server.listen(0, "127.0.0.1", () => {
const server = this.server
@@ -359,47 +370,19 @@ export class BrowserBroker {
sessionId: string,
projectId: string | undefined,
position: { x: number; y: number; width: number; height: number },
detail = true,
): Promise<BrowserInspection> {
return this.serial(this.key(sessionId, projectId), async () => {
this.available()
const entry = this.require(sessionId, undefined, projectId)
await this.point(entry, position)
const element = await entry.page.evaluate(({ x, y }) => {
const root = document.documentElement
if (root.scrollHeight > innerHeight && root.clientWidth === innerWidth) {
root.style.setProperty("scrollbar-gutter", "stable")
}
const node = document.elementFromPoint(x * innerWidth, y * innerHeight)
if (!(node instanceof Element)) return undefined
const tag = node.tagName.toLowerCase()
const id = node.id || undefined
const classes = node.getAttribute("class")?.trim().slice(0, 180) || undefined
const text = (node.textContent ?? "").replace(/\s+/g, " ").trim().slice(0, 400) || undefined
const selector = id ? `#${CSS.escape(id)}` : tag
const rect = node.getBoundingClientRect()
const left = Math.max(0, Math.min(innerWidth, rect.left))
const top = Math.max(0, Math.min(innerHeight, rect.top))
const right = Math.max(left, Math.min(innerWidth, rect.right))
const bottom = Math.max(top, Math.min(innerHeight, rect.bottom))
return {
tag,
id,
classes,
text,
selector,
rect: {
x: left / innerWidth,
y: top / innerHeight,
width: (right - left) / innerWidth,
height: (bottom - top) / innerHeight,
},
}
}, position)
const selected: BrowserElement | undefined = await entry.page.evaluate(element, { ...position, detail })
if (selected?.source) selected.source = await locate(entry.route.directory, selected.source)
await this.update(entry)
return {
url: entry.state.url,
title: entry.state.title,
element,
element: selected,
logs: [...(entry.state.logs ?? [])],
}
})
@@ -469,6 +452,9 @@ export class BrowserBroker {
await new Promise<void>((resolve) => {
if (!this.server) return resolve()
this.server.close(() => resolve())
for (const socket of this.sockets) socket.destroy()
this.sockets.clear()
if (!this.server.listening) resolve()
})
this.server = undefined
this.port = undefined
@@ -749,9 +735,10 @@ export class BrowserBroker {
private authorized(req: IncomingMessage): boolean {
const value = req.headers.authorization
const expected = `Bearer ${this.token}`
if (typeof value !== "string" || value.length !== expected.length) return false
return timingSafeEqual(Buffer.from(value), Buffer.from(expected))
if (typeof value !== "string") return false
const actual = Buffer.from(value)
const expected = Buffer.from(`Bearer ${this.token}`)
return actual.byteLength === expected.byteLength && timingSafeEqual(actual, expected)
}
private status(req: IncomingMessage, res: ServerResponse, route: URL): boolean {
@@ -1,5 +1,5 @@
import { randomBytes, timingSafeEqual } from "node:crypto"
import type { IncomingMessage, Server, ServerResponse } from "node:http"
import { request, type IncomingHttpHeaders, type IncomingMessage, type Server, type ServerResponse } from "node:http"
import type { Duplex } from "node:stream"
import { URL } from "node:url"
import WebSocket, { WebSocketServer, type RawData } from "ws"
@@ -22,7 +22,9 @@ const LIFETIME = 15 * 60 * 1000
const PAYLOAD = 16 * 1024 * 1024
function equal(left: string, right: string): boolean {
return left.length === right.length && timingSafeEqual(Buffer.from(left), Buffer.from(right))
const actual = Buffer.from(left)
const expected = Buffer.from(right)
return actual.byteLength === expected.byteLength && timingSafeEqual(actual, expected)
}
function reject(socket: Duplex, status: number, reason: string): void {
@@ -30,6 +32,30 @@ function reject(socket: Duplex, status: number, reason: string): void {
socket.destroy()
}
function resource(port: number, path: string): Promise<{ status: number; headers: IncomingHttpHeaders; body: Buffer }> {
return new Promise((resolve, reject) => {
const req = request({ hostname: "127.0.0.1", port, path, method: "GET" }, (response) => {
const chunks: Buffer[] = []
let size = 0
response.on("data", (chunk: Buffer) => {
size += chunk.byteLength
if (size > PAYLOAD) {
response.destroy(new Error("Browser developer tools asset exceeds the size limit"))
return
}
chunks.push(chunk)
})
response.once("error", reject)
response.once("end", () =>
resolve({ status: response.statusCode ?? 502, headers: response.headers, body: Buffer.concat(chunks) }),
)
})
req.setTimeout(10_000, () => req.destroy(new Error("Browser developer tools asset request timed out")))
req.once("error", reject)
req.end()
})
}
function inspect(data: RawData): Message | undefined {
const value = Array.isArray(data)
? Buffer.concat(data).toString("utf8")
@@ -104,13 +130,11 @@ export class BrowserDevtools {
res.end(script)
return true
}
const url = new URL(`/devtools/${scope.path}`, `http://127.0.0.1:${scope.target.port}`)
url.search = route.search
try {
const response = await fetch(url, { redirect: "manual" })
const data = Buffer.from(await response.arrayBuffer())
const response = await resource(scope.target.port, `/devtools/${scope.path}${route.search}`)
const data = response.body
const body =
scope.path === "inspector.html" && response.ok
scope.path === "inspector.html" && response.status === 200
? Buffer.from(
data
.toString("utf8")
@@ -126,8 +150,8 @@ export class BrowserDevtools {
"referrer-policy": "no-referrer",
}
for (const name of ["content-type", "content-security-policy"]) {
const value = response.headers.get(name)
if (value) headers[name] = value
const value = response.headers[name]
if (typeof value === "string") headers[name] = value
}
res.writeHead(response.status, headers)
res.end(body)
@@ -0,0 +1,208 @@
import { readFile, realpath, stat } from "node:fs/promises"
import path from "node:path"
function integer(value: number | undefined) {
return typeof value === "number" && Number.isInteger(value) && value > 0 ? value : undefined
}
async function coordinates(file: string, source: { line?: number; column?: number }, size: number) {
const line = integer(source.line)
const text = line && size <= 1_048_576 ? await readFile(file, "utf8").catch(() => undefined) : undefined
const lines = text?.split("\n")
const valid = line && lines && line <= lines.length ? line : undefined
const column = integer(source.column)
return {
line: valid,
column: valid && column && column <= (lines?.[valid - 1]?.length ?? 0) + 1 ? column : undefined,
}
}
export async function locate(directory: string, source?: { file: string; line?: number; column?: number }) {
if (!source?.file || source.file.includes("\0") || /^[a-z][a-z0-9+.-]*:\/\//i.test(source.file)) return undefined
if (!/\.(?:[cm]?[jt]sx?|vue|svelte|html?|css|scss|sass|less)$/i.test(source.file)) return undefined
const candidate = path.resolve(directory, source.file)
const [root, file] = await Promise.all([
realpath(directory).catch(() => undefined),
realpath(candidate).catch(() => undefined),
])
if (!root || !file) return undefined
const relative = path.relative(root, file)
if (!relative || path.isAbsolute(relative) || relative.split(path.sep).includes("..")) return undefined
if (relative.split(path.sep).some((part) => part === "node_modules" || part === ".git")) return undefined
const info = await stat(file).catch(() => undefined)
if (!info?.isFile()) return undefined
return { file: relative.split(path.sep).join("/"), ...(await coordinates(file, source, info.size)) }
}
export function capture(position: { x: number; y: number; detail?: boolean }) {
const root = document.documentElement
if (root.scrollHeight > innerHeight && root.clientWidth === innerWidth) {
root.style.setProperty("scrollbar-gutter", "stable")
}
const node = document.elementFromPoint(position.x * innerWidth, position.y * innerHeight)
if (!(node instanceof Element)) return undefined
const label = (element: Element) => {
const tag = element.tagName.toLowerCase()
if (element.id) return `${tag}#${CSS.escape(element.id.slice(0, 120))}`
const classes = [...element.classList].slice(0, 3).map((item) => `.${CSS.escape(item.slice(0, 60))}`)
return `${tag}${classes.join("")}`
}
const locator = (element: Element) => {
if (element.id && element.id.length <= 180) {
const selector = `#${CSS.escape(element.id)}`
if (document.querySelectorAll(selector).length === 1) return selector
}
for (const name of ["data-testid", "data-test", "data-cy", "aria-label"]) {
const value = element.getAttribute(name)
if (!value || value.length > 160) continue
const selector = `${element.tagName.toLowerCase()}[${name}="${CSS.escape(value)}"]`
if (document.querySelectorAll(selector).length === 1) return selector
}
return undefined
}
const selector = () => {
const direct = locator(node)
if (direct) return direct
const path: string[] = []
let current: Element | null = node
while (current && path.length < 128) {
const anchor = locator(current)
if (anchor) {
path.unshift(anchor)
const value = path.join(" > ")
return value.length <= 2048 && document.querySelectorAll(value).length === 1 ? value : undefined
}
const tag = current.tagName.toLowerCase()
const siblings: Element[] = current.parentElement
? [...current.parentElement.children].filter((item) => item.tagName === current?.tagName)
: []
path.unshift(`${tag}${siblings.length > 1 ? `:nth-of-type(${siblings.indexOf(current) + 1})` : ""}`)
const value = path.join(" > ")
if (value.length > 2048) return undefined
if (document.querySelectorAll(value).length === 1) return value
current = current.parentElement
}
return undefined
}
const identity = selector()
if (!identity) return undefined
const rect = node.getBoundingClientRect()
const left = Math.max(0, Math.min(innerWidth, rect.left))
const top = Math.max(0, Math.min(innerHeight, rect.top))
const right = Math.max(left, Math.min(innerWidth, rect.right))
const bottom = Math.max(top, Math.min(innerHeight, rect.bottom))
const result = {
tag: node.tagName.toLowerCase(),
id: node.id.slice(0, 120) || undefined,
classes: node.getAttribute("class")?.trim().slice(0, 180) || undefined,
selector: identity,
rect: {
x: left / innerWidth,
y: top / innerHeight,
width: (right - left) / innerWidth,
height: (bottom - top) / innerHeight,
},
}
if (position.detail === false) return result
const ancestry: Element[] = []
let parent: Element | null = node
while (parent && ancestry.length < 6) {
ancestry.unshift(parent)
parent = parent.parentElement
}
const hidden = (element: Element) => {
if (
element.matches(
"script,style,noscript,template,input,textarea,select,[hidden],[aria-hidden=true],[contenteditable]:not([contenteditable=false])",
)
) {
return true
}
const style = getComputedStyle(element)
return (
style.display === "none" ||
style.visibility === "hidden" ||
style.visibility === "collapse" ||
style.opacity === "0" ||
style.contentVisibility === "hidden"
)
}
const content = () => {
const editable = node.closest("input,textarea,select,[contenteditable]:not([contenteditable=false])")
if (editable) return editable.getAttribute("aria-label")?.slice(0, 180)
const pending: Node[] = []
let current: Node | null = node.firstChild
let visits = 0
let value = ""
while (current && visits++ < 256 && value.length < 180) {
if (current instanceof Element && !hidden(current) && current.firstChild) {
if (current.nextSibling) pending.push(current.nextSibling)
current = current.firstChild
continue
}
if (current.nodeType === 3) value += (current.textContent ?? "").replace(/\s+/g, " ").slice(0, 180 - value.length)
current = current.nextSibling ?? pending.pop() ?? null
}
return value.trim() || undefined
}
const text = content()
const snippet = node.cloneNode(false) as Element
const allowed = new Set([
"id",
"class",
"role",
"type",
"name",
"aria-label",
"title",
"data-testid",
"data-test",
"data-cy",
])
for (const attribute of [...snippet.attributes]) {
if (!allowed.has(attribute.name)) snippet.removeAttribute(attribute.name)
else snippet.setAttribute(attribute.name, attribute.value.slice(0, 180))
}
snippet.textContent = text ?? ""
const styles = getComputedStyle(node)
const source = () => {
const file = node.getAttribute("data-source-file")
if (file) {
const line = Number(node.getAttribute("data-source-line"))
const column = Number(node.getAttribute("data-source-column"))
return {
file: file.slice(0, 4096),
line: Number.isInteger(line) && line > 0 ? line : undefined,
column: Number.isInteger(column) && column > 0 ? column : undefined,
}
}
const key = Object.keys(node).find(
(item) => item.startsWith("__reactFiber$") || item.startsWith("__reactInternalInstance$"),
)
if (!key) return undefined
type Fiber = {
_debugSource?: { fileName?: string; lineNumber?: number; columnNumber?: number }
_debugOwner?: Fiber
return?: Fiber
}
let fiber: Fiber | undefined = (node as unknown as Record<string, Fiber>)[key]
for (let depth = 0; fiber && depth < 8; depth++) {
const origin = fiber._debugSource
if (origin?.fileName) {
return { file: origin.fileName.slice(0, 4096), line: origin.lineNumber, column: origin.columnNumber }
}
fiber = fiber._debugOwner ?? fiber.return
}
return undefined
}
return {
...result,
text,
hierarchy: ancestry.map(label),
html: snippet.outerHTML.slice(0, 800),
styles: { color: styles.color.slice(0, 80), backgroundColor: styles.backgroundColor.slice(0, 80) },
source: source(),
}
}
@@ -0,0 +1,333 @@
import { partReview, type ReviewMessageData } from "./review-comments"
export interface BrowserReference {
id: string
sessionId: string
selector: string
text?: string
url?: string
title?: string
hierarchy?: string[]
html?: string
styles?: { color?: string; backgroundColor?: string }
source?: { file: string; line?: number; column?: number }
content?: string
}
export interface BrowserFeedbackData {
version: 1
references: BrowserReference[]
}
export interface FeedbackView {
review?: ReviewMessageData
browserFeedback?: BrowserFeedbackData
body: string
}
const REFERENCE_LIMIT = 20
const HIERARCHY_LIMIT = 20
const TOTAL_LIMIT = 200_000
const ID_LIMIT = 512
const SESSION_LIMIT = 512
const SELECTOR_LIMIT = 4_096
const TEXT_LIMIT = 20_000
const URL_LIMIT = 4_096
const TITLE_LIMIT = 2_000
const HIERARCHY_ITEM_LIMIT = 512
const HTML_LIMIT = 20_000
const STYLE_LIMIT = 256
const SOURCE_LIMIT = 4_096
function record(value: unknown): Record<string, unknown> | undefined {
if (!value || typeof value !== "object" || Array.isArray(value)) return undefined
return value as Record<string, unknown>
}
function string(value: unknown, limit: number): string | undefined {
if (typeof value !== "string" || value.length > limit) return undefined
return value
}
function optionalString(value: unknown, limit: number): string | false | undefined {
if (value === undefined) return undefined
const result = string(value, limit)
return result === undefined ? false : result
}
function safePath(value: string): boolean {
const absolute = value.startsWith("/") || value.startsWith("\\") || /^[A-Za-z]:[\\/]/.test(value)
return !absolute && !value.split(/[\\/]/).includes("..") && !value.includes("\0")
}
function url(value: string): string | undefined {
try {
const parsed = new URL(value)
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return undefined
parsed.username = ""
parsed.password = ""
parsed.search = ""
parsed.hash = ""
return parsed.toString()
} catch {
return undefined
}
}
function optionalUrl(value: unknown): string | false | undefined {
if (value === undefined) return undefined
if (typeof value !== "string" || value.length > URL_LIMIT) return false
const result = url(value)
return result === undefined ? false : result
}
function positive(value: unknown): number | false | undefined {
if (value === undefined) return undefined
if (typeof value !== "number" || !Number.isInteger(value) || value < 1) return false
return value
}
function styles(value: unknown): BrowserReference["styles"] | false | undefined {
if (value === undefined) return undefined
const item = record(value)
if (!item) return false
const color = optionalString(item.color, STYLE_LIMIT)
const backgroundColor = optionalString(item.backgroundColor, STYLE_LIMIT)
if (color === false || backgroundColor === false) return false
if (color === undefined && backgroundColor === undefined) return undefined
return {
...(color === undefined ? {} : { color }),
...(backgroundColor === undefined ? {} : { backgroundColor }),
}
}
function source(value: unknown): BrowserReference["source"] | false | undefined {
if (value === undefined) return undefined
const item = record(value)
if (!item) return false
const file = string(item.file, SOURCE_LIMIT)
const line = positive(item.line)
const column = positive(item.column)
if (!file || !safePath(file) || line === false || column === false) return false
return {
file,
...(line === undefined ? {} : { line }),
...(column === undefined ? {} : { column }),
}
}
function hierarchy(value: unknown): string[] | false | undefined {
if (value === undefined) return undefined
if (!Array.isArray(value) || value.length > HIERARCHY_LIMIT) return false
const result = value.map((item) => string(item, HIERARCHY_ITEM_LIMIT))
if (result.some((item) => item === undefined)) return false
return result as string[]
}
function reference(value: unknown): BrowserReference | undefined {
const item = record(value)
if (!item) return undefined
const id = string(item.id, ID_LIMIT)
const sessionId = string(item.sessionId, SESSION_LIMIT)
const selector = string(item.selector, SELECTOR_LIMIT)
if (!id || !sessionId || !selector) return undefined
const text = optionalString(item.text, TEXT_LIMIT)
const pageUrl = optionalUrl(item.url)
const title = optionalString(item.title, TITLE_LIMIT)
const tree = hierarchy(item.hierarchy)
const html = optionalString(item.html, HTML_LIMIT)
const style = styles(item.styles)
const verified = source(item.source)
const content = item.content === undefined ? undefined : optionalString(item.content, 100_000)
if (!valid([text, pageUrl, title, tree, html, style, verified, content])) return undefined
const safeText = unwrap(text)
const safeUrl = unwrap(pageUrl)
const safeTitle = unwrap(title)
const safeTree = unwrap(tree)
const safeHtml = unwrap(html)
const safeStyle = unwrap(style)
const safeSource = unwrap(verified)
const safeContent = unwrap(content)
return {
id,
sessionId,
selector,
...(safeText === undefined ? {} : { text: safeText }),
...(safeUrl === undefined ? {} : { url: safeUrl }),
...(safeTitle === undefined ? {} : { title: safeTitle }),
...(safeTree === undefined ? {} : { hierarchy: safeTree }),
...(safeHtml === undefined ? {} : { html: safeHtml }),
...(safeStyle === undefined ? {} : { styles: safeStyle }),
...(safeSource === undefined ? {} : { source: safeSource }),
...(safeContent === undefined ? {} : { content: safeContent }),
}
}
function valid(
values: Array<string | string[] | BrowserReference["styles"] | BrowserReference["source"] | false | undefined>,
): boolean {
return !values.some((item) => item === false)
}
function unwrap<T>(item: T | false | undefined): T | undefined {
return item === false ? undefined : item
}
function weight(item: BrowserReference): number {
return JSON.stringify(item).length
}
function normalize(references: readonly BrowserReference[]): BrowserReference[] | undefined {
if (references.length === 0 || references.length > REFERENCE_LIMIT) return undefined
const result = references.map(reference)
if (result.some((item) => item === undefined)) return undefined
const list = result as BrowserReference[]
if (list.reduce((total, item) => total + weight(item), 0) > TOTAL_LIMIT) return undefined
return list
}
function escapeInline(value: string): string {
return value.replace(/[\r\n]+/g, " ").replace(/([\\`*_[\]{}()#+\-!|<>])/g, "\\$1")
}
function fenced(value: string): string[] {
const matches = value.match(/`+/g) ?? []
const longest = matches.reduce((max, item) => Math.max(max, item.length), 0)
const fence = "`".repeat(Math.max(3, longest + 1))
return [fence, value, fence]
}
function page(item: BrowserReference): string | undefined {
if (!item.url && !item.title) return undefined
return `Page: ${item.title ? escapeInline(item.title) : "Untitled"}${item.url ? ` (\`${escapeInline(item.url)}\`)` : ""}`
}
function detail(item: BrowserReference, index: number, first: BrowserReference): string[] {
const lines = [`Element ${index + 1}:`, ...fenced(item.selector)]
if (index > 0 && (item.url !== first.url || item.title !== first.title)) {
const value = page(item)
if (value) lines.push(value)
}
if (item.hierarchy?.length) lines.push(...["DOM:", ...fenced(item.hierarchy.join(" > "))])
if (item.text && (!item.html || item.html === item.text)) lines.push(...[`Text:`, ...fenced(item.text)])
if (item.html && item.html !== item.text) lines.push(...[`HTML:`, ...fenced(item.html)])
if (item.styles) {
const values = [
item.styles.color ? `color=${escapeInline(item.styles.color)}` : "",
item.styles.backgroundColor ? `background=${escapeInline(item.styles.backgroundColor)}` : "",
].filter(Boolean)
if (values.length) lines.push(`Styles: ${values.join(", ")}`)
}
if (item.source) {
const location = [item.source.file, item.source.line, item.source.column]
.filter((value) => value !== undefined)
.join(":")
lines.push(...[`Source:`, ...fenced(location)])
}
return lines
}
export function formatBrowserFeedback(references: BrowserReference[]): string {
const list = normalize(references) ?? []
if (list.length === 0) return "## Browser Feedback"
const lines = ["## Browser Feedback", ""]
const heading = page(list[0]!)
if (heading) lines.push(heading, "")
list.forEach((item, index) => lines.push(...detail(item, index, list[0]!), ""))
return lines.join("\n").trimEnd()
}
function view(value: unknown, content: string): { data: BrowserFeedbackData; body: string } | undefined {
const data = record(value)
if (!data || data.version !== 1 || !Array.isArray(data.references)) return undefined
const references = data.references.map(reference)
if (references.length === 0 || references.length > REFERENCE_LIMIT || references.some((item) => item === undefined))
return undefined
const list = (references as BrowserReference[]).map((item) => {
const result = { ...item }
delete result.content
return result
})
if (list.reduce((total, item) => total + weight(item), 0) > TOTAL_LIMIT) return undefined
const prefix = formatBrowserFeedback(list)
if (content === prefix) return { data: { version: 1, references: list }, body: "" }
if (!content.startsWith(`${prefix}\n\n`)) return undefined
return { data: { version: 1, references: list }, body: content.slice(prefix.length + 2) }
}
export function browserFeedbackData(references: BrowserReference[]): BrowserFeedbackData | undefined {
const list = normalize(references)
if (!list) return undefined
return {
version: 1,
references: list.map((item) => {
const result = { ...item }
delete result.content
return result
}),
}
}
export function mergeBrowserReferences(current: BrowserReference[], incoming: BrowserReference): BrowserReference[] {
const selected = browserFeedbackData([incoming])?.references[0]
if (!selected) return current
const kept = current.filter(
(item) => item.id !== selected.id && (item.selector !== selected.selector || item.url !== selected.url),
)
if (kept.length >= REFERENCE_LIMIT) return current
return [...kept, selected]
}
export function browserFeedbackMetadata(data: BrowserFeedbackData): Record<string, unknown> {
return { kilo: { browserFeedback: data } }
}
export function parseBrowserFeedback(value: unknown, content: string): BrowserFeedbackData | undefined {
return view(value, content)?.data
}
export function feedbackMetadata(
review: ReviewMessageData | undefined,
browserFeedback: BrowserFeedbackData | undefined,
): Record<string, unknown> | undefined {
if (!review && !browserFeedback) return undefined
return {
kilo: {
...(review ? { review } : {}),
...(browserFeedback ? { browserFeedback } : {}),
},
}
}
export function partFeedback(metadata: unknown, content: string): FeedbackView | undefined {
const root = record(metadata)
const kilo = record(root?.kilo)
const reviewValue = kilo?.review
const browserValue = kilo?.browserFeedback
let body = content
let review: ReviewMessageData | undefined
let browserFeedback: BrowserFeedbackData | undefined
if (reviewValue !== undefined) {
const parsed = partReview({ kilo: { review: reviewValue } }, body)
if (!parsed) return undefined
review = parsed.data
body = parsed.body
}
if (browserValue !== undefined) {
const parsed = view(browserValue, body)
if (!parsed) return undefined
browserFeedback = parsed.data
body = parsed.body
}
if (!review && !browserFeedback) return undefined
return { review, browserFeedback, body }
}
export function parseFeedback(
metadata: { review?: unknown; browserFeedback?: unknown },
content: string,
): Pick<FeedbackView, "review" | "browserFeedback"> | undefined {
const parsed = partFeedback({ kilo: metadata }, content)
if (!parsed) return undefined
return { review: parsed.review, browserFeedback: parsed.browserFeedback }
}
@@ -29,10 +29,22 @@ describe("Agent Manager diff toggle", () => {
const source = fs.readFileSync(BROWSER_PANEL, "utf-8")
expect(source).toContain("props.state?.navigation")
expect(source).toContain("when={identity()}")
expect(source).toContain("frame?.contentWindow?.location.replace(props.url)")
expect(source).toContain('type: "agentManager.browser.input"')
expect(source).toContain("if (pointing()) input(value, false)")
})
it("keeps browser chrome compact with one close action and no duplicate footer", () => {
const source = fs.readFileSync(BROWSER_PANEL, "utf-8")
expect(source).toContain('class="am-browser-address"')
expect(source).toContain('icon="arrow-right"')
expect(source).toContain('icon="window-cursor"')
expect(source.match(/icon="close"/g)).toHaveLength(1)
expect(source).not.toContain("am-browser-footer")
expect(source).not.toContain("am-browser-selected")
expect(source).not.toContain("am-browser-devtools-toolbar")
})
it("renders live Git stats rather than pull-request stats", () => {
const source = fs.readFileSync(TAB_BAR, "utf-8")
const start = source.indexOf('title={props.t("agentManager.diff.toggle")}')
@@ -1,7 +1,10 @@
import { afterEach, describe, expect, test } from "bun:test"
import { createServer, request } from "node:http"
import { createServer, request, type IncomingMessage } from "node:http"
import { connect } from "node:net"
import { PassThrough } from "node:stream"
import WebSocket, { WebSocketServer } from "ws"
import { BrowserBroker, diagnostic } from "../../src/services/browser-automation/browser-broker"
import { BrowserDevtools } from "../../src/services/browser-automation/browser-devtools"
const brokers: BrowserBroker[] = []
@@ -146,6 +149,10 @@ describe("BrowserBroker", () => {
})
expect(result.status).toBe(401)
expect(JSON.parse(result.body)).toEqual({ error: "Unauthorized" })
const malformed = await fetch(`${env.KILO_BROWSER_BROKER_URL}/browser/status`, {
headers: { authorization: `Bearer ${"é".repeat(64)}` },
})
expect(malformed.status).toBe(401)
})
test("reports experimental availability only to authenticated clients", async () => {
@@ -164,6 +171,33 @@ describe("BrowserBroker", () => {
expect(await (await fetch(url, { headers })).json()).toEqual({ enabled: false })
})
test("writes an explicit forbidden response before closing an untrusted upgrade", () => {
const server = createServer()
const tools = new BrowserDevtools(
server,
4567,
() => {},
() => {},
)
const url = new URL(tools.open("browser", "page", 1234, "dark"))
const endpoint = new URL(`ws://${url.searchParams.get("ws")}`)
const socket = new PassThrough()
const chunks: Buffer[] = []
socket.on("data", (chunk) => chunks.push(chunk))
server.emit(
"upgrade",
{
url: endpoint.pathname,
headers: { host: "127.0.0.1:4567", origin: "http://untrusted.invalid" },
} as IncomingMessage,
socket,
Buffer.alloc(0),
)
expect(Buffer.concat(chunks).toString()).toStartWith("HTTP/1.1 403 Forbidden\r\n")
expect(socket.destroyed).toBe(true)
tools.dispose()
})
test("proxies page-scoped developer tools and rejects invalid capabilities or origins", async () => {
const remote = createServer((req, res) => {
const path = new URL(req.url ?? "/", "http://127.0.0.1").pathname
@@ -270,11 +304,29 @@ describe("BrowserBroker", () => {
const endpoint = `ws://${new URL(first.url).searchParams.get("ws")}`
const forbidden = await new Promise<number>((resolve, reject) => {
const socket = new WebSocket(endpoint, { headers: { origin: "http://untrusted.invalid" } })
socket.once("unexpected-response", (_request, response) => resolve(response.statusCode ?? 0))
const url = new URL(endpoint)
const socket = connect({ host: url.hostname, port: Number(url.port) }, () => {
socket.write(
[
`GET ${url.pathname} HTTP/1.1`,
`Host: ${url.host}`,
"Connection: Upgrade",
"Upgrade: websocket",
"Sec-WebSocket-Version: 13",
`Sec-WebSocket-Key: ${Buffer.from("browser-test-key").toString("base64")}`,
"Origin: http://untrusted.invalid",
"\r\n",
].join("\r\n"),
)
})
socket.once("data", (data) => {
resolve(Number(data.toString().match(/^HTTP\/1\.1 (\d+)/)?.[1] ?? 0))
socket.end()
})
socket.once("end", () => resolve(0))
socket.once("error", reject)
})
expect(forbidden).toBe(403)
expect([0, 403]).toContain(forbidden)
const socket = new WebSocket(endpoint, { headers: { origin: new URL(first.url).origin } })
await new Promise<void>((resolve, reject) => {
@@ -0,0 +1,155 @@
import { afterEach, describe, expect, test } from "bun:test"
import { mkdtemp, mkdir, rm, symlink, writeFile } from "node:fs/promises"
import os from "node:os"
import path from "node:path"
import { Window } from "happy-dom"
import { capture, locate } from "../../src/services/browser-automation/browser-element"
const windows: Window[] = []
afterEach(async () => {
await Promise.all(windows.splice(0).map((window) => window.happyDOM.close()))
})
function inspect(html: string, selector: string, detail = true) {
const window = new Window({ url: "http://localhost:3000/" })
windows.push(window)
window.document.body.innerHTML = html
const node = window.document.querySelector(selector)
if (!node) throw new Error("Selected test element is missing")
Object.defineProperty(window.document, "elementFromPoint", { value: () => node })
const run = new Function(
"document",
"Element",
"CSS",
"innerWidth",
"innerHeight",
"getComputedStyle",
`return (${capture.toString()})(${JSON.stringify({ x: 0.5, y: 0.5, detail })})`,
)
const result = run(
window.document,
window.Element,
window.CSS,
window.innerWidth,
window.innerHeight,
window.getComputedStyle.bind(window),
) as ReturnType<typeof capture> | undefined
if (!result) throw new Error("Browser element capture returned no element")
return { result, node, document: window.document }
}
describe("browser element context", () => {
test("builds a unique selector and bounded ancestry for repeated buttons without ids", () => {
const selected = inspect(
'<main id="app"><section class="hero"><div class="actions"><button class="primary">Save</button><button class="primary">Cancel</button></div></section><section><button class="primary">Save</button></section></main>',
".actions button:first-child",
)
expect(selected.document.querySelectorAll(selected.result.selector)).toHaveLength(1)
expect(selected.document.querySelector(selected.result.selector)).toBe(selected.node)
expect(selected.result.hierarchy).toEqual([
"html",
"body",
"main#app",
"section.hero",
"div.actions",
"button.primary",
])
expect(selected.result.html).toBe('<button class="primary">Save</button>')
})
test("prefers stable test ids and does not trust duplicate element ids", () => {
const stable = inspect('<button data-testid="checkout">Pay</button><button>Pay</button>', "[data-testid]")
expect(stable.result.selector).toBe('button[data-testid="checkout"]')
const duplicate = inspect(
'<div><button id="duplicate">One</button><button id="duplicate">Two</button></div>',
"button:last-child",
)
expect(duplicate.result.selector).not.toBe("#duplicate")
expect(duplicate.document.querySelectorAll(duplicate.result.selector)).toHaveLength(1)
expect(duplicate.document.querySelector(duplicate.result.selector)).toBe(duplicate.node)
})
test("excludes scripts, handlers, arbitrary attributes, hidden text, and input values", () => {
const selected = inspect(
'<button id="save" onclick="secret()" data-api-key="secret-token" aria-label="Save"><span>Save</span><script>secret-code</script><input value="secret-input"><textarea>secret-textarea</textarea><span hidden>secret-hidden</span><span style="display:none">secret-css</span><span style="visibility:hidden">secret-invisible</span><span style="opacity:0">secret-transparent</span><span contenteditable>secret-editable</span></button>',
"#save",
)
expect(selected.result.text).toBe("Save")
expect(selected.result.html).toBe('<button id="save" aria-label="Save">Save</button>')
expect(JSON.stringify(selected.result)).not.toContain("secret")
const password = inspect('<input type="password" value="private-password" aria-label="Password">', "input")
expect(password.result.text).toBe("Password")
expect(JSON.stringify(password.result)).not.toContain("private-password")
})
test("provides relevant colors and keeps hover responses lightweight", () => {
const selected = inspect('<button style="color: white; background-color: rgb(22, 163, 74)">Save</button>', "button")
expect(selected.result.styles?.backgroundColor).toBe("rgb(22, 163, 74)")
expect(selected.result.html).not.toContain("style=")
const hover = inspect('<button id="save">Save</button>', "button", false)
expect(hover.result.selector).toBe("#save")
expect(hover.result).not.toHaveProperty("html")
expect(hover.result).not.toHaveProperty("hierarchy")
expect(hover.result).not.toHaveProperty("source")
})
test("bounds text and HTML instead of copying the entire document", () => {
const selected = inspect(
`<main><button class="${"a".repeat(500)}">${"Save ".repeat(500)}</button></main>`,
"button",
)
expect(selected.result.text?.length).toBeLessThanOrEqual(180)
expect(selected.result.html?.length).toBeLessThanOrEqual(800)
expect(selected.result.classes?.length).toBeLessThanOrEqual(180)
})
test("does not fabricate ambiguous selectors for deeply repeated structures", () => {
const nested = (depth: number) =>
`<section>${"<div>".repeat(depth)}<button>Save</button>${"</div>".repeat(depth)}</section>`
const selected = inspect(nested(26).repeat(2), "section:nth-of-type(2) button")
expect(selected.document.querySelectorAll(selected.result.selector)).toHaveLength(1)
expect(selected.document.querySelector(selected.result.selector)).toBe(selected.node)
expect(() => inspect(nested(150).repeat(2), "section:nth-of-type(2) button")).toThrow(
"Browser element capture returned no element",
)
})
test("bounds traversal before reading a large selected subtree", () => {
const selected = inspect(
`<main id="large">${"<span></span>".repeat(300)}<span>late-private-text</span></main>`,
"main",
)
expect(selected.result.text).toBeUndefined()
expect(selected.result.html).toBe('<main id="large"></main>')
})
test("accepts only existing source files within the owning workspace", async () => {
const root = await mkdtemp(path.join(os.tmpdir(), "kilo-browser-source-"))
try {
const project = path.join(root, "project")
await mkdir(path.join(project, "src"), { recursive: true })
await writeFile(path.join(project, "src", "Button.tsx"), "export const Button = () => {\n\n\n return null\n}")
await writeFile(path.join(root, "private.ts"), "export const privateValue = true")
await writeFile(path.join(project, ".env"), "SECRET=value")
await symlink(path.join(root, "private.ts"), path.join(project, "src", "external.ts"))
expect(await locate(project, { file: "src/Button.tsx", line: 4, column: 2 })).toEqual({
file: "src/Button.tsx",
line: 4,
column: 2,
})
expect(await locate(project, { file: "src/Button.tsx", line: 999, column: 2 })).toEqual({
file: "src/Button.tsx",
line: undefined,
column: undefined,
})
expect(await locate(project, { file: "src/missing.tsx" })).toBeUndefined()
expect(await locate(project, { file: "../private.ts" })).toBeUndefined()
expect(await locate(project, { file: "src/external.ts" })).toBeUndefined()
expect(await locate(project, { file: ".env" })).toBeUndefined()
expect(await locate(project, { file: "https://example.com/Button.tsx" })).toBeUndefined()
} finally {
await rm(root, { recursive: true, force: true })
}
})
})
@@ -0,0 +1,114 @@
import { describe, expect, it } from "bun:test"
import {
browserFeedbackData,
browserFeedbackMetadata,
formatBrowserFeedback,
mergeBrowserReferences,
partFeedback,
parseBrowserFeedback,
type BrowserReference,
} from "../../src/shared/browser-feedback"
import { formatReviewCommentsMarkdown } from "../../webview-ui/src/utils/review-comment-markdown"
const reference = (overrides: Partial<BrowserReference> = {}): BrowserReference => ({
id: "browser-1",
sessionId: "session-1",
selector: "main > button.save",
url: "https://user:secret@example.com/app?token=private#section",
title: "Settings",
hierarchy: ["main", "button.save"],
text: "Save settings",
html: '<button class="save">Save settings</button>',
styles: { color: "rgb(1, 2, 3)", backgroundColor: "white" },
source: { file: "src/settings.tsx", line: 42, column: 7 },
content: "legacy dump and bounds",
...overrides,
})
describe("browser feedback formatter", () => {
it("formats grounded fields and omits legacy content and bounds", () => {
const text = formatBrowserFeedback([reference()])
expect(text).toContain("Page: Settings")
expect(text).toContain("https://example.com/app")
expect(text).toContain("main > button.save")
expect(text).toContain("Save settings")
expect(text).toContain("src/settings.tsx:42:7")
expect(text).not.toContain("secret")
expect(text).not.toContain("token")
expect(text).not.toContain("legacy dump")
expect(text).not.toContain("Bounds")
})
it("keeps equivalent text and html from duplicating context", () => {
const text = formatBrowserFeedback([reference({ html: "Save settings" })])
expect(text.match(/Save settings/g)?.length).toBe(1)
expect(text).not.toContain("HTML:")
})
it("includes readable text only once when a safe HTML snippet already contains it", () => {
const text = formatBrowserFeedback([reference()])
expect(text.match(/Save settings/g)).toHaveLength(1)
expect(text).not.toContain("Text:")
})
it("updates repeated selections without duplicating the same page element", () => {
const first = browserFeedbackData([reference()])!.references
const merged = mergeBrowserReferences(first, reference({ id: "new-selection", text: "Updated settings" }))
expect(merged).toHaveLength(1)
expect(merged[0]?.id).toBe("new-selection")
expect(merged[0]?.text).toBe("Updated settings")
expect(merged[0]?.url).toBe("https://example.com/app")
expect(merged[0]).not.toHaveProperty("content")
})
it("rejects invalid and oversized references", () => {
expect(browserFeedbackData([])).toBeUndefined()
expect(browserFeedbackData([reference({ selector: "x".repeat(5_000) })])).toBeUndefined()
expect(browserFeedbackData(Array.from({ length: 21 }, (_, id) => reference({ id: String(id) })))).toBeUndefined()
expect(browserFeedbackData([reference({ source: { file: "../secret" } })])).toBeUndefined()
expect(browserFeedbackData([reference({ url: "file:///tmp/private" })])).toBeUndefined()
expect(browserFeedbackData([reference({ text: "x".repeat(20_001) })])).toBeUndefined()
})
})
describe("browser feedback metadata", () => {
it("round-trips metadata while ignoring legacy content", () => {
const data = browserFeedbackData([reference()])!
const prefix = formatBrowserFeedback(data.references)
expect(parseBrowserFeedback(data, `${prefix}\n\nFix the save action`)).toEqual(data)
expect(partFeedback(browserFeedbackMetadata(data), `${prefix}\n\nFix the save action`)).toEqual({
browserFeedback: data,
body: "Fix the save action",
})
})
it("rejects arbitrary text that does not match the metadata prefix", () => {
const data = browserFeedbackData([reference()])!
expect(parseBrowserFeedback(data, "unrelated text")).toBeUndefined()
})
it("coexists with local and PR review metadata", () => {
const review = {
version: 1 as const,
comments: [
{
id: "review-1",
file: "src/app.ts",
side: "additions" as const,
line: 3,
comment: "Keep this branch safe",
selectedText: "return value",
},
],
}
const browser = browserFeedbackData([reference()])!
const reviewPrefix = formatReviewCommentsMarkdown(review.comments)
const browserPrefix = formatBrowserFeedback(browser.references)
const content = `${reviewPrefix}\n\n${browserPrefix}\n\nDo both`
expect(partFeedback({ kilo: { review, browserFeedback: browser } }, content)).toEqual({
review,
browserFeedback: browser,
body: "Do both",
})
})
})
@@ -11,28 +11,32 @@ import {
isPendingSend,
promotePendingDraftDiscard,
reviewDrafts,
browserDrafts,
savePromptDraft,
scrollDrafts,
finishPendingSend,
} from "../../webview-ui/src/utils/draft-store"
const stores = [drafts, reviewDrafts, imageDrafts, scrollDrafts]
const stores = [drafts, browserDrafts, reviewDrafts, imageDrafts, scrollDrafts]
beforeEach(() => stores.forEach((store) => store.clear()))
describe("prompt draft storage", () => {
it("stores and clears all prompt artifacts together", () => {
const browser = [{ id: "browser", sessionId: "s1", selector: "#save", content: "legacy" }]
savePromptDraft(
"prompt:default:pending:sidebar-pending:1",
"draft",
[{ id: "review", file: "a.ts", side: "additions", line: 1, comment: "comment", selectedText: "line" }],
[{ id: "image", filename: "a.png", mime: "image/png", dataUrl: "data:image/png;base64,a" }],
42,
browser,
)
expect(drafts.size).toBe(1)
expect(reviewDrafts.size).toBe(1)
expect(imageDrafts.size).toBe(1)
expect(browserDrafts.get("prompt:default:pending:sidebar-pending:1")).toEqual(browser)
expect(scrollDrafts.size).toBe(1)
discardPendingDraft("sidebar-pending:1")
@@ -174,20 +174,25 @@ describe("movePromptDraft", () => {
const comments = new Map([[source, [comment]]])
const images = new Map([[source, [image]]])
const scrolls = new Map([[source, 128]])
const browser = new Map([[source, [{ id: "browser-1", sessionId: "session-1", selector: "#save" }]]])
const expected = browser.get(source)
expect(movePromptDraft({ text, comments, images, scrolls }, source, target)).toEqual({
expect(movePromptDraft({ text, comments, images, scrolls, browsers: browser }, source, target)).toEqual({
text: "Keep this prompt",
comments: [comment],
images: [image],
scroll: 128,
browsers: expected,
})
expect(text.get(target)).toBe("Keep this prompt")
expect(comments.get(target)).toEqual([comment])
expect(images.get(target)).toEqual([image])
expect(browser.get(target)).toEqual([{ id: "browser-1", sessionId: "session-1", selector: "#save" }])
expect(scrolls.get(target)).toBe(128)
expect(text.has(source)).toBe(false)
expect(comments.has(source)).toBe(false)
expect(images.has(source)).toBe(false)
expect(browser.has(source)).toBe(false)
expect(scrolls.has(source)).toBe(false)
})
})
@@ -14,7 +14,7 @@ describe("PromptInput connection guard", () => {
const attachments = src.indexOf("const gitFile = await git.resolveAttachment")
const guard = src.indexOf("if (isDisabled()) {", attachments)
const finish = src.indexOf("finishPending(pendingId)", guard)
const send = src.indexOf("session.sendMessage(message", guard)
const send = src.indexOf("session.sendMessage(", guard)
const clear = src.indexOf("drafts.delete(key)", send)
expect(attachments).toBeGreaterThan(-1)
@@ -49,8 +49,8 @@ describe("PromptInput sandbox toggle", () => {
})
it("captures edits made while sandbox session creation is pending", () => {
const start = src.indexOf('if (message.type === "sessionCreated")')
const end = src.indexOf('if (message.type === "action"', start)
const start = src.indexOf("const created = (message:")
const end = src.indexOf("const unsubscribe", start)
const created = src.slice(start, end)
const save = created.indexOf(
"if (source === draftKey()) saveDraft(source, text(), reviewComments(), imageAttach.images())",
@@ -61,7 +61,10 @@ describe("PromptInput sandbox toggle", () => {
expect(end).toBeGreaterThan(start)
expect(save).toBeGreaterThan(-1)
expect(move).toBeGreaterThan(save)
expect(created).toContain("{ text: drafts, comments: reviewDrafts, images: imageDrafts, scrolls: scrollDrafts }")
expect(created).toContain(
"{ text: drafts, comments: reviewDrafts, images: imageDrafts, scrolls: scrollDrafts, browsers: references }",
)
expect(created).toContain("saveDraft(source, text(), reviewComments(), imageAttach.images())")
})
it("restores each prompt draft's textarea and highlight scroll positions", () => {
@@ -70,8 +73,14 @@ describe("PromptInput sandbox toggle", () => {
expect(src).toContain("textareaRef.scrollTop = scroll")
expect(src).toContain("if (highlightRef) highlightRef.scrollTop = scroll")
expect(src).toContain("scrollDrafts.set(draftKey(), textareaRef.scrollTop)")
expect(src).toContain("images: imageAttach.images(),\n scroll: textareaRef?.scrollTop")
expect(src).toContain("draft.text, draft.comments, draft.images, draft.scroll")
expect(src).toContain(
"images: imageAttach.images(),\n browsers: browsers(),\n scroll: textareaRef?.scrollTop",
)
expect(src).toContain("draft.text,")
expect(src).toContain("draft.comments,")
expect(src).toContain("draft.images,")
expect(src).toContain("draft.scroll,")
expect(src).toContain("draft.browsers")
})
it("tracks in-flight toggles per session while switching", () => {
@@ -387,8 +387,9 @@ describe("PromptInput send origin contract", () => {
})
it("passes the captured origin to message and command sends", () => {
expect(source).toMatch(/session\.sendMessage\([\s\S]*origin \?\? null\)/)
expect(source).toMatch(/session\.sendCommand\([\s\S]*origin \?\? null\)/)
expect(source).toMatch(/session\.sendMessage\([\s\S]*origin \?\? null[\s\S]*browserData[\s\S]*\)/)
const command = source.slice(source.indexOf("session.sendCommand("))
expect(command).toMatch(/origin \?\? null[\s\S]*\{[\s\S]*agent: matched\.agent/)
})
it("records sent prompts before a pending session key change can return", () => {
@@ -665,18 +666,31 @@ describe("browser element reference contract", () => {
})
it("includes browser reference content only when the user sends the prompt", () => {
expect(source).toMatch(/const browser = browsers\(\)[\s\S]*?\.map\(\(item\) => item\.content\)/)
expect(source).toContain('const message = [review, browser, draft].filter(Boolean).join("\\n\\n")')
expect(source).toContain("browserFeedbackData(browsers())")
expect(source).toContain("formatBrowserFeedback(browserData.references)")
expect(source).toContain('const message = [review, browserText, draft].filter(Boolean).join("\\n\\n")')
expect(source).toContain("references.delete(key)")
})
it("restores browser attachments for the correct session and allows attachment-only sends", () => {
expect(source).toContain("setBrowsers(references.get(key) ?? [])")
expect(source).toContain("reference.sessionId === sid()")
expect(source).toContain("if (reference.sessionId !== sid()) return")
expect(source).toContain("mergeBrowserReferences(browsers(), reference)")
expect(source).toContain("browsers().length > 0")
})
})
describe("sent browser feedback rendering contract", () => {
const message = readFile(path.join(ROOT, "webview-ui/src/components/chat/VscodeUserMessage.tsx"))
it("renders validated browser metadata as cards and exposes only the instruction body", () => {
expect(message).toContain("partFeedback")
expect(message).toContain("BrowserReferences")
expect(message).toContain("feedback()?.body")
expect(message).not.toContain("item.content")
})
})
describe("KiloConnectionService pruneSession contract", () => {
const source = readFile(CONNECTION_SERVICE_FILE)
@@ -26,6 +26,7 @@ import {
revertPromptState,
} from "../../webview-ui/src/context/session-utils"
import type { Message, Part, ToolPart } from "../../webview-ui/src/types/messages"
import { formatBrowserFeedback } from "../../src/shared/browser-feedback"
const t = (key: string) => key
@@ -1030,6 +1031,24 @@ describe("revertPromptState", () => {
it("returns empty collections for tool-only messages", () => {
const part: Part = { type: "tool", id: "p1", tool: "bash", state: { status: "running", input: {} } }
const state = revertPromptState([part])
expect(state).toEqual({ text: "", paths: [], sessions: [], images: [] })
expect(state).toEqual({ text: "", paths: [], sessions: [], images: [], review: [], browser: [] })
})
it("restores browser and review metadata without their formatted prefixes", () => {
const browser = {
version: 1 as const,
references: [{ id: "b", sessionId: "s1", selector: "#save", text: "Save" }],
}
const content = `${formatBrowserFeedback(browser.references)}\n\nPlease update it`
const part: Part = {
type: "text",
id: "t1",
text: content,
metadata: { kilo: { browserFeedback: browser } },
}
expect(revertPromptState([part])).toMatchObject({
text: "Please update it",
browser: browser.references,
})
})
})
@@ -1,11 +1,24 @@
import { createEffect, createSignal, For, onCleanup, Show, type Accessor, type Component, type Setter } from "solid-js"
import { Button } from "@kilocode/kilo-ui/button"
import {
createEffect,
createSignal,
For,
on,
onCleanup,
Show,
type Accessor,
type Component,
type Setter,
} from "solid-js"
import { Icon } from "@kilocode/kilo-ui/icon"
import { IconButton } from "@kilocode/kilo-ui/icon-button"
import { Spinner } from "@kilocode/kilo-ui/spinner"
import { TextField } from "@kilocode/kilo-ui/text-field"
import { Tooltip } from "@kilocode/kilo-ui/tooltip"
import { useLanguage } from "../src/context/language"
import { useVSCode } from "../src/context/vscode"
import type { AgentManagerBrowserInspectionMessage, ExtensionMessage, WebviewMessage } from "../src/types/messages"
import { SidePanel } from "./side-panel-layout"
import { formatBrowserFeedback } from "../../src/shared/browser-feedback"
export function createBrowserPanel(
current: Accessor<SidePanel | null>,
@@ -52,30 +65,6 @@ type Inspection = AgentManagerBrowserInspectionMessage
type Position = { x: number; y: number; width: number; height: number }
type Pointer = MouseEvent & { currentTarget: HTMLButtonElement }
function feedback(message: Inspection): string {
const element = message.element
const attrs = [
element?.id ? `id="${element.id}"` : undefined,
element?.classes ? `class="${element.classes}"` : undefined,
]
.filter(Boolean)
.join(" ")
return [
"Browser feedback",
message.url ? `URL: ${message.url}` : undefined,
message.title ? `Page: ${message.title}` : undefined,
element ? `Selected element: <${element.tag}${attrs ? ` ${attrs}` : ""}>` : "Selected element: unavailable",
element?.selector ? `Selector: ${element.selector}` : undefined,
element?.rect
? `Bounds: x=${element.rect.x.toFixed(3)}, y=${element.rect.y.toFixed(3)}, width=${element.rect.width.toFixed(3)}, height=${element.rect.height.toFixed(3)}`
: undefined,
element?.text ? `Visible text: ${element.text}` : undefined,
message.logs.length ? `Console diagnostics:\n${message.logs.map((line) => `- ${line}`).join("\n")}` : undefined,
]
.filter(Boolean)
.join("\n")
}
function position(event: Pointer): Position {
const bounds = event.currentTarget.getBoundingClientRect()
return {
@@ -88,10 +77,13 @@ function position(event: Pointer): Position {
const Toolbar: Component<{
url: string
title?: string
active: boolean
selecting: boolean
docked: boolean
ready: boolean
loading: boolean
errors: number
onUrl: (value: string) => void
onOpen: () => void
onSelect: () => void
@@ -100,55 +92,95 @@ const Toolbar: Component<{
onClose: () => void
}> = (props) => {
const t = useLanguage().t
const diagnostics = () =>
props.errors
? `${t("agentManager.browser.devtoolsTitle")}, ${t("agentManager.browser.errors", { count: props.errors })}`
: t("agentManager.browser.devtoolsTitle")
return (
<div class="am-browser-toolbar">
<TextField
class="am-browser-url"
value={props.url}
onChange={props.onUrl}
placeholder={t("agentManager.browser.urlPlaceholder")}
aria-label={t("agentManager.browser.url")}
onKeyDown={(event: KeyboardEvent) => {
if (event.key === "Enter") props.onOpen()
<Tooltip value={t("agentManager.browser.refresh")} placement="bottom">
<IconButton
icon="refresh"
size="small"
variant="ghost"
aria-label={t("agentManager.browser.refresh")}
onClick={props.onRefresh}
disabled={!props.ready || props.loading}
/>
</Tooltip>
<form
class="am-browser-address"
title={props.title}
onSubmit={(event) => {
event.preventDefault()
if (props.active && props.url.trim() && !props.loading) props.onOpen()
}}
/>
<Button size="small" variant="primary" disabled={!props.url.trim() || !props.active} onClick={props.onOpen}>
{t("agentManager.browser.open")}
</Button>
<Button
size="small"
variant="secondary"
aria-pressed={props.selecting}
disabled={!props.ready}
onClick={props.onSelect}
>
{t("agentManager.browser.inspect")}
</Button>
<IconButton
icon="console"
size="small"
variant="ghost"
aria-label={t("agentManager.browser.devtools")}
aria-pressed={props.docked}
onClick={props.onDevtools}
disabled={!props.ready}
/>
<IconButton
icon="refresh"
size="small"
variant="ghost"
aria-label={t("agentManager.browser.refresh")}
onClick={props.onRefresh}
disabled={!props.ready}
/>
<IconButton
icon="close"
size="small"
variant="ghost"
aria-label={t("agentManager.browser.close")}
onClick={props.onClose}
disabled={!props.ready}
/>
<span class="am-browser-site" aria-hidden="true">
<Show when={props.loading} fallback={<Icon name="globe" size="small" />}>
<Spinner />
</Show>
</span>
<TextField
class="am-browser-url"
variant="ghost"
value={props.url}
onChange={props.onUrl}
placeholder={t("agentManager.browser.urlPlaceholder")}
aria-label={t("agentManager.browser.url")}
spellcheck={false}
autocomplete="off"
onFocus={(event: FocusEvent & { currentTarget: HTMLInputElement }) => event.currentTarget.select()}
/>
<Tooltip value={t("agentManager.browser.open")} placement="bottom">
<IconButton
type="submit"
icon="arrow-right"
size="small"
variant="ghost"
aria-label={t("agentManager.browser.open")}
disabled={!props.url.trim() || !props.active || props.loading}
/>
</Tooltip>
</form>
<Tooltip value={t("agentManager.browser.inspect")} placement="bottom">
<IconButton
icon="window-cursor"
size="small"
variant={props.selecting ? "secondary" : "ghost"}
aria-label={t("agentManager.browser.inspect")}
aria-pressed={props.selecting}
disabled={!props.ready || props.loading}
onClick={props.onSelect}
/>
</Tooltip>
<div class="am-browser-tools-action">
<Tooltip value={diagnostics()} placement="bottom">
<IconButton
icon="console"
size="small"
variant={props.docked ? "secondary" : "ghost"}
aria-label={diagnostics()}
aria-pressed={props.docked}
onClick={props.onDevtools}
disabled={!props.ready || props.loading}
/>
</Tooltip>
<Show when={props.errors > 0}>
<span class="am-browser-error-count" aria-hidden="true">
{props.errors > 99 ? "99+" : props.errors}
</span>
</Show>
</div>
<Tooltip value={t("agentManager.browser.close")} placement="bottom">
<IconButton
icon="close"
size="small"
variant="ghost"
aria-label={t("agentManager.browser.close")}
onClick={props.onClose}
/>
</Tooltip>
</div>
)
}
@@ -191,6 +223,30 @@ const Picker: Component<{
)
}
const Preview: Component<{ url: string; navigation: number }> = (props) => {
const t = useLanguage().t
let frame: HTMLIFrameElement | undefined
createEffect(
on(
() => props.navigation,
(value, previous) => {
if (previous === undefined || value === previous) return
frame?.contentWindow?.location.replace(props.url)
},
),
)
return (
<iframe
ref={frame}
class="am-browser-frame"
src={props.url}
title={t("agentManager.browser.screenshotAlt")}
sandbox="allow-scripts allow-forms allow-same-origin"
referrerpolicy="no-referrer"
/>
)
}
const Viewport: Component<{
state?: State
session?: string
@@ -208,7 +264,7 @@ const Viewport: Component<{
props.state.url
const identity = () => {
const url = page()
return url ? `${props.state?.browserId}:${props.state?.navigation ?? 0}:${url}` : undefined
return url ? `${props.state?.browserId}:${url}` : undefined
}
return (
<div class="am-browser-viewport" aria-live="polite">
@@ -221,15 +277,7 @@ const Viewport: Component<{
</div>
}
>
{(_key) => (
<iframe
class="am-browser-frame"
src={props.state?.url}
title={t("agentManager.browser.screenshotAlt")}
sandbox="allow-scripts allow-forms allow-same-origin"
referrerpolicy="no-referrer"
/>
)}
{(_key) => <Preview url={props.state?.url ?? ""} navigation={props.state?.navigation ?? 0} />}
</Show>
<Picker
active={props.selecting && !!props.state?.url}
@@ -248,14 +296,10 @@ const Viewport: Component<{
)
}
const Tools: Component<{ url: string; onClose: () => void }> = (props) => {
const Tools: Component<{ url: string }> = (props) => {
const t = useLanguage().t
return (
<section class="am-browser-devtools" aria-label={t("agentManager.browser.devtoolsTitle")}>
<div class="am-browser-devtools-toolbar">
<span>{t("agentManager.browser.devtoolsTitle")}</span>
<IconButton icon="close" size="small" variant="ghost" aria-label={t("common.close")} onClick={props.onClose} />
</div>
<iframe
class="am-browser-devtools-frame"
src={props.url}
@@ -281,24 +325,6 @@ const Diagnostics: Component<{ logs: string[] }> = (props) => (
</Show>
)
const Footer: Component<{ state?: State; selected?: Inspection; onClose: () => void }> = (props) => {
const t = useLanguage().t
return (
<div class="am-browser-footer">
<span>{props.state?.url || t("agentManager.browser.localOnly")}</span>
<Show when={props.selected?.element?.selector}>
{(selector) => <span class="am-browser-selected">{selector()}</span>}
</Show>
<Show when={props.state?.errors}>
{(errors) => <span class="am-browser-errors">{t("agentManager.browser.errors", { count: errors() })}</span>}
</Show>
<Button size="small" variant="ghost" onClick={props.onClose}>
{t("agentManager.browser.hide")}
</Button>
</div>
)
}
interface Props {
sessionId: Accessor<string | undefined>
projectId: Accessor<string | undefined>
@@ -312,7 +338,6 @@ const BrowserPanel: Component<Props> = (props) => {
const [selecting, setSelecting] = createSignal(false)
const [pointing, setPointing] = createSignal(false)
const [hovered, setHovered] = createSignal<Inspection>()
const [selected, setSelected] = createSignal<Inspection>()
const [state, setState] = createSignal<State>()
const [tools, setTools] = createSignal<Devtools>()
let frame: number | undefined
@@ -343,7 +368,9 @@ const BrowserPanel: Component<Props> = (props) => {
type,
sessionId: session,
projectId: props.projectId(),
...(type.endsWith("open") ? { url: url().trim() } : {}),
...(type.endsWith("open")
? { url: /^https?:\/\//i.test(url().trim()) ? url().trim() : `http://${url().trim()}` }
: {}),
...(type.endsWith("devtools")
? {
theme:
@@ -439,20 +466,22 @@ const BrowserPanel: Component<Props> = (props) => {
schedule()
return
}
const content = feedback(message)
const element = message.element
const browser = element?.selector
? {
id: crypto.randomUUID(),
sessionId: session,
selector: element.selector,
text: element.text,
url: message.url,
content,
}
: undefined
if (!element?.selector) return
const browser = {
id: crypto.randomUUID(),
sessionId: session,
selector: element.selector,
text: element.text,
url: message.url,
title: message.title,
hierarchy: element.hierarchy,
html: element.html,
styles: element.styles,
source: element.source,
}
const content = formatBrowserFeedback([browser])
window.postMessage({ type: "appendChatBoxMessage", text: content, browser }, "*")
setSelected(message)
setSelecting(false)
stop()
}
@@ -463,7 +492,6 @@ const BrowserPanel: Component<Props> = (props) => {
setUrl("")
setSelecting(false)
setPointing(false)
setSelected(undefined)
setTools(undefined)
stop()
if (!session) return
@@ -493,27 +521,35 @@ const BrowserPanel: Component<Props> = (props) => {
post({ type: "agentManager.browser.state", sessionId: session, projectId: props.projectId() })
})
const loading = () => state()?.status === "loading" || state()?.status === "starting"
const close = () => {
request("agentManager.browser.close")
props.onClose()
}
return (
<div class="am-browser-panel" aria-label={t("agentManager.browser.title")}>
<div
class="am-browser-panel"
aria-label={t("agentManager.browser.title")}
aria-busy={loading()}
data-status={state()?.status ?? "closed"}
>
<Toolbar
url={url()}
title={state()?.title}
active={!!props.sessionId()}
selecting={selecting()}
docked={!!tools()}
ready={!!state()?.url && state()?.status !== "closed"}
loading={loading()}
errors={state()?.errors ?? 0}
onUrl={setUrl}
onOpen={() => request("agentManager.browser.open")}
onSelect={toggle}
onDevtools={dock}
onRefresh={() => request("agentManager.browser.refresh")}
onClose={() => request("agentManager.browser.close")}
onClose={close}
/>
<div class="am-browser-meta">
<span>{state()?.title || t("agentManager.browser.empty")}</span>
<span class="am-browser-status" role="status" aria-live="polite">
{state()?.status ?? t("agentManager.browser.notStarted")}
</span>
</div>
<div class="am-browser-workspace" classList={{ "am-browser-workspace-docked": !!tools() }}>
<Viewport
state={state()}
@@ -524,13 +560,12 @@ const BrowserPanel: Component<Props> = (props) => {
onSelect={choose}
/>
<Show when={tools()} keyed>
{(entry) => <Tools url={entry.url} onClose={() => setTools(undefined)} />}
{(entry) => <Tools url={entry.url} />}
</Show>
</div>
<Show when={!tools()}>
<Diagnostics logs={state()?.logs ?? []} />
</Show>
<Footer state={state()} selected={selected()} onClose={props.onClose} />
</div>
)
}
@@ -12,48 +12,83 @@
.am-browser-toolbar {
display: flex;
flex-shrink: 0;
align-items: center;
gap: 6px;
padding: 8px;
gap: 4px;
padding: 6px;
border-bottom: 1px solid var(--border-weak-base);
}
.am-browser-url {
flex: 1;
min-width: 0;
}
.am-browser-meta,
.am-browser-footer {
.am-browser-address {
display: flex;
align-items: center;
gap: 8px;
min-width: 0;
padding: 6px 8px;
color: var(--text-weak);
font-size: var(--kilo-font-size-11);
}
.am-browser-meta > span:first-child,
.am-browser-footer > span:first-child {
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.am-browser-meta > span:first-child,
.am-browser-footer > span:first-child {
flex: 1;
align-items: center;
gap: 4px;
height: 28px;
min-width: 0;
padding: 0 2px 0 8px;
background: var(--surface-inset-base);
border: 1px solid var(--border-weak-base);
border-radius: 6px;
}
.am-browser-status {
.am-browser-address:focus-within {
border-color: var(--vscode-focusBorder);
}
.am-browser-address [data-component="input"] {
flex: 1;
min-width: 0;
}
.am-browser-address [data-slot="input-wrapper"] {
min-width: 0;
}
.am-browser-address .am-browser-url {
height: 26px;
min-width: 0;
padding: 0;
font-size: var(--font-size-small);
background: transparent;
border: 0;
border-radius: 0;
box-shadow: none;
}
.am-browser-site {
display: flex;
flex-shrink: 0;
text-transform: capitalize;
align-items: center;
justify-content: center;
width: 16px;
height: 16px;
color: var(--text-weak);
}
.am-browser-errors,
.am-browser-error {
color: var(--syntax-diff-delete, #da3319);
.am-browser-site > svg {
width: 16px;
height: 16px;
}
.am-browser-tools-action {
position: relative;
flex-shrink: 0;
}
.am-browser-error-count {
position: absolute;
top: -3px;
right: -3px;
min-width: 12px;
padding: 0 3px;
font-size: var(--kilo-font-size-11);
line-height: 14px;
color: var(--text-on-accent);
text-align: center;
pointer-events: none;
background: var(--syntax-diff-delete, #da3319);
border-radius: 7px;
}
.am-browser-workspace {
@@ -99,18 +134,6 @@
border-top: 1px solid var(--border-weak-base);
}
.am-browser-devtools-toolbar {
display: flex;
height: 30px;
min-height: 30px;
align-items: center;
justify-content: space-between;
padding: 0 8px;
color: var(--text-weak);
font-size: var(--font-size-small);
border-bottom: 1px solid var(--border-weak-base);
}
.am-browser-devtools-frame {
flex: 1;
width: 100%;
@@ -172,14 +195,6 @@
transform: none;
}
.am-browser-selected {
max-width: 240px;
overflow: hidden;
font-family: var(--font-family-mono);
text-overflow: ellipsis;
white-space: nowrap;
}
.am-browser-console {
max-height: 140px;
padding: 8px 12px;
@@ -225,10 +240,6 @@
text-align: center;
}
.am-browser-footer {
border-top: 1px solid var(--border-weak-base);
}
.am-layout {
display: flex;
height: 100vh;
@@ -1,4 +1,5 @@
import type { AgentManagerSendInitialMessage, SendMessageRequest } from "../src/types/messages"
import { formatBrowserFeedback } from "../../src/shared/browser-feedback"
interface VariantSession {
getSessionAgent: (sessionID: string) => string
@@ -7,16 +8,18 @@ interface VariantSession {
export function initialMessage(ev: AgentManagerSendInitialMessage): SendMessageRequest | undefined {
if (!ev.text) return undefined
const text = ev.browserFeedback ? `${formatBrowserFeedback(ev.browserFeedback.references)}\n\n${ev.text}` : ev.text
return {
type: "sendMessage",
...(ev.projectId ? { projectId: ev.projectId } : {}),
text: ev.text,
text,
sessionID: ev.sessionId,
providerID: ev.providerID,
modelID: ev.modelID,
agent: ev.agent,
variant: ev.variant,
files: ev.files,
browserFeedback: ev.browserFeedback,
}
}
@@ -0,0 +1,121 @@
import { For, Show, createSignal, type Component } from "solid-js"
import { Button } from "@kilocode/kilo-ui/button"
import { Icon } from "@kilocode/kilo-ui/icon"
import { browserFeedbackData, type BrowserReference } from "../../../../src/shared/browser-feedback"
import { useLanguage } from "../../context/language"
interface BrowserReferencesProps {
references: BrowserReference[]
variant?: "draft" | "message"
onRemove?: (id: string) => void
onClear?: () => void
}
export const BrowserReferences: Component<BrowserReferencesProps> = (props) => {
const language = useLanguage()
const [open, setOpen] = createSignal(true)
const [full, setFull] = createSignal<string[]>([])
const data = () => browserFeedbackData(props.references)
const rows = () => data()?.references ?? []
const toggle = (id: string) =>
setFull((current) => (current.includes(id) ? current.filter((item) => item !== id) : [...current, id]))
const page = (item: BrowserReference) => item.title || item.url
const styles = (item: BrowserReference) =>
[
item.styles?.color ? `color=${item.styles.color}` : "",
item.styles?.backgroundColor ? `background=${item.styles.backgroundColor}` : "",
]
.filter(Boolean)
.join(", ")
const source = (item: BrowserReference) =>
item.source
? [item.source.file, item.source.line, item.source.column].filter((value) => value !== undefined).join(":")
: ""
return (
<div
class="prompt-review-comments"
classList={{ "prompt-review-comments--message": props.variant === "message" }}
data-component="browser-references"
>
<div class="prompt-review-comments-header">
<button
type="button"
class="prompt-review-comments-toggle"
aria-expanded={open()}
onClick={() => setOpen(!open())}
>
<Icon name={open() ? "chevron-down" : "chevron-right"} size="small" />
<span class="prompt-review-comments-title">
{language.t("agentManager.browser.title")} ({rows().length})
</span>
</button>
<Show when={props.onClear}>
<Button variant="ghost" size="small" onClick={() => props.onClear?.()}>
{language.t("agentManager.review.clearAll")}
</Button>
</Show>
</div>
<Show when={open()}>
<div class="prompt-review-list">
<For each={rows()}>
{(item) => (
<div class="prompt-review-row" classList={{ "prompt-review-row--full": full().includes(item.id) }}>
<div class="prompt-review-row-top">
<span class="prompt-review-row-icon">
<Icon name="globe" size="small" />
</span>
<button
type="button"
class="prompt-review-row-main"
aria-expanded={full().includes(item.id)}
onClick={() => toggle(item.id)}
>
<span class="prompt-review-row-head">
<span class="prompt-review-row-label">{item.selector}</span>
</span>
<Show when={!full().includes(item.id)}>
<span class="prompt-review-row-preview">
{item.text || item.source?.file || page(item) || ""}
</span>
</Show>
</button>
<Show when={props.onRemove}>
<button
type="button"
class="prompt-review-row-remove"
onClick={() => props.onRemove?.(item.id)}
aria-label={language.t("common.delete")}
>
×
</button>
</Show>
</div>
<Show when={full().includes(item.id)}>
<div class="prompt-review-row-detail">
<Show when={item.url || item.title}>
{(value) => <div class="prompt-review-row-path">{value()}</div>}
</Show>
<Show when={item.hierarchy?.length}>
<pre class="prompt-review-row-snippet">{item.hierarchy?.join(" > ")}</pre>
</Show>
<Show when={item.text && (!item.html || item.html === item.text)}>
<pre class="prompt-review-row-snippet">{item.text}</pre>
</Show>
<Show when={item.html && item.html !== item.text}>
<pre class="prompt-review-row-snippet">{item.html}</pre>
</Show>
<Show when={styles(item)}>{(value) => <div class="prompt-review-row-text">{value()}</div>}</Show>
<Show when={source(item)}>{(value) => <code class="prompt-review-row-path">{value()}</code>}</Show>
</div>
</Show>
</div>
)}
</For>
</div>
</Show>
</div>
)
}
@@ -56,13 +56,7 @@ import {
type SandboxDefaultState,
type SandboxState,
} from "./prompt-input-utils"
import type {
BrowserReference,
ExtensionMessage,
ReviewCommentEntry,
SendMessageFailedMessage,
TextPart,
} from "../../types/messages"
import type { ExtensionMessage, ReviewCommentEntry, SendMessageFailedMessage, TextPart } from "../../types/messages"
import { formatReviewCommentsMarkdown } from "../../utils/review-comment-markdown"
import {
createdDraftKey,
@@ -86,7 +80,15 @@ import {
scrollDrafts,
} from "../../utils/draft-store"
import { ReviewComments } from "./ReviewComments"
import { partReview, reviewBody } from "../../../../src/shared/review-comments"
import { BrowserReferences } from "./BrowserReferences"
import {
browserFeedbackData,
formatBrowserFeedback,
mergeBrowserReferences,
partFeedback,
type BrowserReference,
} from "../../../../src/shared/browser-feedback"
import { reviewBody } from "../../../../src/shared/review-comments"
import { isEnterKeyCommitNotIme } from "../../utils/ime-enter"
import { parseMemoryCommand, type ParsedMemoryCommand } from "../../utils/memory-command"
import { useMemory } from "../../context/memory"
@@ -243,11 +245,13 @@ export const PromptInput: Component<PromptInputProps> = (props) => {
comments: ReviewCommentEntry[],
imgs: ImageAttachment[],
scroll = textareaRef?.scrollTop ?? scrollDrafts.get(key) ?? 0,
) => savePromptDraft(key, next, comments, imgs, scroll)
browser: BrowserReference[] = browsers(),
) => savePromptDraft(key, next, comments, imgs, scroll, browser)
const readDraft = () => ({
text: text().trim(),
comments: reviewComments(),
images: imageAttach.images(),
browsers: browsers(),
scroll: textareaRef?.scrollTop ?? scrollDrafts.get(draftKey()) ?? 0,
})
@@ -403,8 +407,9 @@ export const PromptInput: Component<PromptInputProps> = (props) => {
const val = untrack(text)
const comments = untrack(reviewComments)
const imgs = untrack(imageAttach.images)
if (val || comments.length > 0 || imgs.length > 0 || drafts.has(prev)) {
saveDraft(prev, val, comments, imgs)
const browser = untrack(browsers)
if (val || comments.length > 0 || imgs.length > 0 || browser.length > 0 || drafts.has(prev)) {
saveDraft(prev, val, comments, imgs, undefined, browser)
}
}
const draft = drafts.get(key) ?? ""
@@ -444,7 +449,7 @@ export const PromptInput: Component<PromptInputProps> = (props) => {
const parts = session.getParts(m.id)
return parts
.filter((part): part is TextPart => part.type === "text")
.map((part) => partReview(part.metadata, part.text)?.body ?? part.text.replace(REVIEW_PREFIX, ""))
.map((part) => partFeedback(part.metadata, part.text)?.body ?? part.text.replace(REVIEW_PREFIX, ""))
.join("")
})
history.seed(texts)
@@ -489,11 +494,12 @@ export const PromptInput: Component<PromptInputProps> = (props) => {
const draft = text().trim()
const comments = reviewComments()
const imgs = imageAttach.images()
const browser = browsers()
const scroll = textareaRef?.scrollTop ?? 0
const id = tabs?.add()
if (!id) session.clearCurrentSession()
const key = id ? scopeDraftKey(boxKey(), pendingDraftKey(id) ?? "new") : draftKey()
saveDraft(key, draft, comments, imgs, scroll)
saveDraft(key, draft, comments, imgs, scroll, browser)
}
window.addEventListener("newTaskRequest", onNewTaskRequest)
onCleanup(() => window.removeEventListener("newTaskRequest", onNewTaskRequest))
@@ -515,7 +521,14 @@ export const PromptInput: Component<PromptInputProps> = (props) => {
const draft = captured.get(id)
captured.delete(id)
if (!draft) return
saveDraft(scopeDraftKey(box, sessionDraftKey(sid)), draft.text, draft.comments, draft.images, draft.scroll)
saveDraft(
scopeDraftKey(box, sessionDraftKey(sid)),
draft.text,
draft.comments,
draft.images,
draft.scroll,
draft.browsers,
)
}
window.addEventListener("agentManagerApplyDraft", onAgentManagerApplyDraft)
onCleanup(() => window.removeEventListener("agentManagerApplyDraft", onAgentManagerApplyDraft))
@@ -597,7 +610,11 @@ export const PromptInput: Component<PromptInputProps> = (props) => {
})
const restoreFailed = (failed: SendMessageFailedMessage) => {
const draft = failed.review ? reviewBody(failed.review, failed.text) : failed.text
const parsed = failed.browserFeedback
? partFeedback({ review: failed.review, browserFeedback: failed.browserFeedback }, failed.text)
: undefined
if (failed.browserFeedback && !parsed) return
const draft = parsed?.body ?? (failed.review ? reviewBody(failed.review, failed.text) : failed.text)
if (draft === undefined) return
if (
(failed.draftID && isPendingDraftDiscarded(failed.draftID)) ||
@@ -617,6 +634,7 @@ export const PromptInput: Component<PromptInputProps> = (props) => {
: undefined
if (!target) return
const comments = failed.review?.comments ?? []
const browser = parsed?.browserFeedback?.references ?? []
const images = (failed.files ?? [])
.filter((file) => file.mime.startsWith("image/") && file.url.startsWith("data:"))
.map((file) => ({
@@ -626,12 +644,13 @@ export const PromptInput: Component<PromptInputProps> = (props) => {
dataUrl: file.url,
}))
if (target !== draftKey()) {
saveDraft(target, draft, comments, images, scrollDrafts.get(target) ?? 0)
saveDraft(target, draft, comments, images, scrollDrafts.get(target) ?? 0, browser)
return
}
// Do not overwrite a new draft the user started while the send was in flight.
if (text().trim() || reviewComments().length > 0 || imageAttach.images().length > 0) return
if (text().trim() || reviewComments().length > 0 || imageAttach.images().length > 0 || browsers().length > 0) return
replaceReviewComments(comments)
replace(browser)
if (draft) {
setText(draft)
mention.seedFromText(draft)
@@ -732,66 +751,94 @@ export const PromptInput: Component<PromptInputProps> = (props) => {
return true
}
const restoreBox = (message: Extract<ExtensionMessage, { type: "setChatBoxMessage" }>) => {
setText(message.text)
if (message.paths?.length) mention.seedFromParts(message.paths, message.text)
else mention.seedFromText(message.text)
if (message.sessions?.length) mention.seedSessions(message.sessions, message.text)
if (textareaRef) {
textareaRef.value = message.text
adjustHeight()
}
if (message.review || message.browser) {
replaceReviewComments(message.review ?? [])
replace(message.browser ?? [])
}
if (message.images) {
const imgs = message.images.map((img) => ({
id: crypto.randomUUID(),
filename: img.filename ?? "image",
mime: img.mime,
dataUrl: img.dataUrl,
}))
imageAttach.replace(imgs)
imageDrafts.set(draftKey(), imgs)
}
}
const appendBox = (message: Extract<ExtensionMessage, { type: "appendChatBoxMessage" }>) => {
const reference = message.browser
if (reference) {
if (reference.sessionId !== sid()) return
replace(mergeBrowserReferences(browsers(), reference))
textareaRef?.focus()
return
}
const current = text()
const separator = current && !current.endsWith("\n") ? "\n\n" : ""
const next = current + separator + message.text
setText(next)
if (textareaRef) {
textareaRef.value = next
adjustHeight()
textareaRef.focus()
textareaRef.scrollTop = textareaRef.scrollHeight
syncHighlightScroll()
}
}
const appendReviews = (message: Extract<ExtensionMessage, { type: "appendReviewComments" }>) => {
const empty =
!text().trim() && reviewComments().length === 0 && imageAttach.images().length === 0 && browsers().length === 0
replaceReviewComments(mergeReviewComments(reviewComments(), message.comments))
if (message.autoSend && empty && !isDisabled() && !props.blocked?.()) {
void handleSend()
return
}
textareaRef?.focus()
}
const created = (message: Extract<ExtensionMessage, { type: "sessionCreated" }>) => {
const raw = createdDraftKey(message.draftID, sandboxRequest(undefined) !== undefined)
if (!raw) return
const source = scopeDraftKey(boxKey(), raw)
const target = scopeDraftKey(boxKey(), sessionDraftKey(message.session.id))
if (source === draftKey()) saveDraft(source, text(), reviewComments(), imageAttach.images())
movePromptDraft(
{ text: drafts, comments: reviewDrafts, images: imageDrafts, scrolls: scrollDrafts, browsers: references },
source,
target,
)
if (
message.draftID &&
!session.currentSessionID() &&
(props.pendingSessionID ?? session.draftSessionID()) === message.draftID
) {
session.setDraftSessionID(message.session.id)
}
}
const unsubscribe = vscode.onMessage((message) => {
if (handleSandboxMessage(message)) return
if (message.type === "setChatBoxMessage") {
setText(message.text)
// Prefer the exact attachment paths when available (e.g. reverting to a
// message with @mentions) — seedFromText re-derives candidate mentions
// from raw text via regex, which truncates at the first space in a
// filename and cannot be relied on to reconstruct spaced paths correctly.
if (message.paths?.length) mention.seedFromParts(message.paths, message.text)
else mention.seedFromText(message.text)
if (message.sessions?.length) mention.seedSessions(message.sessions, message.text)
if (textareaRef) {
textareaRef.value = message.text
adjustHeight()
}
// When present, images are authoritative: replace current attachments
// (an empty array clears them, e.g. on redo). Absent leaves them alone.
if (message.images) {
const imgs = message.images.map((img) => ({
id: crypto.randomUUID(),
filename: img.filename ?? "image",
mime: img.mime,
dataUrl: img.dataUrl,
}))
imageAttach.replace(imgs)
imageDrafts.set(draftKey(), imgs)
}
restoreBox(message)
}
if (message.type === "appendChatBoxMessage") {
const reference = message.browser
if (reference && reference.sessionId === sid()) {
replace([...browsers().filter((item) => item.id !== reference.id), reference])
textareaRef?.focus()
return
}
const current = text()
const separator = current && !current.endsWith("\n") ? "\n\n" : ""
const next = current + separator + message.text
setText(next)
if (textareaRef) {
textareaRef.value = next
adjustHeight()
textareaRef.focus()
textareaRef.scrollTop = textareaRef.scrollHeight
syncHighlightScroll()
}
}
if (message.type === "appendChatBoxMessage") appendBox(message)
if (message.type === "appendReviewComments") {
const empty =
!text().trim() && reviewComments().length === 0 && imageAttach.images().length === 0 && browsers().length === 0
const merged = mergeReviewComments(reviewComments(), message.comments)
replaceReviewComments(merged)
if (message.autoSend && empty && !isDisabled() && !props.blocked?.()) {
void handleSend()
} else {
textareaRef?.focus()
}
appendReviews(message)
}
if (message.type === "triggerTask") {
@@ -804,26 +851,7 @@ export const PromptInput: Component<PromptInputProps> = (props) => {
restoreFailed(message as SendMessageFailedMessage)
}
if (message.type === "sessionCreated") {
const raw = createdDraftKey(message.draftID, sandboxRequest(undefined) !== undefined)
if (raw) {
const source = scopeDraftKey(boxKey(), raw)
const target = scopeDraftKey(boxKey(), sessionDraftKey(message.session.id))
if (source === draftKey()) saveDraft(source, text(), reviewComments(), imageAttach.images())
movePromptDraft(
{ text: drafts, comments: reviewDrafts, images: imageDrafts, scrolls: scrollDrafts },
source,
target,
)
}
if (
message.draftID &&
!session.currentSessionID() &&
(props.pendingSessionID ?? session.draftSessionID()) === message.draftID
) {
session.setDraftSessionID(message.session.id)
}
}
if (message.type === "sessionCreated") created(message)
if (message.type === "action" && message.action === "focusInput") {
textareaRef?.focus()
@@ -1085,6 +1113,7 @@ export const PromptInput: Component<PromptInputProps> = (props) => {
const context = ctx()
const value = text()
const comments = reviewComments()
const browser = browsers()
const images = imageAttach.images()
speech.stop({
done: () => void handleSend(),
@@ -1094,6 +1123,7 @@ export const PromptInput: Component<PromptInputProps> = (props) => {
ctx() === context &&
text() === value &&
reviewComments() === comments &&
browsers() === browser &&
imageAttach.images() === images,
})
}
@@ -1240,10 +1270,9 @@ export const PromptInput: Component<PromptInputProps> = (props) => {
const imgs = imageAttach.images()
const pending = reviewComments()
const review = pending.length > 0 ? formatReviewCommentsMarkdown(pending) : ""
const browser = browsers()
.map((item) => item.content)
.join("\n\n")
const message = [review, browser, draft].filter(Boolean).join("\n\n")
const browserData = browserFeedbackData(browsers())
const browserText = browserData ? formatBrowserFeedback(browserData.references) : ""
const message = [review, browserText, draft].filter(Boolean).join("\n\n")
const data = review ? { version: 1 as const, comments: pending } : undefined
if (
(!message && imgs.length === 0) ||
@@ -1299,7 +1328,7 @@ export const PromptInput: Component<PromptInputProps> = (props) => {
const attachments = allFiles.length > 0 ? allFiles : undefined
// Server-side slash command (cmdMatch/matched already computed above)
if (matched && !data) {
if (matched && !data && !browserData) {
const args = draft.slice(cmdMatch![0].length).trim()
session.sendCommand(
matched.name,
@@ -1317,7 +1346,17 @@ export const PromptInput: Component<PromptInputProps> = (props) => {
},
)
} else {
session.sendMessage(message, sel?.providerID, sel?.modelID, attachments, pendingId, context, data, origin ?? null)
session.sendMessage(
message,
sel?.providerID,
sel?.modelID,
attachments,
pendingId,
context,
data,
origin ?? null,
browserData,
)
}
drafts.delete(key)
@@ -1356,42 +1395,8 @@ export const PromptInput: Component<PromptInputProps> = (props) => {
/>
</Show>
<Show when={browsers().length > 0}>
<div class="prompt-review-comments" data-component="browser-references">
<div class="prompt-review-comments-header">
<span class="prompt-review-comments-title">{language.t("agentManager.browser.title")}</span>
<Button variant="ghost" size="small" onClick={clear}>
{language.t("agentManager.review.clearAll")}
</Button>
</div>
<div class="prompt-review-list">
<For each={browsers()}>
{(item) => (
<div class="prompt-review-row">
<div class="prompt-review-row-top">
<span class="prompt-review-row-icon">
<Icon name="globe" size="small" />
</span>
<div class="prompt-review-row-main">
<span class="prompt-review-row-head">
<span class="prompt-review-row-label">{item.selector}</span>
</span>
<Show when={item.text || item.url}>
{(value) => <span class="prompt-review-row-preview">{value()}</span>}
</Show>
</div>
<button
type="button"
class="prompt-review-row-remove"
onClick={() => remove(item.id)}
aria-label={language.t("common.delete")}
>
×
</button>
</div>
</div>
)}
</For>
</div>
<div data-component="browser-references">
<BrowserReferences references={browsers()} onRemove={remove} onClear={clear} />
</div>
</Show>
<Show when={mention.showMention()}>
@@ -1,7 +1,8 @@
import { createMemo, type Component } from "solid-js"
import { createMemo, Show, type Component } from "solid-js"
import { UserMessageDisplay } from "@kilocode/kilo-ui/message-part"
import { partReview } from "../../../../src/shared/review-comments"
import { partFeedback } from "../../../../src/shared/browser-feedback"
import type { Message, Part, TextPart } from "../../types/messages"
import { BrowserReferences } from "./BrowserReferences"
import { ReviewComments } from "./ReviewComments"
interface VscodeUserMessageProps {
@@ -16,22 +17,31 @@ interface VscodeUserMessageProps {
export const VscodeUserMessage: Component<VscodeUserMessageProps> = (props) => {
const text = createMemo(() => props.parts.find((part): part is TextPart => part.type === "text" && !part.synthetic))
const review = createMemo(() => {
const feedback = createMemo(() => {
const part = text()
if (!part) return undefined
return partReview(part.metadata, part.text)
return partFeedback(part.metadata, part.text)
})
const body = createMemo(() => review()?.body)
const body = createMemo(() => feedback()?.body)
return (
<UserMessageDisplay
message={props.message as unknown as Parameters<typeof UserMessageDisplay>[0]["message"]}
parts={props.parts as unknown as Parameters<typeof UserMessageDisplay>[0]["parts"]}
text={body()}
copyText={review() ? text()?.text : undefined}
copyText={feedback() ? text()?.text : undefined}
header={
review() ? (
<ReviewComments comments={review()!.data.comments} sessionID={props.message.sessionID} variant="message" />
feedback() ? (
<>
<Show when={feedback()?.review}>
{(review) => (
<ReviewComments comments={review().comments} sessionID={props.message.sessionID} variant="message" />
)}
</Show>
<Show when={feedback()?.browserFeedback}>
{(browser) => <BrowserReferences references={browser().references} variant="message" />}
</Show>
</>
) : undefined
}
interrupted={props.interrupted}
@@ -1,5 +1,7 @@
import { reconcile } from "solid-js/store"
import type { Message, MessageLoadMode, Part, ToolPart } from "../types/messages"
import { partFeedback, type BrowserReference } from "../../../src/shared/browser-feedback"
import type { ReviewCommentEntry } from "../../../src/shared/review-comments"
export const SNAPSHOT_PROGRESS_TEXT = "Initializing snapshot..."
@@ -40,6 +42,8 @@ export interface RevertPromptState {
paths: string[]
sessions: Array<{ id: string; title: string; updated: number }>
images: Array<{ dataUrl: string; mime: string; filename?: string }>
review: ReviewCommentEntry[]
browser: BrowserReference[]
}
/**
@@ -49,10 +53,17 @@ export interface RevertPromptState {
*/
export function revertPromptState(parts: readonly Part[]): RevertPromptState {
const files = parts.filter((p): p is Extract<Part, { type: "file" }> => p.type === "file")
const feedback = parts
.filter((p): p is Extract<Part, { type: "text" }> => p.type === "text" && !p.synthetic)
.map((p) => partFeedback(p.metadata, p.text))
.filter((p): p is NonNullable<typeof p> => p !== undefined)
return {
text: parts
.filter((p) => p.type === "text" && !(p as { synthetic?: boolean }).synthetic)
.map((p) => (p as { text: string }).text ?? "")
.map((p) => {
if (p.type !== "text") return ""
return partFeedback(p.metadata, p.text)?.body ?? p.text
})
.join(""),
paths: files.map((p) => p.source?.path).filter((p): p is string => !!p && !p.startsWith("session:")),
sessions: files
@@ -65,6 +76,8 @@ export function revertPromptState(parts: readonly Part[]): RevertPromptState {
images: files
.filter((p) => p.mime.startsWith("image/") && p.url.startsWith("data:"))
.map((p) => ({ dataUrl: p.url, mime: p.mime, filename: p.filename })),
review: feedback.flatMap((p) => p.review?.comments ?? []),
browser: feedback.flatMap((p) => p.browserFeedback?.references ?? []),
}
}
@@ -85,7 +85,8 @@ import { state as todoState } from "./todo-revert"
import { sessionVariantKeys, transferVariants, variantKey } from "./session-variant-store"
import { createSessionVariants } from "./session-variants"
import { KILO_AUTO, KILO_PROVIDER_ID, parseModelString } from "../../../src/shared/provider-model"
import { reviewMetadata, type ReviewMessageData } from "../../../src/shared/review-comments"
import { type ReviewMessageData } from "../../../src/shared/review-comments"
import { feedbackMetadata, type BrowserFeedbackData } from "../../../src/shared/browser-feedback"
import { activeUserMessageID, visibleMessages as filterVisibleMessages } from "./session-queue"
import { clearSessionDraftDiscarded, deleteDraftsForSession } from "../utils/draft-store"
import { createAbortState } from "./abort-state"
@@ -255,6 +256,7 @@ interface SessionContextValue {
context?: string,
review?: ReviewMessageData,
origin?: string | null,
browserFeedback?: BrowserFeedbackData,
) => void
sendCommand: (
command: string,
@@ -2173,6 +2175,7 @@ export const SessionProvider: ParentComponent = (props) => {
text: string,
files?: FileAttachment[],
review?: ReviewMessageData,
browserFeedback?: BrowserFeedbackData,
) {
const now = Date.now()
const temp: Message = {
@@ -2196,7 +2199,7 @@ export const SessionProvider: ParentComponent = (props) => {
id: partId,
messageID,
text,
metadata: review ? reviewMetadata(review) : undefined,
metadata: feedbackMetadata(review, browserFeedback),
})
}
for (const file of files ?? []) {
@@ -2228,6 +2231,7 @@ export const SessionProvider: ParentComponent = (props) => {
context?: string,
review?: ReviewMessageData,
origin?: string | null,
browserFeedback?: BrowserFeedbackData,
) {
if (!server.isConnected()) {
console.warn("[Kilo New] Cannot send message: not connected")
@@ -2258,6 +2262,7 @@ export const SessionProvider: ParentComponent = (props) => {
variant: currentVariant(scope),
files,
review,
browserFeedback,
})
return
}
@@ -2273,7 +2278,7 @@ export const SessionProvider: ParentComponent = (props) => {
if (!sid && !draftID && effectiveDraftID) agentDrafts.seed(effectiveDraftID)
if (scope) {
clearClose(scope)
addOptimistic(scope, messageID, text, files, review)
addOptimistic(scope, messageID, text, files, review, browserFeedback)
startSubmission(scope, messageID)
if (!sid && (!draftID || draftSessionID() === scope)) {
setUserClearedSession(false)
@@ -2294,6 +2299,7 @@ export const SessionProvider: ParentComponent = (props) => {
variant: currentVariant(scope),
files,
review,
browserFeedback,
agentManagerContext: context,
})
}
@@ -2785,14 +2791,12 @@ export const SessionProvider: ParentComponent = (props) => {
// Restore the reverted user message's prompt text and attachments into the
// input. Dispatch as a window message so PromptInput picks it up via onMessage.
const state = revertPromptState(getParts(messageID))
const { text, paths, sessions, images } = state
const { text, paths, sessions, images, review, browser } = state
// Paths carry the attachments' exact locations so PromptInput can seed them
// directly rather than re-deriving mentions from the text via regex, which
// truncates at the first space in a filename (see PromptInput's
// setChatBoxMessage handler).
if (text || paths.length > 0 || sessions.length > 0 || images.length > 0) {
window.postMessage({ type: "setChatBoxMessage", text, paths, sessions, images }, "*")
}
window.postMessage({ type: "setChatBoxMessage", text, paths, sessions, images, review, browser }, "*")
vscode.postMessage({ type: "revertSession", sessionID: id, messageID, partID })
}
@@ -2800,7 +2804,7 @@ export const SessionProvider: ParentComponent = (props) => {
const id = currentSessionID()
if (!id) return
// Clear the prompt input on full redo (matching TUI/desktop behavior)
window.postMessage({ type: "setChatBoxMessage", text: "", images: [] }, "*")
window.postMessage({ type: "setChatBoxMessage", text: "", images: [], review: [], browser: [] }, "*")
vscode.postMessage({ type: "unrevertSession", sessionID: id })
}
@@ -27,6 +27,7 @@ import { ServerContext } from "../context/server"
import { WorktreeModeProvider } from "../context/worktree-mode"
import type {
Message,
BrowserReference,
Part,
QuestionRequest,
ReviewComment,
@@ -36,6 +37,7 @@ import type {
TodoItem,
} from "../types/messages"
import { formatReviewCommentsMarkdown } from "../utils/review-comment-markdown"
import { feedbackMetadata, formatBrowserFeedback } from "../../../src/shared/browser-feedback"
import { reviewMetadata } from "../../../src/shared/review-comments"
const SESSION_ID = "story-session-chat-001"
@@ -260,6 +262,28 @@ function reviewMessage(comments: ReviewCommentEntry[]) {
return <VscodeUserMessage message={message} parts={parts} />
}
function browserMessage(references: BrowserReference[]) {
const data = { version: 1 as const, references }
const message: Message = {
id: "browser-user-message",
sessionID: SESSION_ID,
role: "user",
createdAt: new Date(0).toISOString(),
time: { created: 0 },
}
const parts: Part[] = [
{
id: "browser-user-part",
sessionID: SESSION_ID,
messageID: message.id,
type: "text",
text: `${formatBrowserFeedback(references)}\n\nPlease fix the selected browser elements.`,
metadata: feedbackMetadata(undefined, data),
},
]
return <VscodeUserMessage message={message} parts={parts} />
}
export const UserMessageReviewComments: Story = {
name: "User message — interactive review comments",
render: () => {
@@ -314,6 +338,39 @@ export const UserMessageManyReviewComments: Story = {
},
}
export const UserMessageBrowserFeedback: Story = {
name: "User message — browser feedback",
render: () => (
<StoryProviders sessionID={SESSION_ID} status="idle">
<div style={{ "max-height": "620px", padding: "12px" }}>
{browserMessage([
{
id: "browser-1",
sessionId: SESSION_ID,
selector: "main > button.save",
url: "https://example.com/settings",
title: "Settings",
hierarchy: ["main", "button.save"],
text: "Save settings",
html: '<button class="save">Save settings</button>',
styles: { color: "rgb(30, 30, 30)", backgroundColor: "white" },
source: { file: "src/settings.tsx", line: 42, column: 7 },
},
{
id: "browser-2",
sessionId: SESSION_ID,
selector: "form input[name=email]",
url: "https://example.com/settings",
title: "Settings",
hierarchy: ["main", "form", "input[name=email]"],
text: "Email address",
},
])}
</div>
</StoryProviders>
),
}
/**
* ChatView with a pending question tool call and an empty input.
*
@@ -17,6 +17,9 @@ import type {
import type { AgentManagerSidebarTarget } from "./webview-messages"
import type { PermissionRequest } from "./permissions"
import type { AnacondaDesktopExtensionMessage } from "../../../../src/shared/anaconda-desktop-messages"
import type { BrowserFeedbackData, BrowserReference } from "../../../../src/shared/browser-feedback"
export type { BrowserReference } from "../../../../src/shared/browser-feedback"
export interface BackgroundJobsLoadedMessage {
type: "backgroundJobsLoaded"
@@ -141,6 +144,7 @@ export interface SendMessageFailedMessage {
messageID?: string
files?: FileAttachment[]
review?: import("../../../../src/shared/review-comments").ReviewMessageData
browserFeedback?: BrowserFeedbackData
}
export interface SessionCommandCompletedMessage {
@@ -335,15 +339,8 @@ export interface SetChatBoxMessage {
* array clears them); absent leaves current attachments untouched.
*/
images?: RestoredImage[]
}
export interface BrowserReference {
id: string
sessionId: string
selector: string
text?: string
url?: string
content: string
review?: import("../../../../src/shared/review-comments").ReviewCommentEntry[]
browser?: BrowserReference[]
}
export interface AppendChatBoxMessage {
@@ -1200,6 +1197,7 @@ export interface AgentManagerSendInitialMessage {
agent?: string
variant?: string
files?: Array<{ mime: string; url: string }>
browserFeedback?: BrowserFeedbackData
}
// Enhance prompt result (extension → webview)
@@ -1450,6 +1448,10 @@ export interface AgentManagerBrowserInspectionMessage {
text?: string
selector?: string
rect?: { x: number; y: number; width: number; height: number }
hierarchy?: string[]
html?: string
styles?: { color?: string; backgroundColor?: string }
source?: { file: string; line?: number; column?: number }
}
logs: string[]
hover?: boolean
@@ -6,6 +6,7 @@ import type { ModelSelection, ProviderConfig } from "./providers"
import type { Config } from "./config"
import type { ModelAllocation, ReviewCommentEntry, TerminalDestination, TerminalPlacement } from "./agent-manager"
import type { ReviewMessageData } from "../../../../src/shared/review-comments"
import type { BrowserFeedbackData } from "../../../../src/shared/browser-feedback"
import type { WorkStyle, WorkStyleState } from "../../../../src/shared/work-style-presets"
import type { RefreshProviderUsageMessage, RequestProviderUsageMessage } from "./provider-usage"
import type { AnacondaDesktopWebviewMessage } from "../../../../src/shared/anaconda-desktop-messages"
@@ -35,6 +36,7 @@ export interface SendMessageRequest {
variant?: string
files?: FileAttachment[]
review?: ReviewMessageData
browserFeedback?: BrowserFeedbackData
agentManagerContext?: string
contextDirectory?: string
}
@@ -144,6 +146,7 @@ export interface ImportAndSendMessage {
variant?: string
files?: FileAttachment[]
review?: ReviewMessageData
browserFeedback?: BrowserFeedbackData
command?: string
commandArgs?: string
}
@@ -17,6 +17,7 @@ export function savePromptDraft(
comments: ReviewCommentEntry[],
images: ImageAttachment[],
scroll = 0,
browsers: BrowserReference[] = [],
) {
if (text) drafts.set(key, text)
else drafts.delete(key)
@@ -24,7 +25,9 @@ export function savePromptDraft(
else reviewDrafts.delete(key)
if (images.length > 0) imageDrafts.set(key, images)
else imageDrafts.delete(key)
if (text || comments.length > 0 || images.length > 0) scrollDrafts.set(key, scroll)
if (browsers.length > 0) browserDrafts.set(key, browsers)
else browserDrafts.delete(key)
if (text || comments.length > 0 || images.length > 0 || browsers.length > 0) scrollDrafts.set(key, scroll)
else scrollDrafts.delete(key)
}
@@ -18,24 +18,33 @@ export function createdDraftKey(draftID?: string, sandbox = false): string | und
return pendingDraftKey(draftID) ?? (sandbox ? "new" : undefined)
}
export function movePromptDraft<T, C, I, S>(
stores: { text: Map<string, T>; comments: Map<string, C>; images: Map<string, I>; scrolls: Map<string, S> },
export function movePromptDraft<T, C, I, S, B>(
stores: {
text: Map<string, T>
comments: Map<string, C>
images: Map<string, I>
scrolls: Map<string, S>
browsers?: Map<string, B>
},
source: string,
target: string,
): { text?: T; comments?: C; images?: I; scroll?: S } {
): { text?: T; comments?: C; images?: I; scroll?: S; browsers?: B } {
const draft = {
text: stores.text.get(source),
comments: stores.comments.get(source),
images: stores.images.get(source),
scroll: stores.scrolls.get(source),
...(stores.browsers?.has(source) ? { browsers: stores.browsers.get(source) } : {}),
}
if (draft.text !== undefined) stores.text.set(target, draft.text)
if (draft.comments !== undefined) stores.comments.set(target, draft.comments)
if (draft.images !== undefined) stores.images.set(target, draft.images)
if (draft.scroll !== undefined) stores.scrolls.set(target, draft.scroll)
if (draft.browsers !== undefined) stores.browsers?.set(target, draft.browsers)
stores.text.delete(source)
stores.comments.delete(source)
stores.images.delete(source)
stores.scrolls.delete(source)
stores.browsers?.delete(source)
return draft
}