From 1af79731a8ed925f1f69aa536ba90a53b89e8dfb Mon Sep 17 00:00:00 2001 From: marius-kilocode Date: Thu, 14 May 2026 14:16:51 +0200 Subject: [PATCH 01/10] feat(vscode): add speech-to-text prompt input --- .changeset/speech-input-gateway.md | 6 + .plans/speech-to-text-client.md | 32 +++ bun.lock | 2 - packages/kilo-gateway/src/server/routes.ts | 67 ++++++ packages/kilo-vscode/package.json | 26 +++ packages/kilo-vscode/script/build.ts | 4 + packages/kilo-vscode/script/ffmpeg-helper.ts | 45 ++++ packages/kilo-vscode/script/local-bin.ts | 3 + packages/kilo-vscode/src/KiloProvider.ts | 31 ++- .../src/kilo-provider/early-message.ts | 20 ++ .../kilo-provider/speech-to-text-capture.ts | 198 ++++++++++++++++++ .../kilo-provider/speech-to-text-handler.ts | 83 ++++++++ .../src/kilo-provider/speech-to-text.ts | 106 ++++++++++ .../kilo-vscode/src/services/input-tools.ts | 53 +++++ .../src/services/speech-to-text-config.ts | 13 ++ .../src/services/speech-to-text-settings.ts | 19 ++ .../src/shared/speech-to-text-models.ts | 48 +++++ .../agent-manager/NewWorktreeDialog.tsx | 46 +++- .../src/components/chat/PromptInput.tsx | 57 ++++- .../components/settings/ExperimentalTab.tsx | 52 ++++- .../settings/speech-to-text-model-selector.ts | 13 ++ .../components/shared/SpeechToTextButton.tsx | 73 +++++++ .../webview-ui/src/context/config.tsx | 12 ++ .../webview-ui/src/hooks/useSpeechToText.ts | 169 +++++++++++++++ .../kilo-vscode/webview-ui/src/i18n/en.ts | 22 ++ .../webview-ui/src/styles/prompt-input.css | 48 +++++ .../src/types/messages/extension-messages.ts | 36 ++++ .../src/types/messages/webview-messages.ts | 25 +++ packages/sdk/js/src/v2/gen/sdk.gen.ts | 59 ++++++ packages/sdk/js/src/v2/gen/types.gen.ts | 39 ++++ 30 files changed, 1395 insertions(+), 12 deletions(-) create mode 100644 .changeset/speech-input-gateway.md create mode 100644 .plans/speech-to-text-client.md create mode 100644 packages/kilo-vscode/script/ffmpeg-helper.ts create mode 100644 packages/kilo-vscode/src/kilo-provider/early-message.ts create mode 100644 packages/kilo-vscode/src/kilo-provider/speech-to-text-capture.ts create mode 100644 packages/kilo-vscode/src/kilo-provider/speech-to-text-handler.ts create mode 100644 packages/kilo-vscode/src/kilo-provider/speech-to-text.ts create mode 100644 packages/kilo-vscode/src/services/input-tools.ts create mode 100644 packages/kilo-vscode/src/services/speech-to-text-config.ts create mode 100644 packages/kilo-vscode/src/services/speech-to-text-settings.ts create mode 100644 packages/kilo-vscode/src/shared/speech-to-text-models.ts create mode 100644 packages/kilo-vscode/webview-ui/src/components/settings/speech-to-text-model-selector.ts create mode 100644 packages/kilo-vscode/webview-ui/src/components/shared/SpeechToTextButton.tsx create mode 100644 packages/kilo-vscode/webview-ui/src/hooks/useSpeechToText.ts diff --git a/.changeset/speech-input-gateway.md b/.changeset/speech-input-gateway.md new file mode 100644 index 00000000000..cf48a6b19bb --- /dev/null +++ b/.changeset/speech-input-gateway.md @@ -0,0 +1,6 @@ +--- +"kilo-code": minor +"@kilocode/kilo-gateway": minor +--- + +Support experimental speech-to-text voice input in VS Code prompt fields through Kilo Gateway. diff --git a/.plans/speech-to-text-client.md b/.plans/speech-to-text-client.md new file mode 100644 index 00000000000..609ad69c086 --- /dev/null +++ b/.plans/speech-to-text-client.md @@ -0,0 +1,32 @@ +# Speech input client plan + +## Assumptions + +- Backend exposes `POST /api/gateway/v1/audio/transcriptions` through the local CLI's Kilo routes. +- Request body is JSON: `{ model, input_audio: { data, format }, language? }`. +- Response body is JSON: `{ text, usage? }`. +- Client MVP is request/response transcription after the user stops recording, not live partial transcription. +- VS Code webviews cannot access `navigator.mediaDevices.getUserMedia`, so microphone capture runs in the extension host. + +## Client scope + +1. Add an experimental Speech to Text setting, default off. +2. Add an extension-side STT handler that uses the existing Kilo auth stored in the CLI backend. +3. Record microphone audio in the extension host with a bundled FFmpeg helper, falling back to system FFmpeg if needed. +4. Add a shared webview hook for `idle -> recording -> transcribing -> error` state. +5. Add a shared prompt action button that starts/stops recording and shows clean state. +6. Wire the same hook/button into `PromptInput` and Agent Manager `NewWorktreeDialog`. +7. Insert successful transcripts at the current caret, preserve existing prompt content, and never auto-submit. + +## Error states + +- Experiment disabled: hide the button. +- Missing Kilo login or disabled Kilo provider: disable settings and hide the button. +- Missing bundled helper or OS microphone denial: show a local recording error. +- Empty recording or empty transcript: show a local error. +- Gateway/network failure: show the backend error and keep input unchanged. + +## Follow-up + +- Replace the static model list with `/api/gateway/v1/transcription-models` discovery if we want automatic model updates. +- Add microphone device selection if users need to choose a non-default input. diff --git a/bun.lock b/bun.lock index 33fca023363..68543a32236 100644 --- a/bun.lock +++ b/bun.lock @@ -615,8 +615,6 @@ "@opentui/solid": "0.2.2", "@pierre/diffs": "1.1.0-beta.18", "@playwright/test": "1.59.1", - "@sentry/solid": "10.36.0", - "@sentry/vite-plugin": "4.6.0", "@solid-primitives/storage": "4.3.3", "@solidjs/meta": "0.29.4", "@solidjs/router": "0.15.4", diff --git a/packages/kilo-gateway/src/server/routes.ts b/packages/kilo-gateway/src/server/routes.ts index 269427b7f44..b4f1c4e8f60 100644 --- a/packages/kilo-gateway/src/server/routes.ts +++ b/packages/kilo-gateway/src/server/routes.ts @@ -132,6 +132,11 @@ export function createKiloRoutes(deps: KiloRoutesDeps) { cost: z.number().optional(), }) + const TranscriptionResponse = z.object({ + text: z.string(), + usage: z.unknown().optional(), + }) + return new Hono() .get( "/profile", @@ -390,6 +395,68 @@ export function createKiloRoutes(deps: KiloRoutesDeps) { }) }, ) + .post( + "/audio/transcriptions", + describeRoute({ + summary: "Speech to text transcription", + description: "Proxy an audio transcription request to the Kilo Gateway", + operationId: "kilo.audio.transcriptions", + responses: { + 200: { + description: "Transcription response", + content: { + "application/json": { + schema: resolver(TranscriptionResponse), + }, + }, + }, + ...errors(400, 401), + }, + }), + validator( + "json", + z.object({ + model: z.string(), + input_audio: z.object({ + data: z.string(), + format: z.string(), + }), + language: z.string().optional(), + temperature: z.number().optional(), + }), + ), + async (c: any) => { + const auth = await Auth.get("kilo") + if (!auth) return c.json({ error: "Not authenticated with Kilo Gateway" }, 401) + + const token = auth.type === "api" ? auth.key : auth.type === "oauth" ? auth.access : undefined + if (!token) return c.json({ error: "No valid token found" }, 401) + + const organizationId = auth.type === "oauth" ? auth.accountId : undefined + const body = c.req.valid("json") + const headers = { + "Content-Type": "application/json", + Authorization: `Bearer ${token}`, + ...buildKiloHeaders(undefined, { kilocodeOrganizationId: organizationId }), + [HEADER_FEATURE]: "vscode-extension", + } + + const response = await fetch(`${KILO_API_BASE}/api/gateway/v1/audio/transcriptions`, { + method: "POST", + headers, + signal: c.req.raw.signal, + body: JSON.stringify(body), + }) + + const text = await response.text() + return new Response(text, { + status: response.status, + headers: { + "Content-Type": response.headers.get("Content-Type") ?? "application/json", + }, + }) + }, + ) .get( "/notifications", describeRoute({ diff --git a/packages/kilo-vscode/package.json b/packages/kilo-vscode/package.json index a66d000db5b..4a7187c1a84 100644 --- a/packages/kilo-vscode/package.json +++ b/packages/kilo-vscode/package.json @@ -776,6 +776,32 @@ "default": false, "description": "Enable chat textarea autocomplete" }, + "kilo-code.new.speechToText.enabled": { + "type": "boolean", + "default": false, + "description": "Enable experimental speech-to-text voice input in prompt fields. Uses your Kilo account through the Kilo Gateway." + }, + "kilo-code.new.speechToText.model": { + "type": "string", + "default": "openai/gpt-4o-mini-transcribe", + "enum": [ + "openai/gpt-4o-mini-transcribe", + "openai/gpt-4o-transcribe", + "openai/whisper-1", + "openai/whisper-large-v3-turbo", + "openai/whisper-large-v3", + "google/chirp-3" + ], + "enumDescriptions": [ + "OpenAI GPT-4o Mini Transcribe", + "OpenAI GPT-4o Transcribe", + "OpenAI Whisper 1", + "Groq-hosted Whisper Large V3 Turbo", + "Groq-hosted Whisper Large V3", + "Google Chirp 3" + ], + "description": "Model to use for experimental speech-to-text voice input. Requires Kilo Gateway." + }, "kilo-code.new.claudeCodeCompat": { "type": "boolean", "default": false, diff --git a/packages/kilo-vscode/script/build.ts b/packages/kilo-vscode/script/build.ts index cb285b327b4..58e71c24095 100644 --- a/packages/kilo-vscode/script/build.ts +++ b/packages/kilo-vscode/script/build.ts @@ -2,6 +2,7 @@ import { $ } from "bun" import { join } from "node:path" import { existsSync, mkdirSync, rmSync, chmodSync } from "node:fs" +import { ensureFfmpegForTarget } from "./ffmpeg-helper" const packageJsonPath = join(import.meta.dir, "..", "package.json") const packageJson = await Bun.file(packageJsonPath).json() @@ -81,6 +82,9 @@ for (const config of targets) { console.log(` ✅ Binary ready at ${targetBinary}`) + console.log(" 🎙️ Adding bundled FFmpeg helper...") + await ensureFfmpegForTarget(config.target, binDir) + console.log(` 📦 Packaging .vsix for ${config.target}${prerelease ? " (pre-release)" : ""}...`) const vsixPath = join(outDir, `kilo-vscode-${config.target}.vsix`) const args = ["--no-dependencies", "--skip-license", "--target", config.target, "-o", vsixPath] diff --git a/packages/kilo-vscode/script/ffmpeg-helper.ts b/packages/kilo-vscode/script/ffmpeg-helper.ts new file mode 100644 index 00000000000..798f513d82b --- /dev/null +++ b/packages/kilo-vscode/script/ffmpeg-helper.ts @@ -0,0 +1,45 @@ +import { $ } from "bun" +import { chmodSync, copyFileSync, existsSync, mkdirSync, rmSync } from "node:fs" +import { join } from "node:path" + +const packages: Record = { + "darwin-x64": "@ffmpeg-installer/darwin-x64@4.1.0", + "darwin-arm64": "@ffmpeg-installer/darwin-arm64@4.1.5", + "linux-x64": "@ffmpeg-installer/linux-x64@4.1.0", + "linux-arm64": "@ffmpeg-installer/linux-arm64@4.1.4", + "alpine-x64": "@ffmpeg-installer/linux-x64@4.1.0", + "alpine-arm64": "@ffmpeg-installer/linux-arm64@4.1.4", + "win32-x64": "@ffmpeg-installer/win32-x64@4.1.0", + "win32-arm64": "@ffmpeg-installer/win32-x64@4.1.0", +} + +export async function ensureFfmpegForTarget(target: string, bin: string): Promise { + const spec = packages[target] + if (!spec) throw new Error(`No FFmpeg helper package configured for target ${target}`) + + const exe = target.startsWith("win32") ? "ffmpeg.exe" : "ffmpeg" + const dest = join(bin, exe) + if (existsSync(dest)) return + + const tmp = join(bin, ".ffmpeg-tmp") + rmSync(tmp, { recursive: true, force: true }) + mkdirSync(tmp, { recursive: true }) + + try { + const packed = await $`npm pack ${spec} --pack-destination ${tmp}`.quiet() + const name = packed.text().trim().split(/\s+/).pop() + if (!name) throw new Error(`npm pack did not return a tarball for ${spec}`) + + await $`tar -xzf ${join(tmp, name)} -C ${tmp}`.quiet() + copyFileSync(join(tmp, "package", exe), dest) + if (!target.startsWith("win32")) chmodSync(dest, 0o755) + } finally { + rmSync(tmp, { recursive: true, force: true }) + } +} + +export function currentFfmpegTarget(): string { + const os = process.platform === "win32" ? "win32" : process.platform + const arch = process.arch === "x64" ? "x64" : process.arch === "arm64" ? "arm64" : process.arch + return `${os}-${arch}` +} diff --git a/packages/kilo-vscode/script/local-bin.ts b/packages/kilo-vscode/script/local-bin.ts index aeb57e81643..a8d2f669bde 100644 --- a/packages/kilo-vscode/script/local-bin.ts +++ b/packages/kilo-vscode/script/local-bin.ts @@ -2,6 +2,7 @@ import { $ } from "bun" import { join, relative, dirname, basename } from "node:path" import { chmodSync, statSync, rmSync, readdirSync, existsSync } from "node:fs" +import { currentFfmpegTarget, ensureFfmpegForTarget } from "./ffmpeg-helper" const forceRebuild = process.argv.includes("--force") @@ -158,6 +159,7 @@ async function main() { log( `CLI binary already present at ${relative(kiloVscodeDir, targetBinPath)} (${Math.round(st.size / 1024 / 1024)}MB). Use --force to rebuild.`, ) + await ensureFfmpegForTarget(currentFfmpegTarget(), targetBinDir) return } @@ -181,6 +183,7 @@ async function main() { await $`mkdir -p ${targetBinDir}` await $`cp ${sourceBinPath} ${targetBinPath}` chmodSync(targetBinPath, 0o755) + await ensureFfmpegForTarget(currentFfmpegTarget(), targetBinDir) // Record the CLI source version so future runs detect when a rebuild is needed const hash = await cliSourceHash() diff --git a/packages/kilo-vscode/src/KiloProvider.ts b/packages/kilo-vscode/src/KiloProvider.ts index cf3c33479e2..7a2344436f1 100644 --- a/packages/kilo-vscode/src/KiloProvider.ts +++ b/packages/kilo-vscode/src/KiloProvider.ts @@ -68,10 +68,12 @@ import { handleNetworkEvent, clearNetworkWaits } from "./kilo-provider/network" import { abortSession } from "./kilo-provider/abort" import { buildAutocompleteSettingsMessage, - routeAutocompleteMessage, validAutocompleteSetting, watchAutocompleteConfig, } from "./services/autocomplete/settings" +import { validSpeechToTextSetting } from "./services/speech-to-text-settings" +import { sendSpeechToTextSettings, watchSpeechToTextConfig } from "./services/speech-to-text-config" +import { routeEarlyMessage } from "./kilo-provider/early-message" import * as ModelState from "./kilo-provider/model-state" import { handleForkSession } from "./kilo-provider/fork-session" import { openConfig } from "./kilo-provider/open-config" @@ -112,7 +114,7 @@ import { handleQuestionReject, fetchAndSendPendingQuestions, } from "./kilo-provider/handlers/question" -import { fetchAndSendPendingSuggestions, routeSuggestionWebviewMessage } from "./kilo-provider/handlers/suggestion" +import { fetchAndSendPendingSuggestions } from "./kilo-provider/handlers/suggestion" import { nativeTitle } from "./kilo-provider/native-tab-title" import { @@ -229,6 +231,7 @@ export class KiloProvider implements vscode.WebviewViewProvider, TelemetryProper private webviewMessageDisposable: vscode.Disposable | null = null private autocompleteConfigDisposable: vscode.Disposable | null = null private telemetryStateDisposable: vscode.Disposable | null = null + private speechToTextDisposable: vscode.Disposable | null = null private viewStateDisposable: vscode.Disposable | null = null private visibilityDisposable: vscode.Disposable | null = null private autoApproveBridge: ReturnType | null = null @@ -592,6 +595,8 @@ export class KiloProvider implements vscode.WebviewViewProvider, TelemetryProper this.webviewMessageDisposable?.dispose() this.autocompleteConfigDisposable?.dispose() this.autocompleteConfigDisposable = watchAutocompleteConfig((msg) => this.postMessage(msg)) + this.speechToTextDisposable?.dispose() + this.speechToTextDisposable = watchSpeechToTextConfig(this) this.telemetryStateDisposable?.dispose() this.telemetryStateDisposable = watchTelemetryState((msg) => this.postMessage(msg)) this.webviewMessageDisposable = webview.onDidReceiveMessage(async (message) => { @@ -604,9 +609,17 @@ export class KiloProvider implements vscode.WebviewViewProvider, TelemetryProper if (intercepted === null) return message = intercepted - await routeSuggestionWebviewMessage(this.questionCtx, message) - if (await ModelState.handleMessage(message.type, message, this.client, (msg) => this.postMessage(msg))) return - if (await routeAutocompleteMessage(message, (msg) => this.postMessage(msg))) return + if ( + await routeEarlyMessage(message, { + question: this.questionCtx, + client: this.client, + connection: this.connectionService, + dir: this.getWorkspaceDirectory(this.currentSession?.id), + post: (msg) => this.postMessage(msg), + }) + ) { + return + } if ( await handleSidebarWorktreeMessage(message, { post: (msg) => this.postMessage(msg), @@ -1301,6 +1314,7 @@ export class KiloProvider implements vscode.WebviewViewProvider, TelemetryProper this.postMessage({ type: "gitStatus", repo: this.cachedGitRepo }) this.sendNotificationSettings() this.sendTimelineSetting() + this.sendSpeechToTextSettings() this.postMessage({ type: "extensionDataReady" }) if (this.cachedGitRepo) this.startStatsPolling() @@ -2341,6 +2355,10 @@ export class KiloProvider implements vscode.WebviewViewProvider, TelemetryProper }) } + private sendSpeechToTextSettings(): void { + sendSpeechToTextSettings(this) + } + /** Returns the number of sessions currently in "busy" state. */ private getBusySessionCount(): number { return getBusySessionCount(this.sessionStatusMap) @@ -2920,6 +2938,7 @@ export class KiloProvider implements vscode.WebviewViewProvider, TelemetryProper private async handleUpdateSetting(key: string, value: unknown): Promise { const { section, leaf } = buildSettingPath(key) if (section === "autocomplete" && !validAutocompleteSetting(leaf, value)) return + if (section === "speechToText" && !validSpeechToTextSetting(leaf, value)) return const config = vscode.workspace.getConfiguration(`kilo-code.new${section ? `.${section}` : ""}`) await config.update(leaf, value, vscode.ConfigurationTarget.Global) } @@ -2960,6 +2979,7 @@ export class KiloProvider implements vscode.WebviewViewProvider, TelemetryProper // Re-send all settings to the webview so the UI reflects the reset this.postMessage(buildAutocompleteSettingsMessage()) + this.sendSpeechToTextSettings() this.sendBrowserSettings() this.sendNotificationSettings() this.sendTimelineSetting() @@ -3460,6 +3480,7 @@ export class KiloProvider implements vscode.WebviewViewProvider, TelemetryProper this.visibilityDisposable?.dispose() this.webviewMessageDisposable?.dispose() this.autocompleteConfigDisposable?.dispose() + this.speechToTextDisposable?.dispose() this.telemetryStateDisposable?.dispose() this.autoApproveBridge?.dispose() this.streams.dispose() diff --git a/packages/kilo-vscode/src/kilo-provider/early-message.ts b/packages/kilo-vscode/src/kilo-provider/early-message.ts new file mode 100644 index 00000000000..74b73a14955 --- /dev/null +++ b/packages/kilo-vscode/src/kilo-provider/early-message.ts @@ -0,0 +1,20 @@ +import { routeSuggestionWebviewMessage } from "./handlers/suggestion" +import * as ModelState from "./model-state" +import { routeInputToolMessage } from "../services/input-tools" +import type { KiloConnectionService } from "../services/cli-backend/connection-service" +import type { SuggestionContext } from "./handlers/suggestion" +import type { KiloClient } from "@kilocode/sdk/v2/client" + +type Ctx = { + question: SuggestionContext + client: KiloClient | null + connection: KiloConnectionService + dir: string + post: (msg: unknown) => void +} + +export async function routeEarlyMessage(message: { type: string }, ctx: Ctx): Promise { + await routeSuggestionWebviewMessage(ctx.question, message) + if (await ModelState.handleMessage(message.type, message, ctx.client, ctx.post)) return true + return await routeInputToolMessage(message, { connection: ctx.connection, dir: ctx.dir, post: ctx.post }) +} diff --git a/packages/kilo-vscode/src/kilo-provider/speech-to-text-capture.ts b/packages/kilo-vscode/src/kilo-provider/speech-to-text-capture.ts new file mode 100644 index 00000000000..89895114298 --- /dev/null +++ b/packages/kilo-vscode/src/kilo-provider/speech-to-text-capture.ts @@ -0,0 +1,198 @@ +import { existsSync } from "fs" +import { readFile, stat, unlink } from "fs/promises" +import * as os from "os" +import * as path from "path" +import type { ChildProcess } from "child_process" +import { exec, spawn } from "../util/process" + +type Input = { + requestId: string + model: string + language?: string +} + +type Recording = Input & { + file: string + proc: ChildProcess + stderr: string[] + stopped: boolean + exit?: { + code: number | null + signal: string | null + } +} + +type Audio = { + data: string + format: "wav" + model: string + language?: string +} + +let active: Recording | undefined + +export async function startSpeechCapture(input: Input): Promise { + if (active) throw new Error("Speech recording is already in progress") + + const bin = await findFFmpeg() + const file = path.join(os.tmpdir(), `kilo-stt-${process.pid}-${Date.now()}.wav`) + const args = ["-y", ...inputArgs(), "-acodec", "pcm_s16le", "-ar", "16000", "-ac", "1", "-f", "wav", file] + const proc = spawn(bin, args, { stdio: ["pipe", "ignore", "pipe"] }) + const state: Recording = { ...input, file, proc, stderr: [], stopped: false } + active = state + + proc.stderr?.on("data", (data: Buffer) => { + if (state.stderr.length < 20) state.stderr.push(data.toString()) + }) + proc.on("exit", (code, signal) => { + state.exit = { code, signal } + if (active === state && !state.stopped) active = undefined + }) + proc.on("error", (err) => { + state.stderr.push(err.message) + if (active === state && !state.stopped) active = undefined + }) + + await waitForStart(state) +} + +export async function stopSpeechCapture(requestId: string): Promise