diff --git a/.changeset/speech-input-gateway.md b/.changeset/speech-input-gateway.md new file mode 100644 index 00000000000..cf48a6b19bb --- /dev/null +++ b/.changeset/speech-input-gateway.md @@ -0,0 +1,6 @@ +--- +"kilo-code": minor +"@kilocode/kilo-gateway": minor +--- + +Support experimental speech-to-text voice input in VS Code prompt fields through Kilo Gateway. diff --git a/packages/kilo-gateway/src/server/routes.ts b/packages/kilo-gateway/src/server/routes.ts index 269427b7f44..f2cd17498fa 100644 --- a/packages/kilo-gateway/src/server/routes.ts +++ b/packages/kilo-gateway/src/server/routes.ts @@ -132,6 +132,21 @@ export function createKiloRoutes(deps: KiloRoutesDeps) { cost: z.number().optional(), }) + const TranscriptionResponse = z.object({ + text: z.string(), + usage: z.unknown().optional(), + }) + + const getProxyAuth = async () => { + const auth = await Auth.get("kilo") + const token = auth?.type === "api" ? auth.key : auth?.type === "oauth" ? auth.access : undefined + return { + auth, + token, + organizationId: auth?.type === "oauth" ? auth.accountId : undefined, + } + } + return new Hono() .get( "/profile", @@ -324,19 +339,16 @@ export function createKiloRoutes(deps: KiloRoutesDeps) { }), ), async (c: any) => { - const auth = await Auth.get("kilo") + const proxy = await getProxyAuth() - if (!auth) { + if (!proxy.auth) { return c.json({ error: "Not authenticated with Kilo Gateway" }, 401) } - const token = auth.type === "api" ? auth.key : auth.type === "oauth" ? auth.access : undefined - if (!token) { + if (!proxy.token) { return c.json({ error: "No valid token found" }, 401) } - const organizationId = auth.type === "oauth" ? auth.accountId : undefined - const { prefix, suffix, model, maxTokens, temperature } = c.req.valid("json") const fimModel = model ?? "mistralai/codestral-2501" const fimMaxTokens = maxTokens ?? 256 @@ -347,8 +359,8 @@ export function createKiloRoutes(deps: KiloRoutesDeps) { const headers = { "Content-Type": "application/json", - Authorization: `Bearer ${token}`, - ...buildKiloHeaders(undefined, { kilocodeOrganizationId: organizationId }), + Authorization: `Bearer ${proxy.token}`, + ...buildKiloHeaders(undefined, { kilocodeOrganizationId: proxy.organizationId }), [HEADER_FEATURE]: "autocomplete", } @@ -390,6 +402,66 @@ export function createKiloRoutes(deps: KiloRoutesDeps) { }) }, ) + .post( + "/audio/transcriptions", + describeRoute({ + summary: "Speech to text transcription", + description: "Proxy an audio transcription request to the Kilo Gateway", + operationId: "kilo.audio.transcriptions", + responses: { + 200: { + description: "Transcription response", + content: { + "application/json": { + schema: resolver(TranscriptionResponse), + }, + }, + }, + ...errors(400, 401), + }, + }), + validator( + "json", + z.object({ + model: z.string(), + input_audio: z.object({ + data: z.string(), + format: z.string(), + }), + language: z.string().optional(), + temperature: z.number().optional(), + }), + ), + async (c: any) => { + const proxy = await getProxyAuth() + if (!proxy.auth) return c.json({ error: "Not authenticated with Kilo Gateway" }, 401) + + if (!proxy.token) return c.json({ error: "No valid token found" }, 401) + + const body = c.req.valid("json") + const headers = { + "Content-Type": "application/json", + Authorization: `Bearer ${proxy.token}`, + ...buildKiloHeaders(undefined, { kilocodeOrganizationId: proxy.organizationId }), + [HEADER_FEATURE]: "vscode-extension", + } + + const response = await fetch(`${KILO_API_BASE}/api/gateway/v1/audio/transcriptions`, { + method: "POST", + headers, + signal: c.req.raw.signal, + body: JSON.stringify(body), + }) + + const text = await response.text() + return new Response(text, { + status: response.status, + headers: { + "Content-Type": response.headers.get("Content-Type") ?? "application/json", + }, + }) + }, + ) .get( "/notifications", describeRoute({ diff --git a/packages/kilo-vscode/package.json b/packages/kilo-vscode/package.json index 686adb943fc..d073c0dde9d 100644 --- a/packages/kilo-vscode/package.json +++ b/packages/kilo-vscode/package.json @@ -776,6 +776,15 @@ "default": false, "description": "Enable chat textarea autocomplete" }, + "kilo-code.new.speechToText.enabled": { + "type": "boolean", + "default": false, + "description": "Enable experimental speech-to-text voice input in prompt fields. Uses your Kilo account through the Kilo Gateway." + }, + "kilo-code.new.speechToText.model": { + "type": "string", + "description": "Model to use for experimental speech-to-text voice input. Requires Kilo Gateway." + }, "kilo-code.new.claudeCodeCompat": { "type": "boolean", "default": false, diff --git a/packages/kilo-vscode/script/build.ts b/packages/kilo-vscode/script/build.ts index cb285b327b4..b611a06e32a 100644 --- a/packages/kilo-vscode/script/build.ts +++ b/packages/kilo-vscode/script/build.ts @@ -2,6 +2,7 @@ import { $ } from "bun" import { join } from "node:path" import { existsSync, mkdirSync, rmSync, chmodSync } from "node:fs" +import { ensureFfmpegForTarget } from "./ffmpeg-helper" const packageJsonPath = join(import.meta.dir, "..", "package.json") const packageJson = await Bun.file(packageJsonPath).json() @@ -81,6 +82,9 @@ for (const config of targets) { console.log(` ✅ Binary ready at ${targetBinary}`) + console.log("Adding bundled FFmpeg helper...") + await ensureFfmpegForTarget(config.target, binDir) + console.log(` 📦 Packaging .vsix for ${config.target}${prerelease ? " (pre-release)" : ""}...`) const vsixPath = join(outDir, `kilo-vscode-${config.target}.vsix`) const args = ["--no-dependencies", "--skip-license", "--target", config.target, "-o", vsixPath] diff --git a/packages/kilo-vscode/script/ffmpeg-helper.ts b/packages/kilo-vscode/script/ffmpeg-helper.ts new file mode 100644 index 00000000000..c0b5e440ef4 --- /dev/null +++ b/packages/kilo-vscode/script/ffmpeg-helper.ts @@ -0,0 +1,49 @@ +import { $ } from "bun" +import { chmodSync, copyFileSync, existsSync, mkdirSync, rmSync } from "node:fs" +import { join } from "node:path" + +const packages: Record = { + "darwin-x64": "@ffmpeg-installer/darwin-x64@4.1.0", + "darwin-arm64": "@ffmpeg-installer/darwin-arm64@4.1.5", + "linux-x64": "@ffmpeg-installer/linux-x64@4.1.0", + "linux-arm64": "@ffmpeg-installer/linux-arm64@4.1.4", + "alpine-x64": "@ffmpeg-installer/linux-x64@4.1.0", + "alpine-arm64": "@ffmpeg-installer/linux-arm64@4.1.4", + "win32-x64": "@ffmpeg-installer/win32-x64@4.1.0", +} + +export async function ensureFfmpegForTarget(target: string, bin: string): Promise { + if (target === "win32-arm64") { + console.warn("No Windows ARM64 FFmpeg helper package is available; speech input will use system FFmpeg.") + return + } + + const spec = packages[target] + if (!spec) throw new Error(`No FFmpeg helper package configured for target ${target}`) + + const exe = target.startsWith("win32") ? "ffmpeg.exe" : "ffmpeg" + const dest = join(bin, exe) + if (existsSync(dest)) return + + const tmp = join(bin, ".ffmpeg-tmp") + rmSync(tmp, { recursive: true, force: true }) + mkdirSync(tmp, { recursive: true }) + + try { + const packed = await $`npm pack ${spec} --pack-destination ${tmp}`.quiet() + const name = packed.text().trim().split(/\s+/).pop() + if (!name) throw new Error(`npm pack did not return a tarball for ${spec}`) + + await $`tar -xzf ${join(tmp, name)} -C ${tmp}`.quiet() + copyFileSync(join(tmp, "package", exe), dest) + if (!target.startsWith("win32")) chmodSync(dest, 0o755) + } finally { + rmSync(tmp, { recursive: true, force: true }) + } +} + +export function currentFfmpegTarget(): string { + const os = process.platform === "win32" ? "win32" : process.platform + const arch = process.arch === "x64" ? "x64" : process.arch === "arm64" ? "arm64" : process.arch + return `${os}-${arch}` +} diff --git a/packages/kilo-vscode/script/local-bin.ts b/packages/kilo-vscode/script/local-bin.ts index aeb57e81643..a8d2f669bde 100644 --- a/packages/kilo-vscode/script/local-bin.ts +++ b/packages/kilo-vscode/script/local-bin.ts @@ -2,6 +2,7 @@ import { $ } from "bun" import { join, relative, dirname, basename } from "node:path" import { chmodSync, statSync, rmSync, readdirSync, existsSync } from "node:fs" +import { currentFfmpegTarget, ensureFfmpegForTarget } from "./ffmpeg-helper" const forceRebuild = process.argv.includes("--force") @@ -158,6 +159,7 @@ async function main() { log( `CLI binary already present at ${relative(kiloVscodeDir, targetBinPath)} (${Math.round(st.size / 1024 / 1024)}MB). Use --force to rebuild.`, ) + await ensureFfmpegForTarget(currentFfmpegTarget(), targetBinDir) return } @@ -181,6 +183,7 @@ async function main() { await $`mkdir -p ${targetBinDir}` await $`cp ${sourceBinPath} ${targetBinPath}` chmodSync(targetBinPath, 0o755) + await ensureFfmpegForTarget(currentFfmpegTarget(), targetBinDir) // Record the CLI source version so future runs detect when a rebuild is needed const hash = await cliSourceHash() diff --git a/packages/kilo-vscode/src/KiloProvider.ts b/packages/kilo-vscode/src/KiloProvider.ts index 0aa9c66aba8..a3ed7bd2215 100644 --- a/packages/kilo-vscode/src/KiloProvider.ts +++ b/packages/kilo-vscode/src/KiloProvider.ts @@ -68,10 +68,12 @@ import { handleNetworkEvent, clearNetworkWaits } from "./kilo-provider/network" import { abortSession } from "./kilo-provider/abort" import { buildAutocompleteSettingsMessage, - routeAutocompleteMessage, validAutocompleteSetting, watchAutocompleteConfig, } from "./services/autocomplete/settings" +import { validSpeechToTextSetting } from "./speech-to-text/settings" +import { sendSpeechToTextSettings, watchSpeechToTextConfig } from "./speech-to-text/config" +import { routeEarlyMessage } from "./kilo-provider/early-message" import * as ModelState from "./kilo-provider/model-state" import { handleForkSession } from "./kilo-provider/fork-session" import { openConfig } from "./kilo-provider/open-config" @@ -112,7 +114,7 @@ import { handleQuestionReject, fetchAndSendPendingQuestions, } from "./kilo-provider/handlers/question" -import { fetchAndSendPendingSuggestions, routeSuggestionWebviewMessage } from "./kilo-provider/handlers/suggestion" +import { fetchAndSendPendingSuggestions } from "./kilo-provider/handlers/suggestion" import { nativeTitle } from "./kilo-provider/native-tab-title" import { @@ -229,6 +231,7 @@ export class KiloProvider implements vscode.WebviewViewProvider, TelemetryProper private webviewMessageDisposable: vscode.Disposable | null = null private autocompleteConfigDisposable: vscode.Disposable | null = null private telemetryStateDisposable: vscode.Disposable | null = null + private speechToTextDisposable: vscode.Disposable | null = null private viewStateDisposable: vscode.Disposable | null = null private visibilityDisposable: vscode.Disposable | null = null private autoApproveBridge: ReturnType | null = null @@ -592,6 +595,8 @@ export class KiloProvider implements vscode.WebviewViewProvider, TelemetryProper this.webviewMessageDisposable?.dispose() this.autocompleteConfigDisposable?.dispose() this.autocompleteConfigDisposable = watchAutocompleteConfig((msg) => this.postMessage(msg)) + this.speechToTextDisposable?.dispose() + this.speechToTextDisposable = watchSpeechToTextConfig(this) this.telemetryStateDisposable?.dispose() this.telemetryStateDisposable = watchTelemetryState((msg) => this.postMessage(msg)) this.webviewMessageDisposable = webview.onDidReceiveMessage(async (message) => { @@ -604,9 +609,17 @@ export class KiloProvider implements vscode.WebviewViewProvider, TelemetryProper if (intercepted === null) return message = intercepted - await routeSuggestionWebviewMessage(this.questionCtx, message) - if (await ModelState.handleMessage(message.type, message, this.client, (msg) => this.postMessage(msg))) return - if (await routeAutocompleteMessage(message, (msg) => this.postMessage(msg))) return + if ( + await routeEarlyMessage(message, { + question: this.questionCtx, + client: this.client, + connection: this.connectionService, + dir: this.getWorkspaceDirectory(this.currentSession?.id), + post: (msg) => this.postMessage(msg), + }) + ) { + return + } if (this.handleEditorOpenMessage(message)) return if ( await handleSidebarWorktreeMessage(message, { @@ -1297,6 +1310,7 @@ export class KiloProvider implements vscode.WebviewViewProvider, TelemetryProper this.postMessage({ type: "gitStatus", repo: this.cachedGitRepo }) this.sendNotificationSettings() this.sendTimelineSetting() + this.sendSpeechToTextSettings() this.postMessage({ type: "extensionDataReady" }) if (this.cachedGitRepo) this.startStatsPolling() @@ -2337,6 +2351,10 @@ export class KiloProvider implements vscode.WebviewViewProvider, TelemetryProper }) } + private sendSpeechToTextSettings(): void { + sendSpeechToTextSettings(this) + } + /** Returns the number of sessions currently in "busy" state. */ private getBusySessionCount(): number { return getBusySessionCount(this.sessionStatusMap) @@ -2953,6 +2971,7 @@ export class KiloProvider implements vscode.WebviewViewProvider, TelemetryProper private async handleUpdateSetting(key: string, value: unknown): Promise { const { section, leaf } = buildSettingPath(key) if (section === "autocomplete" && !validAutocompleteSetting(leaf, value)) return + if (section === "speechToText" && !validSpeechToTextSetting(leaf, value)) return const config = vscode.workspace.getConfiguration(`kilo-code.new${section ? `.${section}` : ""}`) await config.update(leaf, value, vscode.ConfigurationTarget.Global) } @@ -2993,6 +3012,7 @@ export class KiloProvider implements vscode.WebviewViewProvider, TelemetryProper // Re-send all settings to the webview so the UI reflects the reset this.postMessage(buildAutocompleteSettingsMessage()) + this.sendSpeechToTextSettings() this.sendBrowserSettings() this.sendNotificationSettings() this.sendTimelineSetting() @@ -3494,6 +3514,7 @@ export class KiloProvider implements vscode.WebviewViewProvider, TelemetryProper this.visibilityDisposable?.dispose() this.webviewMessageDisposable?.dispose() this.autocompleteConfigDisposable?.dispose() + this.speechToTextDisposable?.dispose() this.telemetryStateDisposable?.dispose() this.autoApproveBridge?.dispose() this.streams.dispose() diff --git a/packages/kilo-vscode/src/kilo-provider/early-message.ts b/packages/kilo-vscode/src/kilo-provider/early-message.ts new file mode 100644 index 00000000000..74b73a14955 --- /dev/null +++ b/packages/kilo-vscode/src/kilo-provider/early-message.ts @@ -0,0 +1,20 @@ +import { routeSuggestionWebviewMessage } from "./handlers/suggestion" +import * as ModelState from "./model-state" +import { routeInputToolMessage } from "../services/input-tools" +import type { KiloConnectionService } from "../services/cli-backend/connection-service" +import type { SuggestionContext } from "./handlers/suggestion" +import type { KiloClient } from "@kilocode/sdk/v2/client" + +type Ctx = { + question: SuggestionContext + client: KiloClient | null + connection: KiloConnectionService + dir: string + post: (msg: unknown) => void +} + +export async function routeEarlyMessage(message: { type: string }, ctx: Ctx): Promise { + await routeSuggestionWebviewMessage(ctx.question, message) + if (await ModelState.handleMessage(message.type, message, ctx.client, ctx.post)) return true + return await routeInputToolMessage(message, { connection: ctx.connection, dir: ctx.dir, post: ctx.post }) +} diff --git a/packages/kilo-vscode/src/services/input-tools.ts b/packages/kilo-vscode/src/services/input-tools.ts new file mode 100644 index 00000000000..86180d9b50c --- /dev/null +++ b/packages/kilo-vscode/src/services/input-tools.ts @@ -0,0 +1,49 @@ +import type { KiloConnectionService } from "./cli-backend/connection-service" +import { routeAutocompleteMessage } from "./autocomplete/settings" +import { buildSpeechToTextSettingsMessage } from "../speech-to-text/settings" +import { handleSpeechToTextCancel, handleSpeechToTextStart, handleSpeechToTextStop } from "../speech-to-text/handler" + +type Msg = { + type: string + requestId?: string + model?: string + language?: string +} + +type Ctx = { + connection: KiloConnectionService + dir: string + post: (msg: unknown) => void +} + +export async function routeInputToolMessage(message: Msg, ctx: Ctx): Promise { + if (await routeAutocompleteMessage(message, ctx.post)) return true + + if (message.type === "requestSpeechToTextSettings") { + ctx.post(buildSpeechToTextSettingsMessage()) + return true + } + + if (message.type === "speechToTextStart") { + if (!message.requestId) return true + handleSpeechToTextStart( + { requestId: message.requestId, model: message.model, language: message.language }, + ctx.post, + ) + return true + } + + if (message.type === "speechToTextStop") { + if (!message.requestId) return true + handleSpeechToTextStop(ctx.connection, { requestId: message.requestId }, ctx.dir, ctx.post) + return true + } + + if (message.type === "speechToTextCancel") { + if (!message.requestId) return true + handleSpeechToTextCancel({ requestId: message.requestId }, ctx.post) + return true + } + + return false +} diff --git a/packages/kilo-vscode/src/speech-to-text/capture.ts b/packages/kilo-vscode/src/speech-to-text/capture.ts new file mode 100644 index 00000000000..3ae3005601b --- /dev/null +++ b/packages/kilo-vscode/src/speech-to-text/capture.ts @@ -0,0 +1,278 @@ +import { existsSync } from "fs" +import { readFile, stat, unlink } from "fs/promises" +import * as os from "os" +import * as path from "path" +import type { ChildProcess } from "child_process" +import { exec, spawn } from "../util/process" + +type Input = { + requestId: string + model: string + language?: string +} + +type Recording = Input & { + file: string + proc: ChildProcess + stderr: string[] + stopped: boolean + exit?: { + code: number | null + signal: string | null + } +} + +type Audio = { + data: string + format: "wav" + model: string + language?: string +} + +let active: Recording | undefined +let starting: string | undefined + +export async function startSpeechCapture(input: Input): Promise { + if (active || starting) throw new Error("Speech recording is already in progress") + + starting = input.requestId + try { + const bin = await findFFmpeg() + const file = path.join(os.tmpdir(), `kilo-stt-${process.pid}-${Date.now()}.wav`) + const state = await startWithArgs(bin, file, input, await inputArgSets(bin)) + return !state.stopped + } finally { + if (starting === input.requestId) starting = undefined + } +} + +export async function stopSpeechCapture(requestId: string): Promise