diff --git a/.changeset/copy-tooltip-top.md b/.changeset/copy-tooltip-top.md new file mode 100644 index 00000000000..ded0f1a640d --- /dev/null +++ b/.changeset/copy-tooltip-top.md @@ -0,0 +1,5 @@ +--- +"@kilocode/kilo-ui": patch +--- + +Show the assistant message copy tooltip above the button so it no longer overlaps the thumbs up/down buttons. diff --git a/.changeset/fresh-snapshot-cli.md b/.changeset/fresh-snapshot-cli.md new file mode 100644 index 00000000000..4bdd7e6cf27 --- /dev/null +++ b/.changeset/fresh-snapshot-cli.md @@ -0,0 +1,5 @@ +--- +"kilo-code": patch +--- + +Ensure snapshot installs package a fresh CLI backend so new extension features are available in local VSIX builds. diff --git a/.changeset/speech-input-gateway.md b/.changeset/speech-input-gateway.md new file mode 100644 index 00000000000..cf48a6b19bb --- /dev/null +++ b/.changeset/speech-input-gateway.md @@ -0,0 +1,6 @@ +--- +"kilo-code": minor +"@kilocode/kilo-gateway": minor +--- + +Support experimental speech-to-text voice input in VS Code prompt fields through Kilo Gateway. diff --git a/packages/kilo-gateway/src/server/routes.ts b/packages/kilo-gateway/src/server/routes.ts index 269427b7f44..f2cd17498fa 100644 --- a/packages/kilo-gateway/src/server/routes.ts +++ b/packages/kilo-gateway/src/server/routes.ts @@ -132,6 +132,21 @@ export function createKiloRoutes(deps: KiloRoutesDeps) { cost: z.number().optional(), }) + const TranscriptionResponse = z.object({ + text: z.string(), + usage: z.unknown().optional(), + }) + + const getProxyAuth = async () => { + const auth = await Auth.get("kilo") + const token = auth?.type === "api" ? auth.key : auth?.type === "oauth" ? auth.access : undefined + return { + auth, + token, + organizationId: auth?.type === "oauth" ? auth.accountId : undefined, + } + } + return new Hono() .get( "/profile", @@ -324,19 +339,16 @@ export function createKiloRoutes(deps: KiloRoutesDeps) { }), ), async (c: any) => { - const auth = await Auth.get("kilo") + const proxy = await getProxyAuth() - if (!auth) { + if (!proxy.auth) { return c.json({ error: "Not authenticated with Kilo Gateway" }, 401) } - const token = auth.type === "api" ? auth.key : auth.type === "oauth" ? auth.access : undefined - if (!token) { + if (!proxy.token) { return c.json({ error: "No valid token found" }, 401) } - const organizationId = auth.type === "oauth" ? auth.accountId : undefined - const { prefix, suffix, model, maxTokens, temperature } = c.req.valid("json") const fimModel = model ?? "mistralai/codestral-2501" const fimMaxTokens = maxTokens ?? 256 @@ -347,8 +359,8 @@ export function createKiloRoutes(deps: KiloRoutesDeps) { const headers = { "Content-Type": "application/json", - Authorization: `Bearer ${token}`, - ...buildKiloHeaders(undefined, { kilocodeOrganizationId: organizationId }), + Authorization: `Bearer ${proxy.token}`, + ...buildKiloHeaders(undefined, { kilocodeOrganizationId: proxy.organizationId }), [HEADER_FEATURE]: "autocomplete", } @@ -390,6 +402,66 @@ export function createKiloRoutes(deps: KiloRoutesDeps) { }) }, ) + .post( + "/audio/transcriptions", + describeRoute({ + summary: "Speech to text transcription", + description: "Proxy an audio transcription request to the Kilo Gateway", + operationId: "kilo.audio.transcriptions", + responses: { + 200: { + description: "Transcription response", + content: { + "application/json": { + schema: resolver(TranscriptionResponse), + }, + }, + }, + ...errors(400, 401), + }, + }), + validator( + "json", + z.object({ + model: z.string(), + input_audio: z.object({ + data: z.string(), + format: z.string(), + }), + language: z.string().optional(), + temperature: z.number().optional(), + }), + ), + async (c: any) => { + const proxy = await getProxyAuth() + if (!proxy.auth) return c.json({ error: "Not authenticated with Kilo Gateway" }, 401) + + if (!proxy.token) return c.json({ error: "No valid token found" }, 401) + + const body = c.req.valid("json") + const headers = { + "Content-Type": "application/json", + Authorization: `Bearer ${proxy.token}`, + ...buildKiloHeaders(undefined, { kilocodeOrganizationId: proxy.organizationId }), + [HEADER_FEATURE]: "vscode-extension", + } + + const response = await fetch(`${KILO_API_BASE}/api/gateway/v1/audio/transcriptions`, { + method: "POST", + headers, + signal: c.req.raw.signal, + body: JSON.stringify(body), + }) + + const text = await response.text() + return new Response(text, { + status: response.status, + headers: { + "Content-Type": response.headers.get("Content-Type") ?? "application/json", + }, + }) + }, + ) .get( "/notifications", describeRoute({ diff --git a/packages/kilo-ui/src/components/message-part.tsx b/packages/kilo-ui/src/components/message-part.tsx index 759c3d96dff..aa9bbbe086a 100644 --- a/packages/kilo-ui/src/components/message-part.tsx +++ b/packages/kilo-ui/src/components/message-part.tsx @@ -1345,7 +1345,7 @@ PART_MAPPING["text"] = function TextPartDisplay(props) {
= { + "darwin-x64": "@ffmpeg-installer/darwin-x64@4.1.0", + "darwin-arm64": "@ffmpeg-installer/darwin-arm64@4.1.5", + "linux-x64": "@ffmpeg-installer/linux-x64@4.1.0", + "linux-arm64": "@ffmpeg-installer/linux-arm64@4.1.4", + "alpine-x64": "@ffmpeg-installer/linux-x64@4.1.0", + "alpine-arm64": "@ffmpeg-installer/linux-arm64@4.1.4", + "win32-x64": "@ffmpeg-installer/win32-x64@4.1.0", +} + +export async function ensureFfmpegForTarget(target: string, bin: string): Promise { + if (target === "win32-arm64") { + console.warn("No Windows ARM64 FFmpeg helper package is available; speech input will use system FFmpeg.") + return + } + + const spec = packages[target] + if (!spec) throw new Error(`No FFmpeg helper package configured for target ${target}`) + + const exe = target.startsWith("win32") ? "ffmpeg.exe" : "ffmpeg" + const dest = join(bin, exe) + if (existsSync(dest)) return + + const tmp = join(bin, ".ffmpeg-tmp") + rmSync(tmp, { recursive: true, force: true }) + mkdirSync(tmp, { recursive: true }) + + try { + const packed = await $`npm pack ${spec} --pack-destination ${tmp}`.quiet() + const name = packed.text().trim().split(/\s+/).pop() + if (!name) throw new Error(`npm pack did not return a tarball for ${spec}`) + + await $`tar -xzf ${join(tmp, name)} -C ${tmp}`.quiet() + copyFileSync(join(tmp, "package", exe), dest) + if (!target.startsWith("win32")) chmodSync(dest, 0o755) + } finally { + rmSync(tmp, { recursive: true, force: true }) + } +} + +export function currentFfmpegTarget(): string { + const os = process.platform === "win32" ? "win32" : process.platform + const arch = process.arch === "x64" ? "x64" : process.arch === "arm64" ? "arm64" : process.arch + return `${os}-${arch}` +} diff --git a/packages/kilo-vscode/script/local-bin.ts b/packages/kilo-vscode/script/local-bin.ts index aeb57e81643..a8d2f669bde 100644 --- a/packages/kilo-vscode/script/local-bin.ts +++ b/packages/kilo-vscode/script/local-bin.ts @@ -2,6 +2,7 @@ import { $ } from "bun" import { join, relative, dirname, basename } from "node:path" import { chmodSync, statSync, rmSync, readdirSync, existsSync } from "node:fs" +import { currentFfmpegTarget, ensureFfmpegForTarget } from "./ffmpeg-helper" const forceRebuild = process.argv.includes("--force") @@ -158,6 +159,7 @@ async function main() { log( `CLI binary already present at ${relative(kiloVscodeDir, targetBinPath)} (${Math.round(st.size / 1024 / 1024)}MB). Use --force to rebuild.`, ) + await ensureFfmpegForTarget(currentFfmpegTarget(), targetBinDir) return } @@ -181,6 +183,7 @@ async function main() { await $`mkdir -p ${targetBinDir}` await $`cp ${sourceBinPath} ${targetBinPath}` chmodSync(targetBinPath, 0o755) + await ensureFfmpegForTarget(currentFfmpegTarget(), targetBinDir) // Record the CLI source version so future runs detect when a rebuild is needed const hash = await cliSourceHash() diff --git a/packages/kilo-vscode/src/KiloProvider.ts b/packages/kilo-vscode/src/KiloProvider.ts index 0aa9c66aba8..a3ed7bd2215 100644 --- a/packages/kilo-vscode/src/KiloProvider.ts +++ b/packages/kilo-vscode/src/KiloProvider.ts @@ -68,10 +68,12 @@ import { handleNetworkEvent, clearNetworkWaits } from "./kilo-provider/network" import { abortSession } from "./kilo-provider/abort" import { buildAutocompleteSettingsMessage, - routeAutocompleteMessage, validAutocompleteSetting, watchAutocompleteConfig, } from "./services/autocomplete/settings" +import { validSpeechToTextSetting } from "./speech-to-text/settings" +import { sendSpeechToTextSettings, watchSpeechToTextConfig } from "./speech-to-text/config" +import { routeEarlyMessage } from "./kilo-provider/early-message" import * as ModelState from "./kilo-provider/model-state" import { handleForkSession } from "./kilo-provider/fork-session" import { openConfig } from "./kilo-provider/open-config" @@ -112,7 +114,7 @@ import { handleQuestionReject, fetchAndSendPendingQuestions, } from "./kilo-provider/handlers/question" -import { fetchAndSendPendingSuggestions, routeSuggestionWebviewMessage } from "./kilo-provider/handlers/suggestion" +import { fetchAndSendPendingSuggestions } from "./kilo-provider/handlers/suggestion" import { nativeTitle } from "./kilo-provider/native-tab-title" import { @@ -229,6 +231,7 @@ export class KiloProvider implements vscode.WebviewViewProvider, TelemetryProper private webviewMessageDisposable: vscode.Disposable | null = null private autocompleteConfigDisposable: vscode.Disposable | null = null private telemetryStateDisposable: vscode.Disposable | null = null + private speechToTextDisposable: vscode.Disposable | null = null private viewStateDisposable: vscode.Disposable | null = null private visibilityDisposable: vscode.Disposable | null = null private autoApproveBridge: ReturnType | null = null @@ -592,6 +595,8 @@ export class KiloProvider implements vscode.WebviewViewProvider, TelemetryProper this.webviewMessageDisposable?.dispose() this.autocompleteConfigDisposable?.dispose() this.autocompleteConfigDisposable = watchAutocompleteConfig((msg) => this.postMessage(msg)) + this.speechToTextDisposable?.dispose() + this.speechToTextDisposable = watchSpeechToTextConfig(this) this.telemetryStateDisposable?.dispose() this.telemetryStateDisposable = watchTelemetryState((msg) => this.postMessage(msg)) this.webviewMessageDisposable = webview.onDidReceiveMessage(async (message) => { @@ -604,9 +609,17 @@ export class KiloProvider implements vscode.WebviewViewProvider, TelemetryProper if (intercepted === null) return message = intercepted - await routeSuggestionWebviewMessage(this.questionCtx, message) - if (await ModelState.handleMessage(message.type, message, this.client, (msg) => this.postMessage(msg))) return - if (await routeAutocompleteMessage(message, (msg) => this.postMessage(msg))) return + if ( + await routeEarlyMessage(message, { + question: this.questionCtx, + client: this.client, + connection: this.connectionService, + dir: this.getWorkspaceDirectory(this.currentSession?.id), + post: (msg) => this.postMessage(msg), + }) + ) { + return + } if (this.handleEditorOpenMessage(message)) return if ( await handleSidebarWorktreeMessage(message, { @@ -1297,6 +1310,7 @@ export class KiloProvider implements vscode.WebviewViewProvider, TelemetryProper this.postMessage({ type: "gitStatus", repo: this.cachedGitRepo }) this.sendNotificationSettings() this.sendTimelineSetting() + this.sendSpeechToTextSettings() this.postMessage({ type: "extensionDataReady" }) if (this.cachedGitRepo) this.startStatsPolling() @@ -2337,6 +2351,10 @@ export class KiloProvider implements vscode.WebviewViewProvider, TelemetryProper }) } + private sendSpeechToTextSettings(): void { + sendSpeechToTextSettings(this) + } + /** Returns the number of sessions currently in "busy" state. */ private getBusySessionCount(): number { return getBusySessionCount(this.sessionStatusMap) @@ -2953,6 +2971,7 @@ export class KiloProvider implements vscode.WebviewViewProvider, TelemetryProper private async handleUpdateSetting(key: string, value: unknown): Promise { const { section, leaf } = buildSettingPath(key) if (section === "autocomplete" && !validAutocompleteSetting(leaf, value)) return + if (section === "speechToText" && !validSpeechToTextSetting(leaf, value)) return const config = vscode.workspace.getConfiguration(`kilo-code.new${section ? `.${section}` : ""}`) await config.update(leaf, value, vscode.ConfigurationTarget.Global) } @@ -2993,6 +3012,7 @@ export class KiloProvider implements vscode.WebviewViewProvider, TelemetryProper // Re-send all settings to the webview so the UI reflects the reset this.postMessage(buildAutocompleteSettingsMessage()) + this.sendSpeechToTextSettings() this.sendBrowserSettings() this.sendNotificationSettings() this.sendTimelineSetting() @@ -3494,6 +3514,7 @@ export class KiloProvider implements vscode.WebviewViewProvider, TelemetryProper this.visibilityDisposable?.dispose() this.webviewMessageDisposable?.dispose() this.autocompleteConfigDisposable?.dispose() + this.speechToTextDisposable?.dispose() this.telemetryStateDisposable?.dispose() this.autoApproveBridge?.dispose() this.streams.dispose() diff --git a/packages/kilo-vscode/src/kilo-provider/early-message.ts b/packages/kilo-vscode/src/kilo-provider/early-message.ts new file mode 100644 index 00000000000..74b73a14955 --- /dev/null +++ b/packages/kilo-vscode/src/kilo-provider/early-message.ts @@ -0,0 +1,20 @@ +import { routeSuggestionWebviewMessage } from "./handlers/suggestion" +import * as ModelState from "./model-state" +import { routeInputToolMessage } from "../services/input-tools" +import type { KiloConnectionService } from "../services/cli-backend/connection-service" +import type { SuggestionContext } from "./handlers/suggestion" +import type { KiloClient } from "@kilocode/sdk/v2/client" + +type Ctx = { + question: SuggestionContext + client: KiloClient | null + connection: KiloConnectionService + dir: string + post: (msg: unknown) => void +} + +export async function routeEarlyMessage(message: { type: string }, ctx: Ctx): Promise { + await routeSuggestionWebviewMessage(ctx.question, message) + if (await ModelState.handleMessage(message.type, message, ctx.client, ctx.post)) return true + return await routeInputToolMessage(message, { connection: ctx.connection, dir: ctx.dir, post: ctx.post }) +} diff --git a/packages/kilo-vscode/src/services/input-tools.ts b/packages/kilo-vscode/src/services/input-tools.ts new file mode 100644 index 00000000000..86180d9b50c --- /dev/null +++ b/packages/kilo-vscode/src/services/input-tools.ts @@ -0,0 +1,49 @@ +import type { KiloConnectionService } from "./cli-backend/connection-service" +import { routeAutocompleteMessage } from "./autocomplete/settings" +import { buildSpeechToTextSettingsMessage } from "../speech-to-text/settings" +import { handleSpeechToTextCancel, handleSpeechToTextStart, handleSpeechToTextStop } from "../speech-to-text/handler" + +type Msg = { + type: string + requestId?: string + model?: string + language?: string +} + +type Ctx = { + connection: KiloConnectionService + dir: string + post: (msg: unknown) => void +} + +export async function routeInputToolMessage(message: Msg, ctx: Ctx): Promise { + if (await routeAutocompleteMessage(message, ctx.post)) return true + + if (message.type === "requestSpeechToTextSettings") { + ctx.post(buildSpeechToTextSettingsMessage()) + return true + } + + if (message.type === "speechToTextStart") { + if (!message.requestId) return true + handleSpeechToTextStart( + { requestId: message.requestId, model: message.model, language: message.language }, + ctx.post, + ) + return true + } + + if (message.type === "speechToTextStop") { + if (!message.requestId) return true + handleSpeechToTextStop(ctx.connection, { requestId: message.requestId }, ctx.dir, ctx.post) + return true + } + + if (message.type === "speechToTextCancel") { + if (!message.requestId) return true + handleSpeechToTextCancel({ requestId: message.requestId }, ctx.post) + return true + } + + return false +} diff --git a/packages/kilo-vscode/src/speech-to-text/capture.ts b/packages/kilo-vscode/src/speech-to-text/capture.ts new file mode 100644 index 00000000000..3ae3005601b --- /dev/null +++ b/packages/kilo-vscode/src/speech-to-text/capture.ts @@ -0,0 +1,278 @@ +import { existsSync } from "fs" +import { readFile, stat, unlink } from "fs/promises" +import * as os from "os" +import * as path from "path" +import type { ChildProcess } from "child_process" +import { exec, spawn } from "../util/process" + +type Input = { + requestId: string + model: string + language?: string +} + +type Recording = Input & { + file: string + proc: ChildProcess + stderr: string[] + stopped: boolean + exit?: { + code: number | null + signal: string | null + } +} + +type Audio = { + data: string + format: "wav" + model: string + language?: string +} + +let active: Recording | undefined +let starting: string | undefined + +export async function startSpeechCapture(input: Input): Promise { + if (active || starting) throw new Error("Speech recording is already in progress") + + starting = input.requestId + try { + const bin = await findFFmpeg() + const file = path.join(os.tmpdir(), `kilo-stt-${process.pid}-${Date.now()}.wav`) + const state = await startWithArgs(bin, file, input, await inputArgSets(bin)) + return !state.stopped + } finally { + if (starting === input.requestId) starting = undefined + } +} + +export async function stopSpeechCapture(requestId: string): Promise
-
+
+ + + +
diff --git a/packages/kilo-vscode/webview-ui/src/components/chat/PromptInput.tsx b/packages/kilo-vscode/webview-ui/src/components/chat/PromptInput.tsx index 3ff6d7650bb..27d984c56a0 100644 --- a/packages/kilo-vscode/webview-ui/src/components/chat/PromptInput.tsx +++ b/packages/kilo-vscode/webview-ui/src/components/chat/PromptInput.tsx @@ -19,8 +19,11 @@ import { useLanguage } from "../../context/language" import { useVSCode } from "../../context/vscode" import { useWorktreeMode } from "../../context/worktree-mode" import { useConfig } from "../../context/config" +import { useProvider } from "../../context/provider" import { ModelSelector } from "../shared/ModelSelector" import { ModeSwitcher } from "../shared/ModeSwitcher" +import { SpeechToTextButton } from "../speech-to-text/SpeechToTextButton" +import { canUseSpeechToText, selectedSpeechToTextModel } from "../speech-to-text/availability" import { ThinkingSelector } from "../shared/ThinkingSelector" import { useFileMention } from "../../hooks/useFileMention" import { useTerminalContext } from "../../hooks/useTerminalContext" @@ -29,11 +32,12 @@ import { hasTerminalMention } from "../../hooks/terminal-context-utils" import { hasGitChangesMention } from "../../hooks/git-changes-context-utils" import { useSlashCommand } from "../../hooks/useSlashCommand" import { useGhostText } from "../../hooks/useGhostText" +import { useSpeechToText } from "../speech-to-text/useSpeechToText" import { useImageAttachments, type ImageAttachment } from "../../hooks/useImageAttachments" import { convertToMentionPath } from "../../utils/path-mentions" import { usePromptHistory } from "../../hooks/usePromptHistory" import { WandSparkles } from "@kilocode/kilo-ui/lucide" -import { fileName, dirName, buildHighlightSegments, atEnd, isPromptBusy } from "./prompt-input-utils" +import { fileName, dirName, buildHighlightSegments, atEnd, insertSpacedText, isPromptBusy } from "./prompt-input-utils" import type { ReviewComment, TextPart } from "../../types/messages" import { formatReviewCommentsMarkdown } from "../../utils/review-comment-markdown" import { pendingDraftKey, scopeDraftKey, sessionDraftKey } from "../../utils/prompt-drafts" @@ -66,7 +70,8 @@ export const PromptInput: Component = (props) => { const session = useSession() const server = useServer() const indexing = useIndexing() - const { features } = useConfig() + const { config, features, settings } = useConfig() + const provider = useProvider() const language = useLanguage() const vscode = useVSCode() const worktree = useWorktreeMode() @@ -135,6 +140,7 @@ export const PromptInput: Component = (props) => { let preEnhanceText: string | null = null const ghost = useGhostText(vscode, text, () => server.isConnected()) + const speech = useSpeechToText(vscode, server, language) const replaceReviewComments = (next: ReviewComment[]) => { setReviewComments(next) @@ -325,8 +331,11 @@ export const PromptInput: Component = (props) => { const isBusy = () => isPromptBusy(session.status(), !!props.suggesting?.(), !!props.questioning?.()) const isDisabled = () => !server.isConnected() + const canUseSpeech = () => canUseSpeechToText(settings(), config(), provider.connected(), server.profileData()) + const speechModel = () => selectedSpeechToTextModel(settings()) const hasInput = () => text().trim().length > 0 || imageAttach.images().length > 0 || reviewComments().length > 0 - const canSend = () => hasInput() && !isDisabled() && !terminal.pending() && !git.pending() && !props.blocked?.() + const canSend = () => + hasInput() && !isDisabled() && !speech.active() && !terminal.pending() && !git.pending() && !props.blocked?.() const showStop = () => isBusy() && !hasInput() const isAtEnd = () => textareaRef ? atEnd(textareaRef.selectionStart, textareaRef.selectionEnd, textareaRef.value.length) : false @@ -645,6 +654,27 @@ export const PromptInput: Component = (props) => { vscode.postMessage({ type: "enhancePrompt", text: draft, requestId: `enhance-${draftKey()}-${enhanceCounter}` }) } + const insertSpeechText = (value: string) => { + const ref = textareaRef + const current = text() + const start = ref?.selectionStart ?? current.length + const end = ref?.selectionEnd ?? start + const result = insertSpacedText(current, value, start, end) + + setText(result.text) + if (!ref) return + ref.value = result.text + ref.setSelectionRange(result.pos, result.pos) + ref.focus() + adjustHeight() + syncHighlightScroll() + ghost.scheduleRequest(result.text, ref) + } + + const startSpeech = () => { + speech.start({ model: speechModel(), insert: insertSpeechText }) + } + const handleSend = async () => { const draft = text().trim() @@ -676,7 +706,14 @@ export const PromptInput: Component = (props) => { const pending = reviewComments() const review = pending.length > 0 ? formatReviewCommentsMarkdown(pending) : "" const message = draft && review ? `${review}\n\n${draft}` : draft || review - if ((!message && imgs.length === 0) || isDisabled() || terminal.pending() || git.pending() || props.blocked?.()) + if ( + (!message && imgs.length === 0) || + isDisabled() || + speech.active() || + terminal.pending() || + git.pending() || + props.blocked?.() + ) return const mentionFiles = mention.parseFileAttachments(draft) @@ -1042,6 +1079,9 @@ export const PromptInput: Component = (props) => { + + + { - const { config, updateConfig } = useConfig() + const { config, settings, updateConfig, updateSetting } = useConfig() const language = useLanguage() + const provider = useProvider() + const server = useServer() const vscode = useVSCode() const [active, setActive] = createSignal(false) @@ -39,6 +46,8 @@ const ExperimentalTab: Component = () => { }) const experimental = createMemo(() => config().experimental ?? {}) + const kiloReady = createMemo(() => hasSpeechToTextAccess(config(), provider.connected(), server.profileData())) + const speechModel = createMemo(() => selectedSpeechToTextModel(settings())) const updateExperimental = (key: string, value: unknown) => { updateConfig({ @@ -193,6 +202,42 @@ const ExperimentalTab: Component = () => { + + updateSetting("speechToText.enabled", checked)} + disabled={!kiloReady()} + hideLabel + > + {language.t("settings.experimental.speechToText.title")} + + + + +