diff --git a/.changeset/faster-macos-speech.md b/.changeset/faster-macos-speech.md new file mode 100644 index 00000000000..2d9bd9cc258 --- /dev/null +++ b/.changeset/faster-macos-speech.md @@ -0,0 +1,5 @@ +--- +"kilo-code": patch +--- + +Start voice input faster on macOS with native AVFoundation capture. diff --git a/packages/kilo-vscode/src/speech-to-text/capture.ts b/packages/kilo-vscode/src/speech-to-text/capture.ts index 6250c0822c1..cbf5c54848a 100644 --- a/packages/kilo-vscode/src/speech-to-text/capture.ts +++ b/packages/kilo-vscode/src/speech-to-text/capture.ts @@ -34,11 +34,32 @@ type Args = { input: string[] } +const macScript = ` +ObjC.import("AVFoundation") +ObjC.import("Foundation") +function run(args) { + const settings = $.NSMutableDictionary.alloc.init + settings.setObjectForKey($.NSNumber.numberWithUnsignedInt(1819304813), $.AVFormatIDKey) + settings.setObjectForKey($.NSNumber.numberWithDouble(16000), $.AVSampleRateKey) + settings.setObjectForKey($.NSNumber.numberWithInt(1), $.AVNumberOfChannelsKey) + settings.setObjectForKey($.NSNumber.numberWithInt(16), $.AVLinearPCMBitDepthKey) + settings.setObjectForKey($.NSNumber.numberWithBool(false), $.AVLinearPCMIsFloatKey) + settings.setObjectForKey($.NSNumber.numberWithBool(false), $.AVLinearPCMIsBigEndianKey) + const error = Ref() + const url = $.NSURL.fileURLWithPath(args[0]) + const recorder = $.AVAudioRecorder.alloc.initWithURLSettingsError(url, settings, error) + if (!recorder || !recorder.prepareToRecord || !recorder.record) throw new Error("Could not start recording") + console.log("ready") + $.NSFileHandle.fileHandleWithStandardInput.readDataToEndOfFile + recorder.stop +}` + let active: Recording | undefined let starting: string | undefined let ffmpeg: Promise | undefined export async function prewarmSpeechCapture(): Promise { + if (useMacCapture(process.platform, process.env)) return await resolveFFmpeg() } @@ -47,8 +68,15 @@ export async function startSpeechCapture(input: Input): Promise { starting = input.requestId try { - const bin = await resolveFFmpeg() const file = path.join(os.tmpdir(), `kilo-stt-${process.pid}-${Date.now()}.wav`) + if (useMacCapture(process.platform, process.env)) { + const result = await startMac(file, input).catch((err: unknown) => { + console.warn("[Kilo New] Native macOS speech capture failed, falling back to FFmpeg", err) + return undefined + }) + if (result) return !result.stopped + } + const bin = await resolveFFmpeg() const state = await startWithArgs(bin, file, input, await inputArgSets(bin)) return !state.stopped } finally { @@ -112,7 +140,7 @@ async function waitForStart(state: Recording): Promise { onError(new Error(summary(state, "Could not start microphone recording"))) } const onData = (data: Buffer) => { - if (/Output #0|Press \[q\]|size=\s*\d+/i.test(data.toString())) done() + if (/^ready$|Output #0|Press \[q\]|size=\s*\d+/im.test(data.toString())) done() } const timer = setTimeout(() => { onError(new Error(summary(state, "Timed out starting microphone recording"))) @@ -129,6 +157,21 @@ async function waitForStart(state: Recording): Promise { }) } +async function startMac(file: string, input: Input): Promise { + const proc = spawn("/usr/bin/osascript", macCaptureArgs(file), { stdio: ["pipe", "ignore", "pipe"] }) + const state = createState(input, file, proc) + await waitForStart(state) + return state +} + +export function macCaptureArgs(file: string): string[] { + return ["-l", "JavaScript", "-e", macScript, file] +} + +export function useMacCapture(platform: NodeJS.Platform, env: NodeJS.ProcessEnv): boolean { + return platform === "darwin" && !env.KILO_FFMPEG_PATH && !env.FFMPEG_PATH +} + async function startWithArgs(bin: string, file: string, input: Input, args: Args[]): Promise { const [first, ...rest] = args if (!first) throw new Error(`Unsupported platform for speech input: ${process.platform}`) @@ -138,6 +181,18 @@ async function startWithArgs(bin: string, file: string, input: Input, args: Args : spawn(bin, ["-y", ...first.input, "-acodec", "pcm_s16le", "-ar", "16000", "-ac", "1", "-f", "wav", file], { stdio: ["pipe", "ignore", "pipe"], }) + const state = createState(input, file, proc) + try { + await waitForStart(state) + return state + } catch (err) { + if (state.stopped) return state + if (rest.length === 0) throw err + return startWithArgs(bin, file, input, rest) + } +} + +function createState(input: Input, file: string, proc: ChildProcess): Recording { const state: Recording = { ...input, file, proc, stderr: [], stopped: false } active = state @@ -152,15 +207,7 @@ async function startWithArgs(bin: string, file: string, input: Input, args: Args state.stderr.push(err.message) if (active === state && !state.stopped) active = undefined }) - - try { - await waitForStart(state) - return state - } catch (err) { - if (state.stopped) return state - if (rest.length === 0) throw err - return startWithArgs(bin, file, input, rest) - } + return state } function pipeProcess(pipe: string[], bin: string, file: string): ChildProcess { diff --git a/packages/kilo-vscode/tests/unit/speech-to-text-capture.test.ts b/packages/kilo-vscode/tests/unit/speech-to-text-capture.test.ts index 8b68ca79c73..e68d29d90b2 100644 --- a/packages/kilo-vscode/tests/unit/speech-to-text-capture.test.ts +++ b/packages/kilo-vscode/tests/unit/speech-to-text-capture.test.ts @@ -1,5 +1,28 @@ import { describe, expect, it } from "bun:test" -import { cleanOutput, parseDshowAudioDevices } from "../../src/speech-to-text/capture" +import { cleanOutput, macCaptureArgs, parseDshowAudioDevices, useMacCapture } from "../../src/speech-to-text/capture" + +describe("macCaptureArgs", () => { + it("records 16 kHz mono PCM with the built-in AVFoundation bridge", () => { + const args = macCaptureArgs("/tmp/speech.wav") + + expect(args.slice(0, 3)).toEqual(["-l", "JavaScript", "-e"]) + expect(args.at(-1)).toBe("/tmp/speech.wav") + expect(args[3]).toContain("AVAudioRecorder") + expect(args[3]).toContain("numberWithDouble(16000)") + expect(args[3]).toContain("numberWithInt(1), $.AVNumberOfChannelsKey") + expect(args[3]).toContain('console.log("ready")') + }) +}) + +describe("useMacCapture", () => { + it("preserves explicit FFmpeg overrides", () => { + expect(useMacCapture("darwin", {})).toBe(true) + expect(useMacCapture("darwin", { KILO_FFMPEG_PATH: "/custom/ffmpeg" })).toBe(false) + expect(useMacCapture("darwin", { FFMPEG_PATH: "/custom/ffmpeg" })).toBe(false) + expect(useMacCapture("linux", {})).toBe(false) + expect(useMacCapture("win32", {})).toBe(false) + }) +}) describe("parseDshowAudioDevices", () => { it("extracts Windows dshow audio device names", () => {