Merge pull request #12983 from Kilo-Org/investigate-large-speech-to-text-payload-errors

fix(vscode): compress speech-to-text audio to AAC to prevent payload errors

.changeset/compressed-speech-to-text-audio.md

@@ -0,0 +1,5 @@
+---
+"kilo-code": patch
+---
+
+Compress speech-to-text audio input to AAC format across macOS, Linux, and Windows to prevent payload size errors on long recordings.

packages/kilo-vscode/src/speech-to-text/capture.ts

@@ -24,7 +24,7 @@ type Recording = Input & {
 
 type Audio = {
   data: string
-  format: "wav"
+  format: "m4a"
   model: string
   language?: string
 }
@@ -39,12 +39,10 @@ ObjC.import("AVFoundation")
 ObjC.import("Foundation")
 function run(args) {
   const settings = $.NSMutableDictionary.alloc.init
-  settings.setObjectForKey($.NSNumber.numberWithUnsignedInt(1819304813), $.AVFormatIDKey)
+  settings.setObjectForKey($.NSNumber.numberWithUnsignedInt(1633772320), $.AVFormatIDKey)
   settings.setObjectForKey($.NSNumber.numberWithDouble(16000), $.AVSampleRateKey)
   settings.setObjectForKey($.NSNumber.numberWithInt(1), $.AVNumberOfChannelsKey)
-  settings.setObjectForKey($.NSNumber.numberWithInt(16), $.AVLinearPCMBitDepthKey)
-  settings.setObjectForKey($.NSNumber.numberWithBool(false), $.AVLinearPCMIsFloatKey)
-  settings.setObjectForKey($.NSNumber.numberWithBool(false), $.AVLinearPCMIsBigEndianKey)
+  settings.setObjectForKey($.NSNumber.numberWithInt(24000), $.AVEncoderBitRateKey)
   const error = Ref()
   const url = $.NSURL.fileURLWithPath(args[0])
   const recorder = $.AVAudioRecorder.alloc.initWithURLSettingsError(url, settings, error)
@@ -68,7 +66,7 @@ export async function startSpeechCapture(input: Input): Promise<boolean> {
 
   starting = input.requestId
   try {
-    const file = path.join(os.tmpdir(), `kilo-stt-${process.pid}-${Date.now()}.wav`)
+    const file = path.join(os.tmpdir(), `kilo-stt-${process.pid}-${Date.now()}.m4a`)
     if (useMacCapture(process.platform, process.env)) {
       const result = await startMac(file, input).catch((err: unknown) => {
         console.warn("[Kilo New] Native macOS speech capture failed, falling back to FFmpeg", err)
@@ -105,7 +103,7 @@ export async function stopSpeechCapture(requestId: string): Promise<Audio> {
 
   const file = await readFile(state.file)
   await removeFile(state.file)
-  return { data: file.toString("base64"), format: "wav", model: state.model, language: state.language }
+  return { data: file.toString("base64"), format: "m4a", model: state.model, language: state.language }
 }
 
 export async function cancelSpeechCapture(requestId: string): Promise<void> {
@@ -168,6 +166,35 @@ export function macCaptureArgs(file: string): string[] {
   return ["-l", "JavaScript", "-e", macScript, file]
 }
 
+export function ffmpegCaptureArgs(input: string[], file: string): string[] {
+  return ["-y", ...input, "-c:a", "aac", "-b:a", "24k", "-ar", "16000", "-ac", "1", "-movflags", "+faststart", file]
+}
+
+export function ffmpegPipeArgs(file: string): string[] {
+  return [
+    "-y",
+    "-f",
+    "s16le",
+    "-ar",
+    "16000",
+    "-ac",
+    "1",
+    "-i",
+    "pipe:0",
+    "-c:a",
+    "aac",
+    "-b:a",
+    "24k",
+    "-ar",
+    "16000",
+    "-ac",
+    "1",
+    "-movflags",
+    "+faststart",
+    file,
+  ]
+}
+
 export function useMacCapture(platform: NodeJS.Platform, env: NodeJS.ProcessEnv): boolean {
   return platform === "darwin" && !env.KILO_FFMPEG_PATH && !env.FFMPEG_PATH
 }
@@ -178,7 +205,7 @@ async function startWithArgs(bin: string, file: string, input: Input, args: Args
 
   const proc = first.pipe
     ? pipeProcess(first.pipe, bin, file)
-    : spawn(bin, ["-y", ...first.input, "-acodec", "pcm_s16le", "-ar", "16000", "-ac", "1", "-f", "wav", file], {
+    : spawn(bin, ffmpegCaptureArgs(first.input, file), {
         stdio: ["pipe", "ignore", "pipe"],
       })
   const state = createState(input, file, proc)
@@ -212,32 +239,9 @@ function createState(input: Input, file: string, proc: ChildProcess): Recording
 
 function pipeProcess(pipe: string[], bin: string, file: string): ChildProcess {
   const source = spawn("pw-record", pipe, { stdio: ["ignore", "pipe", "pipe"] })
-  const proc = spawn(
-    bin,
-    [
-      "-y",
-      "-f",
-      "s16le",
-      "-ar",
-      "16000",
-      "-ac",
-      "1",
-      "-i",
-      "pipe:0",
-      "-acodec",
-      "pcm_s16le",
-      "-ar",
-      "16000",
-      "-ac",
-      "1",
-      "-f",
-      "wav",
-      file,
-    ],
-    {
-      stdio: ["pipe", "ignore", "pipe"],
-    },
-  )
+  const proc = spawn(bin, ffmpegPipeArgs(file), {
+    stdio: ["pipe", "ignore", "pipe"],
+  })
 
   if (source.stdout && proc.stdin) source.stdout.pipe(proc.stdin)
   source.on("error", (err) => proc.emit("error", err))

packages/kilo-vscode/tests/unit/speech-to-text-capture.test.ts

@@ -1,19 +1,80 @@
 import { describe, expect, it } from "bun:test"
-import { cleanOutput, macCaptureArgs, parseDshowAudioDevices, useMacCapture } from "../../src/speech-to-text/capture"
+import {
+  cleanOutput,
+  ffmpegCaptureArgs,
+  ffmpegPipeArgs,
+  macCaptureArgs,
+  parseDshowAudioDevices,
+  useMacCapture,
+} from "../../src/speech-to-text/capture"
 
 describe("macCaptureArgs", () => {
-  it("records 16 kHz mono PCM with the built-in AVFoundation bridge", () => {
-    const args = macCaptureArgs("/tmp/speech.wav")
+  it("records 16 kHz mono AAC at 24 kbps with the built-in AVFoundation bridge", () => {
+    const args = macCaptureArgs("/tmp/speech.m4a")
 
     expect(args.slice(0, 3)).toEqual(["-l", "JavaScript", "-e"])
-    expect(args.at(-1)).toBe("/tmp/speech.wav")
+    expect(args.at(-1)).toBe("/tmp/speech.m4a")
     expect(args[3]).toContain("AVAudioRecorder")
-    expect(args[3]).toContain("numberWithDouble(16000)")
+    expect(args[3]).toContain("numberWithUnsignedInt(1633772320), $.AVFormatIDKey")
+    expect(args[3]).toContain("numberWithDouble(16000), $.AVSampleRateKey")
     expect(args[3]).toContain("numberWithInt(1), $.AVNumberOfChannelsKey")
+    expect(args[3]).toContain("numberWithInt(24000), $.AVEncoderBitRateKey")
     expect(args[3]).toContain('console.log("ready")')
   })
 })
 
+describe("ffmpeg args", () => {
+  it("builds AAC capture arguments with faststart", () => {
+    const args = ffmpegCaptureArgs(["-f", "avfoundation", "-i", ":default"], "/tmp/speech.m4a")
+
+    expect(args).toEqual([
+      "-y",
+      "-f",
+      "avfoundation",
+      "-i",
+      ":default",
+      "-c:a",
+      "aac",
+      "-b:a",
+      "24k",
+      "-ar",
+      "16000",
+      "-ac",
+      "1",
+      "-movflags",
+      "+faststart",
+      "/tmp/speech.m4a",
+    ])
+  })
+
+  it("builds pipe capture arguments for Linux PipeWire", () => {
+    const args = ffmpegPipeArgs("/tmp/speech.m4a")
+
+    expect(args).toEqual([
+      "-y",
+      "-f",
+      "s16le",
+      "-ar",
+      "16000",
+      "-ac",
+      "1",
+      "-i",
+      "pipe:0",
+      "-c:a",
+      "aac",
+      "-b:a",
+      "24k",
+      "-ar",
+      "16000",
+      "-ac",
+      "1",
+      "-movflags",
+      "+faststart",
+      "/tmp/speech.m4a",
+    ])
+  })
+})
+
 describe("useMacCapture", () => {
   it("preserves explicit FFmpeg overrides", () => {
     expect(useMacCapture("darwin", {})).toBe(true)