diff --git a/.changeset/verbatim-speech-transcripts.md b/.changeset/verbatim-speech-transcripts.md new file mode 100644 index 00000000000..65cb57079d0 --- /dev/null +++ b/.changeset/verbatim-speech-transcripts.md @@ -0,0 +1,6 @@ +--- +"kilo-code": patch +"@kilocode/kilo-gateway": patch +--- + +Keep GPT-4o voice transcriptions closer to the words that were spoken. diff --git a/packages/kilo-gateway/src/server/routes.ts b/packages/kilo-gateway/src/server/routes.ts index f2cd17498fa..f92c00f3f31 100644 --- a/packages/kilo-gateway/src/server/routes.ts +++ b/packages/kilo-gateway/src/server/routes.ts @@ -429,6 +429,7 @@ export function createKiloRoutes(deps: KiloRoutesDeps) { format: z.string(), }), language: z.string().optional(), + prompt: z.string().optional(), temperature: z.number().optional(), }), ), diff --git a/packages/kilo-vscode/src/speech-to-text/models.ts b/packages/kilo-vscode/src/speech-to-text/models.ts index 5936641366f..be872d81c96 100644 --- a/packages/kilo-vscode/src/speech-to-text/models.ts +++ b/packages/kilo-vscode/src/speech-to-text/models.ts @@ -2,6 +2,7 @@ export interface SpeechToTextModelDef { readonly id: string readonly label: string readonly provider: string + readonly verbatim?: boolean } const models: SpeechToTextModelDef[] = [ @@ -9,11 +10,13 @@ const models: SpeechToTextModelDef[] = [ id: "openai/gpt-4o-mini-transcribe", label: "GPT-4o Mini Transcribe", provider: "OpenAI", + verbatim: true, }, { id: "openai/gpt-4o-transcribe", label: "GPT-4o Transcribe", provider: "OpenAI", + verbatim: true, }, { id: "openai/whisper-1", diff --git a/packages/kilo-vscode/src/speech-to-text/transcribe.ts b/packages/kilo-vscode/src/speech-to-text/transcribe.ts index 10cbb422f1a..1e978bf028c 100644 --- a/packages/kilo-vscode/src/speech-to-text/transcribe.ts +++ b/packages/kilo-vscode/src/speech-to-text/transcribe.ts @@ -3,6 +3,8 @@ import { getErrorMessage } from "../kilo-provider-utils" import { getSpeechToTextModel } from "./models" const PATH = "/kilo/audio/transcriptions" +const PROMPT = + "Transcribe exactly what is spoken. Do not paraphrase, summarize, infer intent, or rewrite for clarity. Preserve the speaker's original wording as closely as possible, including incomplete phrases and unusual wording when audible." type Req = { model?: string @@ -39,6 +41,8 @@ export async function transcribeSpeech( const auth = Buffer.from(`kilo:${cfg.password}`).toString("base64") const url = new URL(PATH, cfg.baseUrl) + const model = getSpeechToTextModel(input.model) + const prompt = model.verbatim ? PROMPT : undefined if (dir) url.searchParams.set("directory", dir) try { @@ -50,12 +54,13 @@ export async function transcribeSpeech( "Content-Type": "application/json", }, body: JSON.stringify({ - model: getSpeechToTextModel(input.model).id, + model: model.id, input_audio: { data: input.data, format: input.format, }, ...(input.language ? { language: input.language } : {}), + ...(prompt ? { prompt } : {}), }), }) diff --git a/packages/sdk/js/src/v2/gen/sdk.gen.ts b/packages/sdk/js/src/v2/gen/sdk.gen.ts index 199053fd00d..742d0d9367c 100644 --- a/packages/sdk/js/src/v2/gen/sdk.gen.ts +++ b/packages/sdk/js/src/v2/gen/sdk.gen.ts @@ -5763,6 +5763,7 @@ export class Audio extends HeyApiClient { format: string } language?: string + prompt?: string temperature?: number }, options?: Options, @@ -5777,6 +5778,7 @@ export class Audio extends HeyApiClient { { in: "body", key: "model" }, { in: "body", key: "input_audio" }, { in: "body", key: "language" }, + { in: "body", key: "prompt" }, { in: "body", key: "temperature" }, ], }, diff --git a/packages/sdk/js/src/v2/gen/types.gen.ts b/packages/sdk/js/src/v2/gen/types.gen.ts index 7ec2e38bb27..e5323886eed 100644 --- a/packages/sdk/js/src/v2/gen/types.gen.ts +++ b/packages/sdk/js/src/v2/gen/types.gen.ts @@ -7255,6 +7255,7 @@ export type KiloAudioTranscriptionsData = { format: string } language?: string + prompt?: string temperature?: number } path?: never diff --git a/packages/sdk/openapi.json b/packages/sdk/openapi.json index 33c1727634a..2d33a0d2a24 100644 --- a/packages/sdk/openapi.json +++ b/packages/sdk/openapi.json @@ -10481,6 +10481,100 @@ ] } }, + "/kilo/audio/transcriptions": { + "post": { + "operationId": "kilo.audio.transcriptions", + "parameters": [ + { + "in": "query", + "name": "directory", + "schema": { + "type": "string" + } + }, + { + "in": "query", + "name": "workspace", + "schema": { + "type": "string" + } + } + ], + "summary": "Speech to text transcription", + "description": "Proxy an audio transcription request to the Kilo Gateway", + "responses": { + "200": { + "description": "Transcription response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "text": { + "type": "string" + }, + "usage": {} + }, + "required": ["text"] + } + } + } + }, + "400": { + "description": "Bad request", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BadRequestError" + } + } + } + } + }, + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "model": { + "type": "string" + }, + "input_audio": { + "type": "object", + "properties": { + "data": { + "type": "string" + }, + "format": { + "type": "string" + } + }, + "required": ["data", "format"] + }, + "language": { + "type": "string" + }, + "prompt": { + "type": "string" + }, + "temperature": { + "type": "number" + } + }, + "required": ["model", "input_audio"] + } + } + } + }, + "x-codeSamples": [ + { + "lang": "js", + "source": "import { createKiloClient } from \"@kilocode/sdk\n\nconst client = createKiloClient()\nawait client.kilo.audio.transcriptions({\n ...\n})" + } + ] + } + }, "/kilo/notifications": { "get": { "operationId": "kilo.notifications",