mirror of
https://github.com/coder/coder.git
synced 2026-09-24 15:04:27 +08:00
feat(site): add voice-to-text input to agent chat (#23022)
## Summary Adds a microphone button to the agent chat input for browser-native voice-to-text transcription using the Web Speech API. ## Changes ### New: `site/src/hooks/useSpeechRecognition.ts` - Custom React hook wrapping the Web Speech API (`SpeechRecognition` / `webkitSpeechRecognition`) - Feature-detects browser support via `isSupported` - Provides `start()`, `stop()`, and `cancel()` controls - Accumulates real-time transcript from interim and final recognition results - Inline TypeScript declarations for the Web Speech API types ### Modified: `site/src/pages/AgentsPage/AgentChatInput.tsx` - **Mic button**: Appears to the right of the image attach button when the browser supports the Speech Recognition API. Shows a microphone icon when idle, X icon when recording. - **Send button**: Transforms into a checkmark during recording to accept the transcription. Always enabled during recording. - **Editor sync**: Live-updates the Lexical editor with the transcription as the user speaks. Preserves any pre-existing text. - **Cancel**: Restores the editor to its pre-recording content. ## How it works 1. User clicks the mic button → recording starts, real-time transcript appears in the editor 2. User clicks the checkmark (send button) → recording stops, transcribed text stays 3. User clicks X (mic button) → recording stops, editor reverts to original content
This commit is contained in:
Kyle Carberry
@@ -0,0 +1,236 @@
|
||||
import { act, renderHook } from "@testing-library/react";
|
||||
import {
|
||||
isSpeechRecognitionSupported,
|
||||
useSpeechRecognition,
|
||||
} from "./useSpeechRecognition";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Minimal mock for the Web Speech API SpeechRecognition class.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
type ResultHandler = (event: {
|
||||
resultIndex: number;
|
||||
results: {
|
||||
length: number;
|
||||
[i: number]: {
|
||||
isFinal: boolean;
|
||||
0: { transcript: string; confidence: number };
|
||||
};
|
||||
};
|
||||
}) => void;
|
||||
|
||||
class MockSpeechRecognition {
|
||||
lang = "";
|
||||
continuous = false;
|
||||
interimResults = false;
|
||||
onresult: ResultHandler | null = null;
|
||||
onerror: ((event: { error: string; message: string }) => void) | null = null;
|
||||
onend: (() => void) | null = null;
|
||||
|
||||
start = vi.fn();
|
||||
stop = vi.fn(() => {
|
||||
// Browser fires onend after stop.
|
||||
this.onend?.();
|
||||
});
|
||||
abort = vi.fn(() => {
|
||||
this.onend?.();
|
||||
});
|
||||
}
|
||||
|
||||
let lastInstance: MockSpeechRecognition | null = null;
|
||||
|
||||
function installMock() {
|
||||
lastInstance = null;
|
||||
// Use a real class so `new Ctor()` works — vi.fn() arrow
|
||||
// functions are not constructable.
|
||||
class Ctor extends MockSpeechRecognition {
|
||||
constructor() {
|
||||
super();
|
||||
lastInstance = this;
|
||||
}
|
||||
}
|
||||
Object.assign(window, { SpeechRecognition: Ctor });
|
||||
return Ctor;
|
||||
}
|
||||
|
||||
function removeMock() {
|
||||
Object.assign(window, {
|
||||
SpeechRecognition: undefined,
|
||||
webkitSpeechRecognition: undefined,
|
||||
});
|
||||
lastInstance = null;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
afterEach(() => {
|
||||
removeMock();
|
||||
});
|
||||
|
||||
describe("isSpeechRecognitionSupported", () => {
|
||||
it("returns false when API is not available", () => {
|
||||
removeMock();
|
||||
expect(isSpeechRecognitionSupported()).toBe(false);
|
||||
});
|
||||
|
||||
it("returns true when SpeechRecognition is on window", () => {
|
||||
installMock();
|
||||
expect(isSpeechRecognitionSupported()).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("useSpeechRecognition", () => {
|
||||
it("reports isSupported=false when API is missing", () => {
|
||||
removeMock();
|
||||
const { result } = renderHook(() => useSpeechRecognition());
|
||||
expect(result.current.isSupported).toBe(false);
|
||||
expect(result.current.isRecording).toBe(false);
|
||||
});
|
||||
|
||||
it("reports isSupported=true when API is present", () => {
|
||||
installMock();
|
||||
const { result } = renderHook(() => useSpeechRecognition());
|
||||
expect(result.current.isSupported).toBe(true);
|
||||
});
|
||||
|
||||
it("starts recording and sets isRecording", () => {
|
||||
installMock();
|
||||
const { result } = renderHook(() => useSpeechRecognition());
|
||||
|
||||
act(() => {
|
||||
result.current.start();
|
||||
});
|
||||
|
||||
expect(result.current.isRecording).toBe(true);
|
||||
expect(result.current.transcript).toBe("");
|
||||
expect(lastInstance?.start).toHaveBeenCalled();
|
||||
expect(lastInstance?.continuous).toBe(true);
|
||||
expect(lastInstance?.interimResults).toBe(true);
|
||||
});
|
||||
|
||||
it("accumulates transcript from result events", () => {
|
||||
installMock();
|
||||
const { result } = renderHook(() => useSpeechRecognition());
|
||||
|
||||
act(() => {
|
||||
result.current.start();
|
||||
});
|
||||
|
||||
// Simulate an interim result.
|
||||
act(() => {
|
||||
lastInstance?.onresult?.({
|
||||
resultIndex: 0,
|
||||
results: {
|
||||
length: 1,
|
||||
0: { isFinal: false, 0: { transcript: "hello", confidence: 0.9 } },
|
||||
},
|
||||
});
|
||||
});
|
||||
expect(result.current.transcript).toBe("hello");
|
||||
|
||||
// Simulate it becoming final + a new interim.
|
||||
act(() => {
|
||||
lastInstance?.onresult?.({
|
||||
resultIndex: 0,
|
||||
results: {
|
||||
length: 2,
|
||||
0: { isFinal: true, 0: { transcript: "hello ", confidence: 0.99 } },
|
||||
1: { isFinal: false, 0: { transcript: "world", confidence: 0.8 } },
|
||||
},
|
||||
});
|
||||
});
|
||||
expect(result.current.transcript).toBe("hello world");
|
||||
});
|
||||
|
||||
it("stop() keeps transcript and sets isRecording=false", () => {
|
||||
installMock();
|
||||
const { result } = renderHook(() => useSpeechRecognition());
|
||||
|
||||
act(() => {
|
||||
result.current.start();
|
||||
});
|
||||
|
||||
act(() => {
|
||||
lastInstance?.onresult?.({
|
||||
resultIndex: 0,
|
||||
results: {
|
||||
length: 1,
|
||||
0: { isFinal: true, 0: { transcript: "kept", confidence: 1 } },
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
act(() => {
|
||||
result.current.stop();
|
||||
});
|
||||
|
||||
expect(result.current.isRecording).toBe(false);
|
||||
expect(result.current.transcript).toBe("kept");
|
||||
expect(lastInstance?.stop).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("cancel() clears transcript and sets isRecording=false", () => {
|
||||
installMock();
|
||||
const { result } = renderHook(() => useSpeechRecognition());
|
||||
|
||||
act(() => {
|
||||
result.current.start();
|
||||
});
|
||||
|
||||
act(() => {
|
||||
lastInstance?.onresult?.({
|
||||
resultIndex: 0,
|
||||
results: {
|
||||
length: 1,
|
||||
0: {
|
||||
isFinal: false,
|
||||
0: { transcript: "discard me", confidence: 0.5 },
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
expect(result.current.transcript).toBe("discard me");
|
||||
|
||||
act(() => {
|
||||
result.current.cancel();
|
||||
});
|
||||
|
||||
expect(result.current.isRecording).toBe(false);
|
||||
expect(result.current.transcript).toBe("");
|
||||
expect(lastInstance?.abort).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("cleans up on onerror", () => {
|
||||
installMock();
|
||||
const { result } = renderHook(() => useSpeechRecognition());
|
||||
|
||||
act(() => {
|
||||
result.current.start();
|
||||
});
|
||||
expect(result.current.isRecording).toBe(true);
|
||||
|
||||
act(() => {
|
||||
lastInstance?.onerror?.({ error: "not-allowed", message: "" });
|
||||
});
|
||||
expect(result.current.isRecording).toBe(false);
|
||||
});
|
||||
|
||||
it("start() aborts a previous instance", () => {
|
||||
installMock();
|
||||
const { result } = renderHook(() => useSpeechRecognition());
|
||||
|
||||
act(() => {
|
||||
result.current.start();
|
||||
});
|
||||
const first = lastInstance;
|
||||
|
||||
act(() => {
|
||||
result.current.start();
|
||||
});
|
||||
|
||||
expect(first?.abort).toHaveBeenCalled();
|
||||
expect(lastInstance).not.toBe(first);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,166 @@
|
||||
import { useCallback, useRef, useState } from "react";
|
||||
|
||||
// Inline type declarations for the Web Speech API, which is not covered
|
||||
// by TypeScript's built-in lib types in all environments.
|
||||
|
||||
interface SpeechRecognitionResultItem {
|
||||
readonly transcript: string;
|
||||
readonly confidence: number;
|
||||
}
|
||||
|
||||
interface SpeechRecognitionResult {
|
||||
readonly length: number;
|
||||
readonly isFinal: boolean;
|
||||
item(index: number): SpeechRecognitionResultItem;
|
||||
[index: number]: SpeechRecognitionResultItem;
|
||||
}
|
||||
|
||||
interface SpeechRecognitionResultList {
|
||||
readonly length: number;
|
||||
item(index: number): SpeechRecognitionResult;
|
||||
[index: number]: SpeechRecognitionResult;
|
||||
}
|
||||
|
||||
interface SpeechRecognitionEvent extends Event {
|
||||
readonly resultIndex: number;
|
||||
readonly results: SpeechRecognitionResultList;
|
||||
}
|
||||
|
||||
interface SpeechRecognitionErrorEvent extends Event {
|
||||
readonly error: string;
|
||||
readonly message: string;
|
||||
}
|
||||
|
||||
interface SpeechRecognitionInstance extends EventTarget {
|
||||
lang: string;
|
||||
continuous: boolean;
|
||||
interimResults: boolean;
|
||||
onresult: ((event: SpeechRecognitionEvent) => void) | null;
|
||||
onerror: ((event: SpeechRecognitionErrorEvent) => void) | null;
|
||||
onend: (() => void) | null;
|
||||
start(): void;
|
||||
stop(): void;
|
||||
abort(): void;
|
||||
}
|
||||
|
||||
interface SpeechRecognitionConstructor {
|
||||
new (): SpeechRecognitionInstance;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the SpeechRecognition constructor if the browser supports it,
|
||||
* or undefined otherwise.
|
||||
*/
|
||||
function getSpeechRecognitionCtor(): SpeechRecognitionConstructor | undefined {
|
||||
// The Web Speech API is available as SpeechRecognition in standards-
|
||||
// compliant browsers and as webkitSpeechRecognition in WebKit-based
|
||||
// browsers. We check both to maximise compatibility.
|
||||
const win = window as Window &
|
||||
typeof globalThis & {
|
||||
SpeechRecognition?: SpeechRecognitionConstructor;
|
||||
webkitSpeechRecognition?: SpeechRecognitionConstructor;
|
||||
};
|
||||
return win.SpeechRecognition ?? win.webkitSpeechRecognition;
|
||||
}
|
||||
|
||||
/**
|
||||
* Standalone helper that can be called outside of React to check whether
|
||||
* the Web Speech API is available in the current browser.
|
||||
*/
|
||||
export function isSpeechRecognitionSupported(): boolean {
|
||||
return getSpeechRecognitionCtor() !== undefined;
|
||||
}
|
||||
|
||||
export function useSpeechRecognition(): {
|
||||
isSupported: boolean;
|
||||
isRecording: boolean;
|
||||
transcript: string;
|
||||
start: () => void;
|
||||
stop: () => void;
|
||||
cancel: () => void;
|
||||
} {
|
||||
const [isRecording, setIsRecording] = useState(false);
|
||||
const [transcript, setTranscript] = useState("");
|
||||
const recognitionRef = useRef<SpeechRecognitionInstance | null>(null);
|
||||
|
||||
// Cache the constructor lookup once per hook instance so we don't hit
|
||||
// the window property on every render.
|
||||
const ctorRef = useRef<SpeechRecognitionConstructor | undefined>(
|
||||
getSpeechRecognitionCtor(),
|
||||
);
|
||||
|
||||
const isSupported = ctorRef.current !== undefined;
|
||||
|
||||
const start = useCallback(() => {
|
||||
const Ctor = ctorRef.current;
|
||||
if (!Ctor) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Tear down any lingering instance before creating a new one.
|
||||
if (recognitionRef.current) {
|
||||
recognitionRef.current.abort();
|
||||
recognitionRef.current = null;
|
||||
}
|
||||
|
||||
const recognition = new Ctor();
|
||||
recognition.lang = navigator.language;
|
||||
recognition.continuous = true;
|
||||
recognition.interimResults = true;
|
||||
|
||||
// We accumulate finalized text in a local variable so that the
|
||||
// onresult handler can build the full transcript from both final
|
||||
// and interim segments without depending on React state timing.
|
||||
let finalizedText = "";
|
||||
|
||||
recognition.onresult = (event: SpeechRecognitionEvent) => {
|
||||
let interim = "";
|
||||
for (let i = event.resultIndex; i < event.results.length; i++) {
|
||||
const result = event.results[i];
|
||||
if (result.isFinal) {
|
||||
finalizedText += result[0].transcript;
|
||||
} else {
|
||||
interim += result[0].transcript;
|
||||
}
|
||||
}
|
||||
setTranscript(finalizedText + interim);
|
||||
};
|
||||
|
||||
recognition.onerror = () => {
|
||||
setIsRecording(false);
|
||||
recognitionRef.current = null;
|
||||
};
|
||||
|
||||
recognition.onend = () => {
|
||||
setIsRecording(false);
|
||||
recognitionRef.current = null;
|
||||
};
|
||||
|
||||
recognitionRef.current = recognition;
|
||||
setTranscript("");
|
||||
setIsRecording(true);
|
||||
recognition.start();
|
||||
}, []);
|
||||
|
||||
const stop = useCallback(() => {
|
||||
// stop() lets the browser deliver any remaining final results
|
||||
// before firing the onend event.
|
||||
if (recognitionRef.current) {
|
||||
recognitionRef.current.stop();
|
||||
recognitionRef.current = null;
|
||||
}
|
||||
setIsRecording(false);
|
||||
}, []);
|
||||
|
||||
const cancel = useCallback(() => {
|
||||
// abort() discards any pending audio and results immediately.
|
||||
if (recognitionRef.current) {
|
||||
recognitionRef.current.abort();
|
||||
recognitionRef.current = null;
|
||||
}
|
||||
setIsRecording(false);
|
||||
setTranscript("");
|
||||
}, []);
|
||||
|
||||
return { isSupported, isRecording, transcript, start, stop, cancel };
|
||||
}
|
||||
@@ -14,15 +14,25 @@ import {
|
||||
TooltipContent,
|
||||
TooltipTrigger,
|
||||
} from "components/Tooltip/Tooltip";
|
||||
import { useSpeechRecognition } from "hooks/useSpeechRecognition";
|
||||
import {
|
||||
AlertTriangleIcon,
|
||||
ArrowUpIcon,
|
||||
CheckIcon,
|
||||
ImageIcon,
|
||||
MicIcon,
|
||||
Square,
|
||||
XIcon,
|
||||
} from "lucide-react";
|
||||
import type React from "react";
|
||||
import { memo, type ReactNode, useCallback, useRef, useState } from "react";
|
||||
import {
|
||||
memo,
|
||||
type ReactNode,
|
||||
useCallback,
|
||||
useEffect,
|
||||
useRef,
|
||||
useState,
|
||||
} from "react";
|
||||
import { cn } from "utils/cn";
|
||||
import { ImageLightbox } from "./ImageLightbox";
|
||||
import { formatProviderLabel } from "./modelOptions";
|
||||
@@ -351,6 +361,22 @@ export const AgentChatInput = memo<AgentChatInputProps>(
|
||||
|
||||
const [hasFileReferences, setHasFileReferences] = useState(false);
|
||||
|
||||
const speech = useSpeechRecognition();
|
||||
const [preRecordingValue, setPreRecordingValue] = useState<string>("");
|
||||
|
||||
useEffect(() => {
|
||||
if (!speech.isRecording) return;
|
||||
const editor = internalRef.current;
|
||||
if (!editor) return;
|
||||
editor.clear();
|
||||
const combined = preRecordingValue
|
||||
? `${preRecordingValue} ${speech.transcript}`
|
||||
: speech.transcript;
|
||||
if (combined) {
|
||||
editor.insertText(combined);
|
||||
}
|
||||
}, [speech.transcript, speech.isRecording, preRecordingValue]);
|
||||
|
||||
// Merge the external inputRef with our internal ref so both
|
||||
// point to the same ChatMessageInputRef instance.
|
||||
const setRef = useCallback(
|
||||
@@ -497,6 +523,28 @@ export const AgentChatInput = memo<AgentChatInputProps>(
|
||||
queuedMessages,
|
||||
onPromoteQueuedMessage,
|
||||
]);
|
||||
const handleStartRecording = useCallback(() => {
|
||||
setPreRecordingValue(internalRef.current?.getValue()?.trim() ?? "");
|
||||
speech.start();
|
||||
}, [speech]);
|
||||
|
||||
const handleAcceptRecording = useCallback(() => {
|
||||
speech.stop();
|
||||
}, [speech]);
|
||||
|
||||
const handleCancelRecording = useCallback(() => {
|
||||
const original = preRecordingValue;
|
||||
speech.cancel();
|
||||
const editor = internalRef.current;
|
||||
if (editor) {
|
||||
editor.clear();
|
||||
if (original) {
|
||||
editor.insertText(original);
|
||||
}
|
||||
}
|
||||
setPreRecordingValue("");
|
||||
}, [speech, preRecordingValue]);
|
||||
|
||||
const handleKeyDown = (e: React.KeyboardEvent) => {
|
||||
if (e.key === "Escape") {
|
||||
if (editingQueuedMessageID !== null) {
|
||||
@@ -646,6 +694,25 @@ export const AgentChatInput = memo<AgentChatInputProps>(
|
||||
</Button>
|
||||
</>
|
||||
)}
|
||||
{speech.isSupported && !isStreaming && (
|
||||
<Button
|
||||
type="button"
|
||||
variant="subtle"
|
||||
size="icon"
|
||||
className="size-7 shrink-0 rounded-full [&>svg]:!size-icon-sm [&>svg]:p-0"
|
||||
onClick={
|
||||
speech.isRecording
|
||||
? handleCancelRecording
|
||||
: handleStartRecording
|
||||
}
|
||||
disabled={isDisabled}
|
||||
aria-label={
|
||||
speech.isRecording ? "Cancel voice input" : "Voice input"
|
||||
}
|
||||
>
|
||||
{speech.isRecording ? <XIcon /> : <MicIcon />}
|
||||
</Button>
|
||||
)}
|
||||
{contextUsage !== undefined && (
|
||||
<ContextUsageIndicator usage={contextUsage} />
|
||||
)}{" "}
|
||||
@@ -666,15 +733,23 @@ export const AgentChatInput = memo<AgentChatInputProps>(
|
||||
size="icon"
|
||||
variant="default"
|
||||
className="size-7 rounded-full transition-colors [&>svg]:!size-5 [&>svg]:p-0"
|
||||
onClick={handleSubmit}
|
||||
disabled={!canSend}
|
||||
onClick={
|
||||
speech.isRecording ? handleAcceptRecording : handleSubmit
|
||||
}
|
||||
disabled={speech.isRecording ? false : !canSend}
|
||||
>
|
||||
{isLoading ? (
|
||||
<Spinner size="sm" loading aria-hidden="true" />
|
||||
) : speech.isRecording ? (
|
||||
<CheckIcon />
|
||||
) : (
|
||||
<ArrowUpIcon />
|
||||
)}
|
||||
<span className="sr-only">{sendButtonLabel}</span>
|
||||
<span className="sr-only">
|
||||
{speech.isRecording
|
||||
? "Accept voice input"
|
||||
: sendButtonLabel}
|
||||
</span>
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
|
||||
Reference in New Issue
Block a user