diff --git a/site/src/hooks/useSpeechRecognition.test.ts b/site/src/hooks/useSpeechRecognition.test.ts new file mode 100644 index 0000000000..ef8b906555 --- /dev/null +++ b/site/src/hooks/useSpeechRecognition.test.ts @@ -0,0 +1,236 @@ +import { act, renderHook } from "@testing-library/react"; +import { + isSpeechRecognitionSupported, + useSpeechRecognition, +} from "./useSpeechRecognition"; + +// --------------------------------------------------------------------------- +// Minimal mock for the Web Speech API SpeechRecognition class. +// --------------------------------------------------------------------------- + +type ResultHandler = (event: { + resultIndex: number; + results: { + length: number; + [i: number]: { + isFinal: boolean; + 0: { transcript: string; confidence: number }; + }; + }; +}) => void; + +class MockSpeechRecognition { + lang = ""; + continuous = false; + interimResults = false; + onresult: ResultHandler | null = null; + onerror: ((event: { error: string; message: string }) => void) | null = null; + onend: (() => void) | null = null; + + start = vi.fn(); + stop = vi.fn(() => { + // Browser fires onend after stop. + this.onend?.(); + }); + abort = vi.fn(() => { + this.onend?.(); + }); +} + +let lastInstance: MockSpeechRecognition | null = null; + +function installMock() { + lastInstance = null; + // Use a real class so `new Ctor()` works — vi.fn() arrow + // functions are not constructable. + class Ctor extends MockSpeechRecognition { + constructor() { + super(); + lastInstance = this; + } + } + Object.assign(window, { SpeechRecognition: Ctor }); + return Ctor; +} + +function removeMock() { + Object.assign(window, { + SpeechRecognition: undefined, + webkitSpeechRecognition: undefined, + }); + lastInstance = null; +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +afterEach(() => { + removeMock(); +}); + +describe("isSpeechRecognitionSupported", () => { + it("returns false when API is not available", () => { + removeMock(); + expect(isSpeechRecognitionSupported()).toBe(false); + }); + + it("returns true when SpeechRecognition is on window", () => { + installMock(); + expect(isSpeechRecognitionSupported()).toBe(true); + }); +}); + +describe("useSpeechRecognition", () => { + it("reports isSupported=false when API is missing", () => { + removeMock(); + const { result } = renderHook(() => useSpeechRecognition()); + expect(result.current.isSupported).toBe(false); + expect(result.current.isRecording).toBe(false); + }); + + it("reports isSupported=true when API is present", () => { + installMock(); + const { result } = renderHook(() => useSpeechRecognition()); + expect(result.current.isSupported).toBe(true); + }); + + it("starts recording and sets isRecording", () => { + installMock(); + const { result } = renderHook(() => useSpeechRecognition()); + + act(() => { + result.current.start(); + }); + + expect(result.current.isRecording).toBe(true); + expect(result.current.transcript).toBe(""); + expect(lastInstance?.start).toHaveBeenCalled(); + expect(lastInstance?.continuous).toBe(true); + expect(lastInstance?.interimResults).toBe(true); + }); + + it("accumulates transcript from result events", () => { + installMock(); + const { result } = renderHook(() => useSpeechRecognition()); + + act(() => { + result.current.start(); + }); + + // Simulate an interim result. + act(() => { + lastInstance?.onresult?.({ + resultIndex: 0, + results: { + length: 1, + 0: { isFinal: false, 0: { transcript: "hello", confidence: 0.9 } }, + }, + }); + }); + expect(result.current.transcript).toBe("hello"); + + // Simulate it becoming final + a new interim. + act(() => { + lastInstance?.onresult?.({ + resultIndex: 0, + results: { + length: 2, + 0: { isFinal: true, 0: { transcript: "hello ", confidence: 0.99 } }, + 1: { isFinal: false, 0: { transcript: "world", confidence: 0.8 } }, + }, + }); + }); + expect(result.current.transcript).toBe("hello world"); + }); + + it("stop() keeps transcript and sets isRecording=false", () => { + installMock(); + const { result } = renderHook(() => useSpeechRecognition()); + + act(() => { + result.current.start(); + }); + + act(() => { + lastInstance?.onresult?.({ + resultIndex: 0, + results: { + length: 1, + 0: { isFinal: true, 0: { transcript: "kept", confidence: 1 } }, + }, + }); + }); + + act(() => { + result.current.stop(); + }); + + expect(result.current.isRecording).toBe(false); + expect(result.current.transcript).toBe("kept"); + expect(lastInstance?.stop).toHaveBeenCalled(); + }); + + it("cancel() clears transcript and sets isRecording=false", () => { + installMock(); + const { result } = renderHook(() => useSpeechRecognition()); + + act(() => { + result.current.start(); + }); + + act(() => { + lastInstance?.onresult?.({ + resultIndex: 0, + results: { + length: 1, + 0: { + isFinal: false, + 0: { transcript: "discard me", confidence: 0.5 }, + }, + }, + }); + }); + expect(result.current.transcript).toBe("discard me"); + + act(() => { + result.current.cancel(); + }); + + expect(result.current.isRecording).toBe(false); + expect(result.current.transcript).toBe(""); + expect(lastInstance?.abort).toHaveBeenCalled(); + }); + + it("cleans up on onerror", () => { + installMock(); + const { result } = renderHook(() => useSpeechRecognition()); + + act(() => { + result.current.start(); + }); + expect(result.current.isRecording).toBe(true); + + act(() => { + lastInstance?.onerror?.({ error: "not-allowed", message: "" }); + }); + expect(result.current.isRecording).toBe(false); + }); + + it("start() aborts a previous instance", () => { + installMock(); + const { result } = renderHook(() => useSpeechRecognition()); + + act(() => { + result.current.start(); + }); + const first = lastInstance; + + act(() => { + result.current.start(); + }); + + expect(first?.abort).toHaveBeenCalled(); + expect(lastInstance).not.toBe(first); + }); +}); diff --git a/site/src/hooks/useSpeechRecognition.ts b/site/src/hooks/useSpeechRecognition.ts new file mode 100644 index 0000000000..0ce8994fe0 --- /dev/null +++ b/site/src/hooks/useSpeechRecognition.ts @@ -0,0 +1,166 @@ +import { useCallback, useRef, useState } from "react"; + +// Inline type declarations for the Web Speech API, which is not covered +// by TypeScript's built-in lib types in all environments. + +interface SpeechRecognitionResultItem { + readonly transcript: string; + readonly confidence: number; +} + +interface SpeechRecognitionResult { + readonly length: number; + readonly isFinal: boolean; + item(index: number): SpeechRecognitionResultItem; + [index: number]: SpeechRecognitionResultItem; +} + +interface SpeechRecognitionResultList { + readonly length: number; + item(index: number): SpeechRecognitionResult; + [index: number]: SpeechRecognitionResult; +} + +interface SpeechRecognitionEvent extends Event { + readonly resultIndex: number; + readonly results: SpeechRecognitionResultList; +} + +interface SpeechRecognitionErrorEvent extends Event { + readonly error: string; + readonly message: string; +} + +interface SpeechRecognitionInstance extends EventTarget { + lang: string; + continuous: boolean; + interimResults: boolean; + onresult: ((event: SpeechRecognitionEvent) => void) | null; + onerror: ((event: SpeechRecognitionErrorEvent) => void) | null; + onend: (() => void) | null; + start(): void; + stop(): void; + abort(): void; +} + +interface SpeechRecognitionConstructor { + new (): SpeechRecognitionInstance; +} + +/** + * Returns the SpeechRecognition constructor if the browser supports it, + * or undefined otherwise. + */ +function getSpeechRecognitionCtor(): SpeechRecognitionConstructor | undefined { + // The Web Speech API is available as SpeechRecognition in standards- + // compliant browsers and as webkitSpeechRecognition in WebKit-based + // browsers. We check both to maximise compatibility. + const win = window as Window & + typeof globalThis & { + SpeechRecognition?: SpeechRecognitionConstructor; + webkitSpeechRecognition?: SpeechRecognitionConstructor; + }; + return win.SpeechRecognition ?? win.webkitSpeechRecognition; +} + +/** + * Standalone helper that can be called outside of React to check whether + * the Web Speech API is available in the current browser. + */ +export function isSpeechRecognitionSupported(): boolean { + return getSpeechRecognitionCtor() !== undefined; +} + +export function useSpeechRecognition(): { + isSupported: boolean; + isRecording: boolean; + transcript: string; + start: () => void; + stop: () => void; + cancel: () => void; +} { + const [isRecording, setIsRecording] = useState(false); + const [transcript, setTranscript] = useState(""); + const recognitionRef = useRef(null); + + // Cache the constructor lookup once per hook instance so we don't hit + // the window property on every render. + const ctorRef = useRef( + getSpeechRecognitionCtor(), + ); + + const isSupported = ctorRef.current !== undefined; + + const start = useCallback(() => { + const Ctor = ctorRef.current; + if (!Ctor) { + return; + } + + // Tear down any lingering instance before creating a new one. + if (recognitionRef.current) { + recognitionRef.current.abort(); + recognitionRef.current = null; + } + + const recognition = new Ctor(); + recognition.lang = navigator.language; + recognition.continuous = true; + recognition.interimResults = true; + + // We accumulate finalized text in a local variable so that the + // onresult handler can build the full transcript from both final + // and interim segments without depending on React state timing. + let finalizedText = ""; + + recognition.onresult = (event: SpeechRecognitionEvent) => { + let interim = ""; + for (let i = event.resultIndex; i < event.results.length; i++) { + const result = event.results[i]; + if (result.isFinal) { + finalizedText += result[0].transcript; + } else { + interim += result[0].transcript; + } + } + setTranscript(finalizedText + interim); + }; + + recognition.onerror = () => { + setIsRecording(false); + recognitionRef.current = null; + }; + + recognition.onend = () => { + setIsRecording(false); + recognitionRef.current = null; + }; + + recognitionRef.current = recognition; + setTranscript(""); + setIsRecording(true); + recognition.start(); + }, []); + + const stop = useCallback(() => { + // stop() lets the browser deliver any remaining final results + // before firing the onend event. + if (recognitionRef.current) { + recognitionRef.current.stop(); + recognitionRef.current = null; + } + setIsRecording(false); + }, []); + + const cancel = useCallback(() => { + // abort() discards any pending audio and results immediately. + if (recognitionRef.current) { + recognitionRef.current.abort(); + recognitionRef.current = null; + } + setIsRecording(false); + setTranscript(""); + }, []); + + return { isSupported, isRecording, transcript, start, stop, cancel }; +} diff --git a/site/src/pages/AgentsPage/AgentChatInput.tsx b/site/src/pages/AgentsPage/AgentChatInput.tsx index 421df3cd9d..c3fe104efe 100644 --- a/site/src/pages/AgentsPage/AgentChatInput.tsx +++ b/site/src/pages/AgentsPage/AgentChatInput.tsx @@ -14,15 +14,25 @@ import { TooltipContent, TooltipTrigger, } from "components/Tooltip/Tooltip"; +import { useSpeechRecognition } from "hooks/useSpeechRecognition"; import { AlertTriangleIcon, ArrowUpIcon, + CheckIcon, ImageIcon, + MicIcon, Square, XIcon, } from "lucide-react"; import type React from "react"; -import { memo, type ReactNode, useCallback, useRef, useState } from "react"; +import { + memo, + type ReactNode, + useCallback, + useEffect, + useRef, + useState, +} from "react"; import { cn } from "utils/cn"; import { ImageLightbox } from "./ImageLightbox"; import { formatProviderLabel } from "./modelOptions"; @@ -351,6 +361,22 @@ export const AgentChatInput = memo( const [hasFileReferences, setHasFileReferences] = useState(false); + const speech = useSpeechRecognition(); + const [preRecordingValue, setPreRecordingValue] = useState(""); + + useEffect(() => { + if (!speech.isRecording) return; + const editor = internalRef.current; + if (!editor) return; + editor.clear(); + const combined = preRecordingValue + ? `${preRecordingValue} ${speech.transcript}` + : speech.transcript; + if (combined) { + editor.insertText(combined); + } + }, [speech.transcript, speech.isRecording, preRecordingValue]); + // Merge the external inputRef with our internal ref so both // point to the same ChatMessageInputRef instance. const setRef = useCallback( @@ -497,6 +523,28 @@ export const AgentChatInput = memo( queuedMessages, onPromoteQueuedMessage, ]); + const handleStartRecording = useCallback(() => { + setPreRecordingValue(internalRef.current?.getValue()?.trim() ?? ""); + speech.start(); + }, [speech]); + + const handleAcceptRecording = useCallback(() => { + speech.stop(); + }, [speech]); + + const handleCancelRecording = useCallback(() => { + const original = preRecordingValue; + speech.cancel(); + const editor = internalRef.current; + if (editor) { + editor.clear(); + if (original) { + editor.insertText(original); + } + } + setPreRecordingValue(""); + }, [speech, preRecordingValue]); + const handleKeyDown = (e: React.KeyboardEvent) => { if (e.key === "Escape") { if (editingQueuedMessageID !== null) { @@ -646,6 +694,25 @@ export const AgentChatInput = memo( )} + {speech.isSupported && !isStreaming && ( + + )} {contextUsage !== undefined && ( )}{" "} @@ -666,15 +733,23 @@ export const AgentChatInput = memo( size="icon" variant="default" className="size-7 rounded-full transition-colors [&>svg]:!size-5 [&>svg]:p-0" - onClick={handleSubmit} - disabled={!canSend} + onClick={ + speech.isRecording ? handleAcceptRecording : handleSubmit + } + disabled={speech.isRecording ? false : !canSend} > {isLoading ? (