mirror of
https://github.com/simstudioai/sim.git
synced 2026-10-13 15:48:31 +08:00
chore(chat): remove deployed-chat voice mode (#6215)
Removes the voice-first interface and TTS playback from the deployed chat, keeping workspace dictation, which is a separate feature. - Deletes the VoiceInterface UI and its particles canvas, the chat mic input, the TTS audio-streaming hook, the /api/proxy/tts/stream relay and its contract, and the voice-settings query hook. - Unpicks the voice wiring in chat.tsx and use-chat-streaming: the audio stream handler, sentence-splitting for speech, voice-first mode state, and the isVoiceInput plumbing through ChatInput. - Drops the now-dead chatId branch from /api/speech/token. It was the anonymous public-chat path; with no caller left it would have stayed an unauthenticated relay spending the platform key. That leaves resolveDeployedChatCaller unused, so it goes too. - Removes code the above orphaned: MAX_CHAT_SESSION_MS, the noop util, the audio/position refs in use-chat-streaming that were only ever written, and the chatId field on the speech contract. Keeps /api/settings/voice, lib/speech and use-speech-to-text: the workspace home input still uses them. Keeps the voice-output usage source, enum and label — Postgres cannot drop an enum value, and historical usage_log rows still need a label to render.
This commit is contained in:
Waleed
@@ -3,8 +3,6 @@
|
||||
import { type RefObject, useCallback, useEffect, useMemo, useRef, useState } from 'react'
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { generateId } from '@sim/utils/id'
|
||||
import { DEFAULT_TTS_VOICE_ID } from '@/lib/api/contracts/media/tts-stream'
|
||||
import { noop } from '@/lib/core/utils/request'
|
||||
import {
|
||||
AGENT_STREAM_PROTOCOL_HEADER,
|
||||
AGENT_STREAM_PROTOCOL_V1,
|
||||
@@ -19,23 +17,15 @@ import {
|
||||
ChatMessageContainer,
|
||||
EmailAuth,
|
||||
PasswordAuth,
|
||||
VoiceInterface,
|
||||
} from '@/app/(interfaces)/chat/components'
|
||||
import { CHAT_ERROR_MESSAGES, CHAT_REQUEST_TIMEOUT_MS } from '@/app/(interfaces)/chat/constants'
|
||||
import { useAudioStreaming, useChatStreaming } from '@/app/(interfaces)/chat/hooks'
|
||||
import { useChatStreaming } from '@/app/(interfaces)/chat/hooks'
|
||||
import SSOAuth from '@/ee/sso/components/sso-auth'
|
||||
import { useDeployedChatConfig } from '@/hooks/queries/chats'
|
||||
import { useGitHubStars } from '@/hooks/queries/github-stars'
|
||||
import { useVoiceSettings } from '@/hooks/queries/voice-settings'
|
||||
|
||||
const logger = createLogger('ChatClient')
|
||||
|
||||
interface AudioStreamingOptions {
|
||||
voiceId: string
|
||||
chatId: string
|
||||
onError: (error: Error) => void
|
||||
}
|
||||
|
||||
interface ChatRequestFile {
|
||||
name: string
|
||||
size: number
|
||||
@@ -49,10 +39,6 @@ interface ChatRequestPayload {
|
||||
files?: ChatRequestFile[]
|
||||
}
|
||||
|
||||
const DEFAULT_VOICE_SETTINGS = {
|
||||
voiceId: DEFAULT_TTS_VOICE_ID,
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts a File object to a base64 data URL
|
||||
*/
|
||||
@@ -65,33 +51,6 @@ function fileToBase64(file: File): Promise<string> {
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates an audio stream handler for text-to-speech conversion
|
||||
* @param streamTextToAudio - Function to stream text to audio
|
||||
* @param voiceId - The voice ID to use for TTS
|
||||
* @param chatId - Optional chat ID for deployed chat authentication
|
||||
* @returns Audio stream handler function or undefined
|
||||
*/
|
||||
function createAudioStreamHandler(
|
||||
streamTextToAudio: (text: string, options: AudioStreamingOptions) => Promise<void>,
|
||||
voiceId: string,
|
||||
chatId: string
|
||||
) {
|
||||
return async (text: string) => {
|
||||
try {
|
||||
await streamTextToAudio(text, {
|
||||
voiceId,
|
||||
chatId,
|
||||
onError: (error: Error) => {
|
||||
logger.error('Audio streaming error:', error)
|
||||
},
|
||||
})
|
||||
} catch (error) {
|
||||
logger.error('TTS error:', error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default function ChatClient({ identifier }: { identifier: string }) {
|
||||
const [messages, setMessages] = useState<ChatMessage[]>([])
|
||||
const [inputValue, setInputValue] = useState('')
|
||||
@@ -105,13 +64,9 @@ export default function ChatClient({ identifier }: { identifier: string }) {
|
||||
const stickToBottomRef = useRef(true)
|
||||
const ignoreScrollRef = useRef(false)
|
||||
|
||||
const [isVoiceFirstMode, setIsVoiceFirstMode] = useState(false)
|
||||
|
||||
const { data: chatConfigResult, error: chatConfigError } = useDeployedChatConfig(identifier)
|
||||
const { data: voiceSettings } = useVoiceSettings()
|
||||
const { data: starCount } = useGitHubStars()
|
||||
|
||||
const sttAvailable = voiceSettings?.sttAvailable === true
|
||||
const authRequired = chatConfigResult?.kind === 'auth' ? chatConfigResult.authType : null
|
||||
const chatConfig = chatConfigResult?.kind === 'config' ? chatConfigResult.config : null
|
||||
|
||||
@@ -135,8 +90,6 @@ export default function ChatClient({ identifier }: { identifier: string }) {
|
||||
|
||||
const { isStreamingResponse, abortControllerRef, stopStreaming, handleStreamedResponse } =
|
||||
useChatStreaming()
|
||||
const audioContextRef = useRef<AudioContext | null>(null)
|
||||
const { isPlayingAudio, streamTextToAudio, stopAudio } = useAudioStreaming(audioContextRef)
|
||||
|
||||
const NEAR_BOTTOM_THRESHOLD_PX = 100
|
||||
|
||||
@@ -208,11 +161,10 @@ export default function ChatClient({ identifier }: { identifier: string }) {
|
||||
|
||||
container.addEventListener('scroll', handleScroll, { passive: true })
|
||||
return () => container.removeEventListener('scroll', handleScroll)
|
||||
}, [chatConfig, isVoiceFirstMode, authRequired])
|
||||
}, [chatConfig, authRequired])
|
||||
|
||||
const handleSendMessage = async (
|
||||
messageParam?: string,
|
||||
isVoiceInput = false,
|
||||
files?: Array<{
|
||||
id: string
|
||||
name: string
|
||||
@@ -227,7 +179,6 @@ export default function ChatClient({ identifier }: { identifier: string }) {
|
||||
|
||||
logger.info('Sending message:', {
|
||||
messageToSend,
|
||||
isVoiceInput,
|
||||
conversationId,
|
||||
filesCount: files?.length,
|
||||
})
|
||||
@@ -316,30 +267,12 @@ export default function ChatClient({ identifier }: { identifier: string }) {
|
||||
throw new Error('Response body is missing')
|
||||
}
|
||||
|
||||
const shouldPlayAudio = isVoiceInput || isVoiceFirstMode
|
||||
const audioHandler =
|
||||
shouldPlayAudio && chatConfig?.id
|
||||
? createAudioStreamHandler(
|
||||
streamTextToAudio,
|
||||
DEFAULT_VOICE_SETTINGS.voiceId,
|
||||
chatConfig.id
|
||||
)
|
||||
: undefined
|
||||
|
||||
logger.info('Starting to handle streamed response:', { shouldPlayAudio })
|
||||
|
||||
await handleStreamedResponse(
|
||||
response,
|
||||
setMessages,
|
||||
setIsLoading,
|
||||
() => scrollToBottom({ behavior: 'auto' }),
|
||||
{
|
||||
voiceSettings: {
|
||||
isVoiceEnabled: shouldPlayAudio,
|
||||
voiceId: DEFAULT_VOICE_SETTINGS.voiceId,
|
||||
autoPlayResponses: shouldPlayAudio,
|
||||
},
|
||||
audioStreamHandler: audioHandler,
|
||||
outputConfigs: chatConfig?.outputConfigs,
|
||||
abortController,
|
||||
}
|
||||
@@ -365,41 +298,6 @@ export default function ChatClient({ identifier }: { identifier: string }) {
|
||||
}
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
stopAudio()
|
||||
if (audioContextRef.current && audioContextRef.current.state !== 'closed') {
|
||||
audioContextRef.current.close()
|
||||
}
|
||||
}
|
||||
}, [stopAudio])
|
||||
|
||||
const handleVoiceInterruption = useCallback(() => {
|
||||
stopAudio()
|
||||
|
||||
if (isStreamingResponse) {
|
||||
stopStreaming(setMessages)
|
||||
}
|
||||
}, [isStreamingResponse, stopStreaming, setMessages, stopAudio])
|
||||
|
||||
const handleVoiceStart = useCallback(() => {
|
||||
if (!sttAvailable) return
|
||||
setIsVoiceFirstMode(true)
|
||||
}, [sttAvailable])
|
||||
|
||||
const handleExitVoiceMode = useCallback(() => {
|
||||
setIsVoiceFirstMode(false)
|
||||
stopAudio()
|
||||
}, [stopAudio])
|
||||
|
||||
const handleVoiceTranscript = useCallback(
|
||||
(transcript: string) => {
|
||||
logger.info('Received voice transcript:', transcript)
|
||||
handleSendMessage(transcript, true)
|
||||
},
|
||||
[handleSendMessage]
|
||||
)
|
||||
|
||||
if (chatConfigError) {
|
||||
logger.error('Error fetching chat config:', chatConfigError)
|
||||
return <ChatErrorState error={CHAT_ERROR_MESSAGES.CHAT_UNAVAILABLE} />
|
||||
@@ -421,26 +319,6 @@ export default function ChatClient({ identifier }: { identifier: string }) {
|
||||
return <ChatLoadingState />
|
||||
}
|
||||
|
||||
if (isVoiceFirstMode) {
|
||||
return (
|
||||
<VoiceInterface
|
||||
onCallEnd={handleExitVoiceMode}
|
||||
onVoiceTranscript={handleVoiceTranscript}
|
||||
onVoiceStart={noop}
|
||||
onVoiceEnd={noop}
|
||||
onInterrupt={handleVoiceInterruption}
|
||||
isStreaming={isStreamingResponse}
|
||||
isPlayingAudio={isPlayingAudio}
|
||||
audioContextRef={audioContextRef}
|
||||
chatId={chatConfig?.id}
|
||||
messages={displayMessages.map((msg) => ({
|
||||
content: typeof msg.content === 'string' ? msg.content : JSON.stringify(msg.content),
|
||||
type: msg.type,
|
||||
}))}
|
||||
/>
|
||||
)
|
||||
}
|
||||
|
||||
return (
|
||||
<div className='light desktop-title-bar-page fixed inset-0 z-[100] flex flex-col bg-[var(--bg)] text-[var(--text-primary)]'>
|
||||
<DesktopTitleBarLane />
|
||||
@@ -463,13 +341,11 @@ export default function ChatClient({ identifier }: { identifier: string }) {
|
||||
<div className='relative p-3 pb-4 md:p-4 md:pb-6'>
|
||||
<div className='relative mx-auto max-w-3xl md:max-w-[748px]'>
|
||||
<ChatInput
|
||||
onSubmit={(value, isVoiceInput, files) => {
|
||||
void handleSendMessage(value, isVoiceInput, files)
|
||||
onSubmit={(value, files) => {
|
||||
void handleSendMessage(value, files)
|
||||
}}
|
||||
isStreaming={isStreamingResponse}
|
||||
onStopStreaming={() => stopStreaming(setMessages)}
|
||||
onVoiceStart={handleVoiceStart}
|
||||
sttAvailable={sttAvailable}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -6,4 +6,3 @@ export { ChatInput } from './input/input'
|
||||
export { ChatLoadingState } from './loading-state/loading-state'
|
||||
export type { ChatMessage } from './message/message'
|
||||
export { ChatMessageContainer } from './message-container/message-container'
|
||||
export { VoiceInterface } from './voice-interface/voice-interface'
|
||||
|
||||
@@ -5,9 +5,8 @@ import { useCallback, useLayoutEffect, useRef, useState } from 'react'
|
||||
import { Badge, Button, cn, handleKeyboardActivation, Tooltip } from '@sim/emcn'
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { generateId } from '@sim/utils/id'
|
||||
import { ArrowUp, Mic, Paperclip, X } from 'lucide-react'
|
||||
import { ArrowUp, Paperclip, X } from 'lucide-react'
|
||||
import { CHAT_ACCEPT_ATTRIBUTE } from '@/lib/uploads/utils/validation'
|
||||
import { VoiceInput } from '@/app/(interfaces)/chat/components/input/voice-input'
|
||||
|
||||
const logger = createLogger('ChatInput')
|
||||
|
||||
@@ -23,20 +22,10 @@ interface AttachedFile {
|
||||
}
|
||||
|
||||
export const ChatInput: React.FC<{
|
||||
onSubmit?: (value: string, isVoiceInput?: boolean, files?: AttachedFile[]) => void
|
||||
onSubmit?: (value: string, files?: AttachedFile[]) => void
|
||||
isStreaming?: boolean
|
||||
onStopStreaming?: () => void
|
||||
onVoiceStart?: () => void
|
||||
voiceOnly?: boolean
|
||||
sttAvailable?: boolean
|
||||
}> = ({
|
||||
onSubmit,
|
||||
isStreaming = false,
|
||||
onStopStreaming,
|
||||
onVoiceStart,
|
||||
voiceOnly = false,
|
||||
sttAvailable = false,
|
||||
}) => {
|
||||
}> = ({ onSubmit, isStreaming = false, onStopStreaming }) => {
|
||||
const fileInputRef = useRef<HTMLInputElement>(null)
|
||||
const textareaRef = useRef<HTMLTextAreaElement>(null)
|
||||
const [inputValue, setInputValue] = useState('')
|
||||
@@ -114,7 +103,7 @@ export const ChatInput: React.FC<{
|
||||
const handleSubmit = useCallback(() => {
|
||||
if (isStreaming) return
|
||||
if (!inputValue.trim() && attachedFiles.length === 0) return
|
||||
onSubmit?.(inputValue.trim(), false, attachedFiles)
|
||||
onSubmit?.(inputValue.trim(), attachedFiles)
|
||||
setInputValue('')
|
||||
setAttachedFiles([])
|
||||
setUploadErrors([])
|
||||
@@ -141,31 +130,6 @@ export const ChatInput: React.FC<{
|
||||
|
||||
const canSubmit = (inputValue.trim().length > 0 || attachedFiles.length > 0) && !isStreaming
|
||||
|
||||
if (voiceOnly) {
|
||||
return (
|
||||
<Tooltip.Provider>
|
||||
<div className='flex items-center justify-center'>
|
||||
{sttAvailable && (
|
||||
<Tooltip.Root>
|
||||
<Tooltip.Trigger asChild>
|
||||
<div>
|
||||
<VoiceInput
|
||||
onVoiceStart={onVoiceStart ?? (() => {})}
|
||||
disabled={isStreaming}
|
||||
large={true}
|
||||
/>
|
||||
</div>
|
||||
</Tooltip.Trigger>
|
||||
<Tooltip.Content side='top'>
|
||||
<p>Start voice conversation</p>
|
||||
</Tooltip.Content>
|
||||
</Tooltip.Root>
|
||||
)}
|
||||
</div>
|
||||
</Tooltip.Provider>
|
||||
)
|
||||
}
|
||||
|
||||
return (
|
||||
<Tooltip.Provider>
|
||||
<div className='fixed right-0 bottom-0 left-0 flex w-full items-center justify-center bg-gradient-to-t from-[var(--bg)] to-transparent px-4 pb-4 md:px-0 md:pb-4'>
|
||||
@@ -302,26 +266,8 @@ export const ChatInput: React.FC<{
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* Right: mic + send */}
|
||||
{/* Right: send */}
|
||||
<div className='flex items-center gap-1.5'>
|
||||
{sttAvailable && (
|
||||
<Tooltip.Root>
|
||||
<Tooltip.Trigger asChild>
|
||||
<Button
|
||||
variant='quiet'
|
||||
onClick={onVoiceStart}
|
||||
disabled={isStreaming}
|
||||
className='size-[28px] rounded-full p-0'
|
||||
>
|
||||
<Mic className='size-[16px]' strokeWidth={2} />
|
||||
</Button>
|
||||
</Tooltip.Trigger>
|
||||
<Tooltip.Content side='top'>
|
||||
<p>Start voice conversation</p>
|
||||
</Tooltip.Content>
|
||||
</Tooltip.Root>
|
||||
)}
|
||||
|
||||
{isStreaming ? (
|
||||
<Button
|
||||
variant='primary'
|
||||
|
||||
@@ -1,89 +0,0 @@
|
||||
'use client'
|
||||
|
||||
import { useCallback } from 'react'
|
||||
import { domAnimation, LazyMotion, m } from 'framer-motion'
|
||||
import { Mic } from 'lucide-react'
|
||||
|
||||
interface VoiceInputProps {
|
||||
onVoiceStart: () => void
|
||||
isListening?: boolean
|
||||
disabled?: boolean
|
||||
large?: boolean
|
||||
minimal?: boolean
|
||||
}
|
||||
|
||||
export function VoiceInput({
|
||||
onVoiceStart,
|
||||
isListening = false,
|
||||
disabled = false,
|
||||
large = false,
|
||||
minimal = false,
|
||||
}: VoiceInputProps) {
|
||||
const handleVoiceClick = useCallback(() => {
|
||||
if (disabled) return
|
||||
onVoiceStart()
|
||||
}, [disabled, onVoiceStart])
|
||||
|
||||
if (minimal) {
|
||||
return (
|
||||
<button
|
||||
type='button'
|
||||
onClick={handleVoiceClick}
|
||||
disabled={disabled}
|
||||
className={`flex items-center justify-center rounded-full p-1.5 text-gray-600 transition-colors duration-200 hover:bg-gray-100 md:p-2 ${
|
||||
disabled ? 'cursor-not-allowed opacity-50' : 'cursor-pointer'
|
||||
}`}
|
||||
title='Start voice conversation'
|
||||
>
|
||||
<Mic size={16} className='md:h-5 md:w-5' />
|
||||
</button>
|
||||
)
|
||||
}
|
||||
|
||||
if (large) {
|
||||
return (
|
||||
<div className='flex flex-col items-center'>
|
||||
<LazyMotion features={domAnimation}>
|
||||
<m.button
|
||||
type='button'
|
||||
onClick={handleVoiceClick}
|
||||
disabled={disabled}
|
||||
className={`flex items-center justify-center rounded-full border-2 p-6 transition-all duration-200 ${
|
||||
isListening
|
||||
? 'border-red-400 bg-red-500/20 text-red-600 hover:bg-red-500/30'
|
||||
: 'border-blue-300 bg-blue-500/10 text-blue-600 hover:bg-blue-500/20'
|
||||
} ${disabled ? 'cursor-not-allowed opacity-50' : 'cursor-pointer'}`}
|
||||
whileHover={{ scale: 1.05 }}
|
||||
whileTap={{ scale: 0.95 }}
|
||||
title='Start voice conversation'
|
||||
>
|
||||
<Mic size={32} />
|
||||
</m.button>
|
||||
</LazyMotion>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
return (
|
||||
<div className='flex items-center'>
|
||||
<LazyMotion features={domAnimation}>
|
||||
<m.button
|
||||
type='button'
|
||||
onClick={handleVoiceClick}
|
||||
disabled={disabled}
|
||||
className={`flex items-center justify-center rounded-full p-2.5 transition-all duration-200 md:p-3 ${
|
||||
isListening
|
||||
? 'bg-red-500 text-white hover:bg-red-600'
|
||||
: 'bg-black text-white hover:bg-zinc-700'
|
||||
} ${disabled ? 'cursor-not-allowed opacity-50' : 'cursor-pointer'}`}
|
||||
whileHover={{ scale: 1.05 }}
|
||||
whileTap={{ scale: 0.95 }}
|
||||
title='Start voice conversation'
|
||||
>
|
||||
<Mic size={16} className='md:hidden' />
|
||||
<Mic size={18} className='hidden md:block' />
|
||||
</m.button>
|
||||
</LazyMotion>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -1,503 +0,0 @@
|
||||
'use client'
|
||||
|
||||
import { useCallback, useEffect, useRef } from 'react'
|
||||
import { createLogger } from '@sim/logger'
|
||||
import * as THREE from 'three'
|
||||
|
||||
const logger = createLogger('Particles')
|
||||
|
||||
interface ShaderUniforms {
|
||||
u_time: { type: string; value: number }
|
||||
u_frequency: { type: string; value: number }
|
||||
u_red: { type: string; value: number }
|
||||
u_green: { type: string; value: number }
|
||||
u_blue: { type: string; value: number }
|
||||
}
|
||||
|
||||
interface ParticlesProps {
|
||||
audioLevels: number[]
|
||||
isListening: boolean
|
||||
isPlayingAudio: boolean
|
||||
isStreaming: boolean
|
||||
isMuted: boolean
|
||||
isProcessingInterruption?: boolean
|
||||
className?: string
|
||||
}
|
||||
|
||||
class SimpleBloomComposer {
|
||||
private renderer: THREE.WebGLRenderer
|
||||
private scene: THREE.Scene
|
||||
private camera: THREE.Camera
|
||||
private bloomScene: THREE.Scene
|
||||
private bloomMaterial: THREE.ShaderMaterial
|
||||
private renderTarget: THREE.WebGLRenderTarget
|
||||
private quad: THREE.Mesh
|
||||
|
||||
constructor(renderer: THREE.WebGLRenderer, scene: THREE.Scene, camera: THREE.Camera) {
|
||||
this.renderer = renderer
|
||||
this.scene = scene
|
||||
this.camera = camera
|
||||
|
||||
this.bloomScene = new THREE.Scene()
|
||||
|
||||
this.renderTarget = new THREE.WebGLRenderTarget(
|
||||
renderer.domElement.width,
|
||||
renderer.domElement.height,
|
||||
{
|
||||
minFilter: THREE.LinearFilter,
|
||||
magFilter: THREE.LinearFilter,
|
||||
format: THREE.RGBAFormat,
|
||||
}
|
||||
)
|
||||
|
||||
this.bloomMaterial = new THREE.ShaderMaterial({
|
||||
uniforms: {
|
||||
tDiffuse: { value: null },
|
||||
strength: { value: 1.5 },
|
||||
threshold: { value: 0.3 },
|
||||
radius: { value: 0.8 },
|
||||
},
|
||||
vertexShader: `
|
||||
varying vec2 vUv;
|
||||
void main() {
|
||||
vUv = uv;
|
||||
gl_Position = projectionMatrix * modelViewMatrix * vec4(position, 1.0);
|
||||
}
|
||||
`,
|
||||
fragmentShader: `
|
||||
uniform sampler2D tDiffuse;
|
||||
uniform float strength;
|
||||
uniform float threshold;
|
||||
uniform float radius;
|
||||
varying vec2 vUv;
|
||||
|
||||
void main() {
|
||||
vec4 color = texture2D(tDiffuse, vUv);
|
||||
|
||||
// Simple bloom effect
|
||||
float brightness = dot(color.rgb, vec3(0.299, 0.587, 0.114));
|
||||
if (brightness > threshold) {
|
||||
color.rgb *= strength;
|
||||
}
|
||||
|
||||
gl_FragColor = color;
|
||||
}
|
||||
`,
|
||||
})
|
||||
|
||||
const geometry = new THREE.PlaneGeometry(2, 2)
|
||||
this.quad = new THREE.Mesh(geometry, this.bloomMaterial)
|
||||
this.bloomScene.add(this.quad)
|
||||
}
|
||||
|
||||
render() {
|
||||
this.renderer.setRenderTarget(this.renderTarget)
|
||||
this.renderer.render(this.scene, this.camera)
|
||||
|
||||
this.bloomMaterial.uniforms.tDiffuse.value = this.renderTarget.texture
|
||||
this.renderer.setRenderTarget(null)
|
||||
this.renderer.render(this.bloomScene, new THREE.OrthographicCamera(-1, 1, 1, -1, 0, 1))
|
||||
}
|
||||
|
||||
setSize(width: number, height: number) {
|
||||
this.renderTarget.setSize(width, height)
|
||||
}
|
||||
|
||||
dispose() {
|
||||
this.renderTarget.dispose()
|
||||
this.bloomMaterial.dispose()
|
||||
}
|
||||
}
|
||||
|
||||
const vertexShader = `
|
||||
vec3 mod289(vec3 x)
|
||||
{
|
||||
return x - floor(x * (1.0 / 289.0)) * 289.0;
|
||||
}
|
||||
|
||||
vec4 mod289(vec4 x)
|
||||
{
|
||||
return x - floor(x * (1.0 / 289.0)) * 289.0;
|
||||
}
|
||||
|
||||
vec4 permute(vec4 x)
|
||||
{
|
||||
return mod289(((x*34.0)+10.0)*x);
|
||||
}
|
||||
|
||||
vec4 taylorInvSqrt(vec4 r)
|
||||
{
|
||||
return 1.79284291400159 - 0.85373472095314 * r;
|
||||
}
|
||||
|
||||
vec3 fade(vec3 t) {
|
||||
return t*t*t*(t*(t*6.0-15.0)+10.0);
|
||||
}
|
||||
|
||||
float pnoise(vec3 P, vec3 rep)
|
||||
{
|
||||
vec3 Pi0 = mod(floor(P), rep); // Integer part, modulo period
|
||||
vec3 Pi1 = mod(Pi0 + vec3(1.0), rep); // Integer part + 1, mod period
|
||||
Pi0 = mod289(Pi0);
|
||||
Pi1 = mod289(Pi1);
|
||||
vec3 Pf0 = fract(P); // Fractional part for interpolation
|
||||
vec3 Pf1 = Pf0 - vec3(1.0); // Fractional part - 1.0
|
||||
vec4 ix = vec4(Pi0.x, Pi1.x, Pi0.x, Pi1.x);
|
||||
vec4 iy = vec4(Pi0.yy, Pi1.yy);
|
||||
vec4 iz0 = Pi0.zzzz;
|
||||
vec4 iz1 = Pi1.zzzz;
|
||||
|
||||
vec4 ixy = permute(permute(ix) + iy);
|
||||
vec4 ixy0 = permute(ixy + iz0);
|
||||
vec4 ixy1 = permute(ixy + iz1);
|
||||
|
||||
vec4 gx0 = ixy0 * (1.0 / 7.0);
|
||||
vec4 gy0 = fract(floor(gx0) * (1.0 / 7.0)) - 0.5;
|
||||
gx0 = fract(gx0);
|
||||
vec4 gz0 = vec4(0.5) - abs(gx0) - abs(gy0);
|
||||
vec4 sz0 = step(gz0, vec4(0.0));
|
||||
gx0 -= sz0 * (step(0.0, gx0) - 0.5);
|
||||
gy0 -= sz0 * (step(0.0, gy0) - 0.5);
|
||||
|
||||
vec4 gx1 = ixy1 * (1.0 / 7.0);
|
||||
vec4 gy1 = fract(floor(gx1) * (1.0 / 7.0)) - 0.5;
|
||||
gx1 = fract(gx1);
|
||||
vec4 gz1 = vec4(0.5) - abs(gx1) - abs(gy1);
|
||||
vec4 sz1 = step(gz1, vec4(0.0));
|
||||
gx1 -= sz1 * (step(0.0, gx1) - 0.5);
|
||||
gy1 -= sz1 * (step(0.0, gy1) - 0.5);
|
||||
|
||||
vec3 g000 = vec3(gx0.x,gy0.x,gz0.x);
|
||||
vec3 g100 = vec3(gx0.y,gy0.y,gz0.y);
|
||||
vec3 g010 = vec3(gx0.z,gy0.z,gz0.z);
|
||||
vec3 g110 = vec3(gx0.w,gy0.w,gz0.w);
|
||||
vec3 g001 = vec3(gx1.x,gy1.x,gz1.x);
|
||||
vec3 g101 = vec3(gx1.y,gy1.y,gz1.y);
|
||||
vec3 g011 = vec3(gx1.z,gy1.z,gz1.z);
|
||||
vec3 g111 = vec3(gx1.w,gy1.w,gz1.w);
|
||||
|
||||
vec4 norm0 = taylorInvSqrt(vec4(dot(g000, g000), dot(g010, g010), dot(g100, g100), dot(g110, g110)));
|
||||
g000 *= norm0.x;
|
||||
g010 *= norm0.y;
|
||||
g100 *= norm0.z;
|
||||
g110 *= norm0.w;
|
||||
vec4 norm1 = taylorInvSqrt(vec4(dot(g001, g001), dot(g011, g011), dot(g101, g101), dot(g111, g111)));
|
||||
g001 *= norm1.x;
|
||||
g011 *= norm1.y;
|
||||
g101 *= norm1.z;
|
||||
g111 *= norm1.w;
|
||||
|
||||
float n000 = dot(g000, Pf0);
|
||||
float n100 = dot(g100, vec3(Pf1.x, Pf0.yz));
|
||||
float n010 = dot(g010, vec3(Pf0.x, Pf1.y, Pf0.z));
|
||||
float n110 = dot(g110, vec3(Pf1.xy, Pf0.z));
|
||||
float n001 = dot(g001, vec3(Pf0.xy, Pf1.z));
|
||||
float n101 = dot(g101, vec3(Pf1.x, Pf0.y, Pf1.z));
|
||||
float n011 = dot(g011, vec3(Pf0.x, Pf1.yz));
|
||||
float n111 = dot(g111, Pf1);
|
||||
|
||||
vec3 fade_xyz = fade(Pf0);
|
||||
vec4 n_z = mix(vec4(n000, n100, n010, n110), vec4(n001, n101, n011, n111), fade_xyz.z);
|
||||
vec2 n_yz = mix(n_z.xy, n_z.zw, fade_xyz.y);
|
||||
float n_xyz = mix(n_yz.x, n_yz.y, fade_xyz.x);
|
||||
return 2.2 * n_xyz;
|
||||
}
|
||||
|
||||
uniform float u_time;
|
||||
uniform float u_frequency;
|
||||
|
||||
void main() {
|
||||
float noise = 5. * pnoise(position + u_time, vec3(10.));
|
||||
|
||||
float displacement = (u_frequency / 30.) * (noise / 10.);
|
||||
|
||||
vec3 newPosition = position + normal * displacement;
|
||||
gl_Position = projectionMatrix * modelViewMatrix * vec4(newPosition, 1.0);
|
||||
}
|
||||
`
|
||||
|
||||
const fragmentShader = `
|
||||
uniform float u_red;
|
||||
uniform float u_blue;
|
||||
uniform float u_green;
|
||||
|
||||
void main() {
|
||||
gl_FragColor = vec4(vec3(u_red, u_green, u_blue), 1.0);
|
||||
}
|
||||
`
|
||||
|
||||
export function ParticlesVisualization({
|
||||
audioLevels,
|
||||
isListening,
|
||||
isPlayingAudio,
|
||||
isStreaming,
|
||||
isMuted,
|
||||
isProcessingInterruption,
|
||||
className,
|
||||
}: ParticlesProps) {
|
||||
const containerRef = useRef<HTMLDivElement>(null)
|
||||
const rendererRef = useRef<THREE.WebGLRenderer | null>(null)
|
||||
const sceneRef = useRef<THREE.Scene | null>(null)
|
||||
const cameraRef = useRef<THREE.PerspectiveCamera | null>(null)
|
||||
const meshRef = useRef<THREE.Mesh | null>(null)
|
||||
const uniformsRef = useRef<ShaderUniforms | null>(null)
|
||||
const clockRef = useRef<THREE.Clock | null>(null)
|
||||
const bloomComposerRef = useRef<SimpleBloomComposer | null>(null)
|
||||
const animationFrameRef = useRef<number>(0)
|
||||
const mouseRef = useRef({ x: 0, y: 0 })
|
||||
const isInitializedRef = useRef(false)
|
||||
|
||||
const cleanup = useCallback(() => {
|
||||
if (animationFrameRef.current) {
|
||||
cancelAnimationFrame(animationFrameRef.current)
|
||||
animationFrameRef.current = 0
|
||||
}
|
||||
|
||||
if (bloomComposerRef.current) {
|
||||
bloomComposerRef.current.dispose()
|
||||
bloomComposerRef.current = null
|
||||
}
|
||||
|
||||
if (rendererRef.current) {
|
||||
if (rendererRef.current.domElement?.parentNode) {
|
||||
rendererRef.current.domElement.parentNode.removeChild(rendererRef.current.domElement)
|
||||
}
|
||||
rendererRef.current.dispose()
|
||||
rendererRef.current = null
|
||||
}
|
||||
|
||||
sceneRef.current = null
|
||||
cameraRef.current = null
|
||||
meshRef.current = null
|
||||
uniformsRef.current = null
|
||||
clockRef.current = null
|
||||
isInitializedRef.current = false
|
||||
}, [])
|
||||
|
||||
useEffect(() => {
|
||||
if (!containerRef.current || isInitializedRef.current) return
|
||||
|
||||
const container = containerRef.current
|
||||
const containerWidth = 400
|
||||
const containerHeight = 400
|
||||
|
||||
isInitializedRef.current = true
|
||||
|
||||
while (container.firstChild) {
|
||||
container.removeChild(container.firstChild)
|
||||
}
|
||||
|
||||
const renderer = new THREE.WebGLRenderer({ antialias: true, alpha: true })
|
||||
renderer.setSize(containerWidth, containerHeight)
|
||||
renderer.setClearColor(0x000000, 0)
|
||||
renderer.outputColorSpace = THREE.SRGBColorSpace
|
||||
container.appendChild(renderer.domElement)
|
||||
rendererRef.current = renderer
|
||||
|
||||
const scene = new THREE.Scene()
|
||||
sceneRef.current = scene
|
||||
|
||||
const camera = new THREE.PerspectiveCamera(45, containerWidth / containerHeight, 0.1, 1000)
|
||||
camera.position.set(0, -2, 14)
|
||||
camera.lookAt(0, 0, 0)
|
||||
cameraRef.current = camera
|
||||
|
||||
const uniforms = {
|
||||
u_time: { type: 'f', value: 0.0 },
|
||||
u_frequency: { type: 'f', value: 0.0 },
|
||||
u_red: { type: 'f', value: 0.8 },
|
||||
u_green: { type: 'f', value: 0.6 },
|
||||
u_blue: { type: 'f', value: 1.0 },
|
||||
}
|
||||
uniformsRef.current = uniforms
|
||||
|
||||
let mat: THREE.Material
|
||||
try {
|
||||
mat = new THREE.ShaderMaterial({
|
||||
uniforms,
|
||||
vertexShader,
|
||||
fragmentShader,
|
||||
})
|
||||
} catch (error) {
|
||||
logger.error('❌ Shader compilation error, using fallback material:', error)
|
||||
mat = new THREE.MeshBasicMaterial({
|
||||
color: 0xb794f6, // Light purple color
|
||||
wireframe: true,
|
||||
})
|
||||
}
|
||||
|
||||
const geo = new THREE.IcosahedronGeometry(4, 30) // Match tutorial: radius 4, subdivisions 30
|
||||
const mesh = new THREE.Mesh(geo, mat)
|
||||
|
||||
if (mat instanceof THREE.ShaderMaterial || mat instanceof THREE.MeshBasicMaterial) {
|
||||
mat.wireframe = true
|
||||
}
|
||||
|
||||
scene.add(mesh)
|
||||
meshRef.current = mesh
|
||||
|
||||
const bloomComposer = new SimpleBloomComposer(renderer, scene, camera)
|
||||
bloomComposerRef.current = bloomComposer
|
||||
|
||||
const clock = new THREE.Clock()
|
||||
clockRef.current = clock
|
||||
|
||||
const handleMouseMove = (e: MouseEvent) => {
|
||||
const rect = container.getBoundingClientRect()
|
||||
const windowHalfX = containerWidth / 2
|
||||
const windowHalfY = containerHeight / 2
|
||||
mouseRef.current.x = (e.clientX - rect.left - windowHalfX) / 100
|
||||
mouseRef.current.y = (e.clientY - rect.top - windowHalfY) / 100
|
||||
}
|
||||
|
||||
container.addEventListener('mousemove', handleMouseMove)
|
||||
|
||||
const updateCameraPosition = () => {
|
||||
if (!camera || !scene) return
|
||||
camera.position.x += (mouseRef.current.x - camera.position.x) * 0.05
|
||||
camera.position.y += (-mouseRef.current.y - camera.position.y) * 0.5
|
||||
camera.lookAt(scene.position)
|
||||
}
|
||||
|
||||
const calculateAudioIntensity = (elapsedTime: number, avgLevel: number) => {
|
||||
const baselineIntensity = 8 + Math.sin(elapsedTime * 0.5) * 3
|
||||
let audioIntensity = baselineIntensity
|
||||
|
||||
if (isMuted) {
|
||||
// When muted, only show minimal baseline animation
|
||||
audioIntensity = baselineIntensity * 0.2
|
||||
} else if (isProcessingInterruption) {
|
||||
// Special pulsing effect during interruption processing
|
||||
audioIntensity = 35 + Math.sin(elapsedTime * 4) * 10
|
||||
} else if (isPlayingAudio) {
|
||||
// Strong animation when AI is speaking - use simulated levels + enhancement
|
||||
const aiIntensity = 60 + Math.sin(elapsedTime * 3) * 20
|
||||
audioIntensity = Math.max(avgLevel * 0.8, aiIntensity)
|
||||
} else if (isStreaming) {
|
||||
// Pulsing animation when AI is thinking/streaming
|
||||
audioIntensity = 40 + Math.sin(elapsedTime * 2) * 15
|
||||
} else if (isListening && avgLevel > 0) {
|
||||
// Scale user input more dramatically for better visual feedback
|
||||
const userVoiceIntensity = avgLevel * 2.5 // Amplify user voice significantly
|
||||
audioIntensity = Math.max(userVoiceIntensity, baselineIntensity * 1.5)
|
||||
|
||||
// Add some dynamic variation based on audio levels
|
||||
const variationFactor = Math.min(avgLevel / 20, 1) // Cap at reasonable level
|
||||
audioIntensity += Math.sin(elapsedTime * 8) * (10 * variationFactor)
|
||||
} else {
|
||||
// Idle state - subtle breathing animation
|
||||
audioIntensity = baselineIntensity
|
||||
}
|
||||
|
||||
// Clamp to reasonable range
|
||||
audioIntensity = Math.max(audioIntensity, 3) // Never completely still
|
||||
audioIntensity = Math.min(audioIntensity, 120) // Prevent excessive animation
|
||||
|
||||
return audioIntensity
|
||||
}
|
||||
|
||||
const updateShaderColors = (
|
||||
uniforms: ShaderUniforms,
|
||||
elapsedTime: number,
|
||||
avgLevel: number
|
||||
) => {
|
||||
if (isMuted) {
|
||||
// Muted: dim purple-gray
|
||||
uniforms.u_red.value = 0.25
|
||||
uniforms.u_green.value = 0.1
|
||||
uniforms.u_blue.value = 0.5
|
||||
} else if (isProcessingInterruption) {
|
||||
// Interruption: bright purple
|
||||
uniforms.u_red.value = 0.6
|
||||
uniforms.u_green.value = 0.2
|
||||
uniforms.u_blue.value = 0.9
|
||||
} else if (isPlayingAudio) {
|
||||
// AI speaking: brand purple (#701FFC)
|
||||
uniforms.u_red.value = 0.44
|
||||
uniforms.u_green.value = 0.12
|
||||
uniforms.u_blue.value = 0.99
|
||||
} else if (isListening && avgLevel > 10) {
|
||||
// User speaking: lighter purple with intensity-based variation
|
||||
const intensity = Math.min(avgLevel / 50, 1)
|
||||
uniforms.u_red.value = 0.35 + intensity * 0.15
|
||||
uniforms.u_green.value = 0.1 + intensity * 0.1
|
||||
uniforms.u_blue.value = 0.8 + intensity * 0.2
|
||||
} else if (isStreaming) {
|
||||
// AI thinking: pulsing brand purple
|
||||
const pulse = (Math.sin(elapsedTime * 2) + 1) / 2
|
||||
uniforms.u_red.value = 0.35 + pulse * 0.15
|
||||
uniforms.u_green.value = 0.08 + pulse * 0.08
|
||||
uniforms.u_blue.value = 0.95 + pulse * 0.05
|
||||
} else {
|
||||
// Default idle: soft brand purple
|
||||
uniforms.u_red.value = 0.4
|
||||
uniforms.u_green.value = 0.15
|
||||
uniforms.u_blue.value = 0.9
|
||||
}
|
||||
}
|
||||
|
||||
const animate = () => {
|
||||
if (!camera || !clock || !scene || !bloomComposer || !isInitializedRef.current) return
|
||||
|
||||
updateCameraPosition()
|
||||
|
||||
if (uniforms) {
|
||||
const elapsedTime = clock.getElapsedTime()
|
||||
const avgLevel = audioLevels.reduce((sum, level) => sum + level, 0) / audioLevels.length
|
||||
|
||||
uniforms.u_time.value = elapsedTime
|
||||
|
||||
const audioIntensity = calculateAudioIntensity(elapsedTime, avgLevel)
|
||||
updateShaderColors(uniforms, elapsedTime, avgLevel)
|
||||
|
||||
uniforms.u_frequency.value = audioIntensity
|
||||
}
|
||||
|
||||
bloomComposer.render()
|
||||
animationFrameRef.current = requestAnimationFrame(animate)
|
||||
}
|
||||
|
||||
animate()
|
||||
|
||||
return () => {
|
||||
container.removeEventListener('mousemove', handleMouseMove)
|
||||
cleanup()
|
||||
}
|
||||
}, [])
|
||||
|
||||
useEffect(() => {
|
||||
const handleResize = () => {
|
||||
if (
|
||||
rendererRef.current &&
|
||||
cameraRef.current &&
|
||||
bloomComposerRef.current &&
|
||||
containerRef.current
|
||||
) {
|
||||
const containerWidth = 400
|
||||
const containerHeight = 400
|
||||
|
||||
cameraRef.current.aspect = containerWidth / containerHeight
|
||||
cameraRef.current.updateProjectionMatrix()
|
||||
rendererRef.current.setSize(containerWidth, containerHeight)
|
||||
bloomComposerRef.current.setSize(containerWidth, containerHeight)
|
||||
}
|
||||
}
|
||||
|
||||
window.addEventListener('resize', handleResize)
|
||||
return () => window.removeEventListener('resize', handleResize)
|
||||
}, [])
|
||||
|
||||
return (
|
||||
<div
|
||||
ref={containerRef}
|
||||
className={className}
|
||||
style={{
|
||||
width: '400px',
|
||||
height: '400px',
|
||||
display: 'flex',
|
||||
justifyContent: 'center',
|
||||
alignItems: 'center',
|
||||
}}
|
||||
/>
|
||||
)
|
||||
}
|
||||
@@ -1,582 +0,0 @@
|
||||
'use client'
|
||||
|
||||
import { type RefObject, useCallback, useEffect, useRef, useState } from 'react'
|
||||
import { Button, cn } from '@sim/emcn'
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { Mic, MicOff, Phone } from 'lucide-react'
|
||||
import dynamic from 'next/dynamic'
|
||||
import { requestJson } from '@/lib/api/client/request'
|
||||
import { speechTokenContract } from '@/lib/api/contracts/media/speech'
|
||||
import { arrayBufferToBase64, floatTo16BitPCM } from '@/lib/speech/audio'
|
||||
import {
|
||||
CHUNK_SEND_INTERVAL_MS,
|
||||
ELEVENLABS_WS_URL,
|
||||
MAX_CHAT_SESSION_MS,
|
||||
SAMPLE_RATE,
|
||||
} from '@/lib/speech/config'
|
||||
import { DesktopTitleBarLane } from '@/app/_shell/desktop-title-bar'
|
||||
|
||||
const ParticlesVisualization = dynamic(
|
||||
() =>
|
||||
import('@/app/(interfaces)/chat/components/voice-interface/components/particles').then(
|
||||
(mod) => mod.ParticlesVisualization
|
||||
),
|
||||
{ ssr: false }
|
||||
)
|
||||
|
||||
const logger = createLogger('VoiceInterface')
|
||||
|
||||
interface VoiceInterfaceProps {
|
||||
onCallEnd?: () => void
|
||||
onVoiceTranscript?: (transcript: string) => void
|
||||
onVoiceStart?: () => void
|
||||
onVoiceEnd?: () => void
|
||||
onInterrupt?: () => void
|
||||
isStreaming?: boolean
|
||||
isPlayingAudio?: boolean
|
||||
audioContextRef?: RefObject<AudioContext | null>
|
||||
messages?: Array<{ content: string; type: 'user' | 'assistant' }>
|
||||
className?: string
|
||||
chatId?: string
|
||||
}
|
||||
|
||||
const EMPTY_MESSAGES: Array<{ content: string; type: 'user' | 'assistant' }> = []
|
||||
|
||||
export function VoiceInterface({
|
||||
onCallEnd,
|
||||
onVoiceTranscript,
|
||||
onVoiceStart,
|
||||
onVoiceEnd,
|
||||
onInterrupt,
|
||||
isStreaming = false,
|
||||
isPlayingAudio = false,
|
||||
audioContextRef: sharedAudioContextRef,
|
||||
messages = EMPTY_MESSAGES,
|
||||
className,
|
||||
chatId,
|
||||
}: VoiceInterfaceProps) {
|
||||
const [state, setState] = useState<'idle' | 'listening' | 'agent_speaking'>('idle')
|
||||
const [isInitialized, setIsInitialized] = useState(false)
|
||||
const [isMuted, setIsMuted] = useState(false)
|
||||
const [audioLevels, setAudioLevels] = useState<number[]>(() => new Array(200).fill(0))
|
||||
const permissionStatusRef = useRef<'prompt' | 'granted' | 'denied'>('prompt')
|
||||
const [currentTranscript, setCurrentTranscript] = useState('')
|
||||
|
||||
const currentStateRef = useRef<'idle' | 'listening' | 'agent_speaking'>('idle')
|
||||
const isCallEndedRef = useRef(false)
|
||||
|
||||
const updateState = useCallback((next: 'idle' | 'listening' | 'agent_speaking') => {
|
||||
setState(next)
|
||||
currentStateRef.current = next
|
||||
}, [])
|
||||
|
||||
const mediaStreamRef = useRef<MediaStream | null>(null)
|
||||
const audioContextRef = useRef<AudioContext | null>(null)
|
||||
const analyserRef = useRef<AnalyserNode | null>(null)
|
||||
const animationFrameRef = useRef<number | null>(null)
|
||||
const isMutedRef = useRef(false)
|
||||
|
||||
const wsRef = useRef<WebSocket | null>(null)
|
||||
const processorRef = useRef<ScriptProcessorNode | null>(null)
|
||||
const pcmBufferRef = useRef<Float32Array[]>([])
|
||||
const sendIntervalRef = useRef<ReturnType<typeof setInterval> | null>(null)
|
||||
const sessionTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null)
|
||||
const committedTextRef = useRef('')
|
||||
const lastPartialRef = useRef('')
|
||||
const onVoiceTranscriptRef = useRef(onVoiceTranscript)
|
||||
|
||||
onVoiceTranscriptRef.current = onVoiceTranscript
|
||||
|
||||
const updateIsMuted = useCallback((next: boolean) => {
|
||||
setIsMuted(next)
|
||||
isMutedRef.current = next
|
||||
}, [])
|
||||
|
||||
const stopSendingAudio = useCallback(() => {
|
||||
if (sessionTimerRef.current) {
|
||||
clearTimeout(sessionTimerRef.current)
|
||||
sessionTimerRef.current = null
|
||||
}
|
||||
if (sendIntervalRef.current) {
|
||||
clearInterval(sendIntervalRef.current)
|
||||
sendIntervalRef.current = null
|
||||
}
|
||||
pcmBufferRef.current = []
|
||||
}, [])
|
||||
|
||||
const flushAudioBuffer = useCallback(() => {
|
||||
const ws = wsRef.current
|
||||
if (!ws || ws.readyState !== WebSocket.OPEN) return
|
||||
|
||||
const chunks = pcmBufferRef.current
|
||||
if (chunks.length === 0) return
|
||||
pcmBufferRef.current = []
|
||||
|
||||
let totalLength = 0
|
||||
for (const chunk of chunks) totalLength += chunk.length
|
||||
const merged = new Float32Array(totalLength)
|
||||
let offset = 0
|
||||
for (const chunk of chunks) {
|
||||
merged.set(chunk, offset)
|
||||
offset += chunk.length
|
||||
}
|
||||
|
||||
const pcm16 = floatTo16BitPCM(merged)
|
||||
|
||||
ws.send(
|
||||
JSON.stringify({
|
||||
message_type: 'input_audio_chunk',
|
||||
audio_base_64: arrayBufferToBase64(pcm16),
|
||||
sample_rate: SAMPLE_RATE,
|
||||
commit: false,
|
||||
})
|
||||
)
|
||||
}, [])
|
||||
|
||||
const startSendingAudio = useCallback(() => {
|
||||
if (sendIntervalRef.current) return
|
||||
pcmBufferRef.current = []
|
||||
sendIntervalRef.current = setInterval(flushAudioBuffer, CHUNK_SEND_INTERVAL_MS)
|
||||
}, [flushAudioBuffer])
|
||||
|
||||
const closeWebSocket = useCallback(() => {
|
||||
stopSendingAudio()
|
||||
if (wsRef.current) {
|
||||
if (
|
||||
wsRef.current.readyState === WebSocket.OPEN ||
|
||||
wsRef.current.readyState === WebSocket.CONNECTING
|
||||
) {
|
||||
wsRef.current.close()
|
||||
}
|
||||
wsRef.current = null
|
||||
}
|
||||
}, [stopSendingAudio])
|
||||
|
||||
const connectWebSocket = useCallback(async (): Promise<boolean> => {
|
||||
try {
|
||||
let tokenData: Awaited<ReturnType<typeof requestJson<typeof speechTokenContract>>>
|
||||
try {
|
||||
tokenData = await requestJson(speechTokenContract, {
|
||||
body: chatId ? { chatId } : {},
|
||||
})
|
||||
} catch (err) {
|
||||
logger.error('Failed to get STT token', err)
|
||||
return false
|
||||
}
|
||||
|
||||
const token = typeof tokenData.token === 'string' ? tokenData.token : undefined
|
||||
if (!token) {
|
||||
logger.error('STT token missing from response')
|
||||
return false
|
||||
}
|
||||
|
||||
const params = new URLSearchParams({
|
||||
token,
|
||||
model_id: 'scribe_v2_realtime',
|
||||
audio_format: 'pcm_16000',
|
||||
commit_strategy: 'vad',
|
||||
vad_silence_threshold_secs: '1.0',
|
||||
})
|
||||
|
||||
const ws = new WebSocket(`${ELEVENLABS_WS_URL}?${params.toString()}`)
|
||||
wsRef.current = ws
|
||||
committedTextRef.current = ''
|
||||
|
||||
return new Promise<boolean>((resolve) => {
|
||||
ws.onopen = () => resolve(true)
|
||||
ws.onerror = () => {
|
||||
logger.error('STT WebSocket connection error')
|
||||
resolve(false)
|
||||
}
|
||||
|
||||
ws.onmessage = (event) => {
|
||||
if (isCallEndedRef.current) return
|
||||
|
||||
try {
|
||||
const msg = JSON.parse(event.data)
|
||||
|
||||
if (msg.message_type === 'partial_transcript') {
|
||||
if (msg.text) {
|
||||
lastPartialRef.current = msg.text
|
||||
setCurrentTranscript(msg.text)
|
||||
}
|
||||
} else if (
|
||||
msg.message_type === 'committed_transcript' ||
|
||||
msg.message_type === 'committed_transcript_with_timestamps'
|
||||
) {
|
||||
const finalText = msg.text || lastPartialRef.current
|
||||
lastPartialRef.current = ''
|
||||
if (finalText) {
|
||||
committedTextRef.current = committedTextRef.current
|
||||
? `${committedTextRef.current} ${finalText}`
|
||||
: finalText
|
||||
setCurrentTranscript('')
|
||||
onVoiceTranscriptRef.current?.(finalText)
|
||||
}
|
||||
} else if (
|
||||
msg.message_type === 'error' ||
|
||||
msg.message_type === 'auth_error' ||
|
||||
msg.message_type === 'quota_exceeded'
|
||||
) {
|
||||
logger.error('ElevenLabs STT error', { type: msg.message_type, error: msg.error })
|
||||
}
|
||||
} catch {
|
||||
// Ignore non-JSON messages
|
||||
}
|
||||
}
|
||||
|
||||
ws.onclose = () => {
|
||||
wsRef.current = null
|
||||
if (currentStateRef.current === 'listening' && !isCallEndedRef.current) {
|
||||
stopSendingAudio()
|
||||
updateState('idle')
|
||||
}
|
||||
}
|
||||
})
|
||||
} catch (error) {
|
||||
logger.error('Failed to connect STT WebSocket', error)
|
||||
return false
|
||||
}
|
||||
}, [chatId])
|
||||
|
||||
const setupAudioPipeline = useCallback(async () => {
|
||||
try {
|
||||
const stream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: {
|
||||
echoCancellation: true,
|
||||
noiseSuppression: true,
|
||||
autoGainControl: true,
|
||||
channelCount: 1,
|
||||
sampleRate: SAMPLE_RATE,
|
||||
},
|
||||
})
|
||||
|
||||
permissionStatusRef.current = 'granted'
|
||||
mediaStreamRef.current = stream
|
||||
|
||||
const ac = new AudioContext({ sampleRate: SAMPLE_RATE })
|
||||
audioContextRef.current = ac
|
||||
|
||||
if (ac.state === 'suspended') {
|
||||
await ac.resume()
|
||||
}
|
||||
|
||||
const source = ac.createMediaStreamSource(stream)
|
||||
|
||||
const analyser = ac.createAnalyser()
|
||||
analyser.fftSize = 256
|
||||
analyser.smoothingTimeConstant = 0.8
|
||||
source.connect(analyser)
|
||||
analyserRef.current = analyser
|
||||
|
||||
const processor = ac.createScriptProcessor(4096, 1, 1)
|
||||
processor.onaudioprocess = (e) => {
|
||||
if (!isMutedRef.current && currentStateRef.current === 'listening') {
|
||||
pcmBufferRef.current.push(new Float32Array(e.inputBuffer.getChannelData(0)))
|
||||
}
|
||||
}
|
||||
source.connect(processor)
|
||||
processor.connect(ac.destination)
|
||||
processorRef.current = processor
|
||||
|
||||
const updateVisualization = () => {
|
||||
if (!analyserRef.current) return
|
||||
const bufferLength = analyserRef.current.frequencyBinCount
|
||||
const dataArray = new Uint8Array(bufferLength)
|
||||
analyserRef.current.getByteFrequencyData(dataArray)
|
||||
|
||||
const levels = []
|
||||
for (let i = 0; i < 200; i++) {
|
||||
const dataIndex = Math.floor((i / 200) * bufferLength)
|
||||
const value = dataArray[dataIndex] || 0
|
||||
levels.push((value / 255) * 100)
|
||||
}
|
||||
|
||||
setAudioLevels(levels)
|
||||
animationFrameRef.current = requestAnimationFrame(updateVisualization)
|
||||
}
|
||||
updateVisualization()
|
||||
|
||||
return true
|
||||
} catch (error) {
|
||||
logger.error('Error setting up audio pipeline:', error)
|
||||
permissionStatusRef.current = 'denied'
|
||||
return false
|
||||
}
|
||||
}, [])
|
||||
|
||||
const startListening = useCallback(async () => {
|
||||
if (currentStateRef.current !== 'idle' || isMutedRef.current || isCallEndedRef.current) return
|
||||
|
||||
if (!wsRef.current || wsRef.current.readyState !== WebSocket.OPEN) {
|
||||
const connected = await connectWebSocket()
|
||||
if (!connected || isCallEndedRef.current) return
|
||||
}
|
||||
|
||||
updateState('listening')
|
||||
setCurrentTranscript('')
|
||||
startSendingAudio()
|
||||
|
||||
sessionTimerRef.current = setTimeout(() => {
|
||||
logger.info('Voice session reached max duration, stopping')
|
||||
stopSendingAudio()
|
||||
closeWebSocket()
|
||||
updateState('idle')
|
||||
}, MAX_CHAT_SESSION_MS)
|
||||
}, [connectWebSocket, updateState, startSendingAudio, stopSendingAudio, closeWebSocket])
|
||||
|
||||
const stopListening = useCallback(() => {
|
||||
stopSendingAudio()
|
||||
updateState('idle')
|
||||
setCurrentTranscript('')
|
||||
}, [updateState, stopSendingAudio])
|
||||
|
||||
useEffect(() => {
|
||||
if (isPlayingAudio && state === 'listening') {
|
||||
stopSendingAudio()
|
||||
closeWebSocket()
|
||||
updateState('agent_speaking')
|
||||
setCurrentTranscript('')
|
||||
|
||||
updateIsMuted(true)
|
||||
if (mediaStreamRef.current) {
|
||||
mediaStreamRef.current.getAudioTracks().forEach((track) => {
|
||||
track.enabled = false
|
||||
})
|
||||
}
|
||||
} else if (!isPlayingAudio && state === 'agent_speaking') {
|
||||
updateState('idle')
|
||||
setCurrentTranscript('')
|
||||
|
||||
updateIsMuted(false)
|
||||
if (mediaStreamRef.current) {
|
||||
mediaStreamRef.current.getAudioTracks().forEach((track) => {
|
||||
track.enabled = true
|
||||
})
|
||||
}
|
||||
}
|
||||
}, [isPlayingAudio, state, updateState, updateIsMuted, stopSendingAudio, closeWebSocket])
|
||||
|
||||
const handleInterrupt = useCallback(() => {
|
||||
if (state === 'agent_speaking') {
|
||||
onInterrupt?.()
|
||||
|
||||
updateIsMuted(false)
|
||||
if (mediaStreamRef.current) {
|
||||
mediaStreamRef.current.getAudioTracks().forEach((track) => {
|
||||
track.enabled = true
|
||||
})
|
||||
}
|
||||
|
||||
updateState('idle')
|
||||
setCurrentTranscript('')
|
||||
}
|
||||
}, [state, onInterrupt, updateState, updateIsMuted])
|
||||
|
||||
const handleCallEnd = useCallback(() => {
|
||||
isCallEndedRef.current = true
|
||||
|
||||
stopSendingAudio()
|
||||
closeWebSocket()
|
||||
updateState('idle')
|
||||
setCurrentTranscript('')
|
||||
updateIsMuted(false)
|
||||
|
||||
if (processorRef.current) {
|
||||
processorRef.current.disconnect()
|
||||
processorRef.current = null
|
||||
}
|
||||
|
||||
if (mediaStreamRef.current) {
|
||||
mediaStreamRef.current.getTracks().forEach((track) => track.stop())
|
||||
mediaStreamRef.current = null
|
||||
}
|
||||
|
||||
if (audioContextRef.current && audioContextRef.current.state !== 'closed') {
|
||||
audioContextRef.current.close().catch(() => {})
|
||||
audioContextRef.current = null
|
||||
}
|
||||
|
||||
if (animationFrameRef.current) {
|
||||
cancelAnimationFrame(animationFrameRef.current)
|
||||
animationFrameRef.current = null
|
||||
}
|
||||
|
||||
onInterrupt?.()
|
||||
onCallEnd?.()
|
||||
}, [onCallEnd, onInterrupt, updateState, updateIsMuted, stopSendingAudio, closeWebSocket])
|
||||
|
||||
useEffect(() => {
|
||||
const handleKeyDown = (event: KeyboardEvent) => {
|
||||
if (event.code === 'Space') {
|
||||
event.preventDefault()
|
||||
handleInterrupt()
|
||||
}
|
||||
}
|
||||
|
||||
document.addEventListener('keydown', handleKeyDown)
|
||||
return () => document.removeEventListener('keydown', handleKeyDown)
|
||||
}, [handleInterrupt])
|
||||
|
||||
const toggleMute = useCallback(() => {
|
||||
if (state === 'agent_speaking') {
|
||||
handleInterrupt()
|
||||
return
|
||||
}
|
||||
|
||||
const newMutedState = !isMuted
|
||||
updateIsMuted(newMutedState)
|
||||
|
||||
if (mediaStreamRef.current) {
|
||||
mediaStreamRef.current.getAudioTracks().forEach((track) => {
|
||||
track.enabled = !newMutedState
|
||||
})
|
||||
}
|
||||
|
||||
if (newMutedState) {
|
||||
stopListening()
|
||||
} else if (state === 'idle') {
|
||||
startListening()
|
||||
}
|
||||
}, [isMuted, state, handleInterrupt, stopListening, startListening, updateIsMuted])
|
||||
|
||||
useEffect(() => {
|
||||
isCallEndedRef.current = false
|
||||
let cancelled = false
|
||||
|
||||
async function init() {
|
||||
const audioOk = await setupAudioPipeline()
|
||||
if (!audioOk || cancelled) return
|
||||
|
||||
const wsOk = await connectWebSocket()
|
||||
if (!wsOk || cancelled) return
|
||||
|
||||
setIsInitialized(true)
|
||||
}
|
||||
|
||||
init()
|
||||
|
||||
return () => {
|
||||
cancelled = true
|
||||
}
|
||||
}, [setupAudioPipeline, connectWebSocket])
|
||||
|
||||
useEffect(() => {
|
||||
if (isInitialized && !isMuted && state === 'idle') {
|
||||
startListening()
|
||||
}
|
||||
}, [isInitialized, isMuted, state, startListening])
|
||||
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
isCallEndedRef.current = true
|
||||
|
||||
stopSendingAudio()
|
||||
|
||||
if (wsRef.current) {
|
||||
wsRef.current.close()
|
||||
wsRef.current = null
|
||||
}
|
||||
|
||||
if (processorRef.current) {
|
||||
processorRef.current.disconnect()
|
||||
processorRef.current = null
|
||||
}
|
||||
|
||||
if (mediaStreamRef.current) {
|
||||
mediaStreamRef.current.getTracks().forEach((track) => track.stop())
|
||||
mediaStreamRef.current = null
|
||||
}
|
||||
|
||||
if (audioContextRef.current) {
|
||||
audioContextRef.current.close()
|
||||
audioContextRef.current = null
|
||||
}
|
||||
|
||||
if (animationFrameRef.current) {
|
||||
cancelAnimationFrame(animationFrameRef.current)
|
||||
animationFrameRef.current = null
|
||||
}
|
||||
}
|
||||
}, [stopSendingAudio])
|
||||
|
||||
const getStatusText = () => {
|
||||
switch (state) {
|
||||
case 'listening':
|
||||
return 'Listening...'
|
||||
case 'agent_speaking':
|
||||
return 'Press Space or tap to interrupt'
|
||||
default:
|
||||
return isInitialized ? 'Ready' : 'Initializing...'
|
||||
}
|
||||
}
|
||||
|
||||
const getButtonContent = () => {
|
||||
if (state === 'agent_speaking') {
|
||||
return (
|
||||
<svg className='size-6' viewBox='0 0 24 24' fill='currentColor'>
|
||||
<rect x='6' y='6' width='12' height='12' rx='2' />
|
||||
</svg>
|
||||
)
|
||||
}
|
||||
return isMuted ? <MicOff className='size-6' /> : <Mic className='size-6' />
|
||||
}
|
||||
|
||||
return (
|
||||
<div
|
||||
className={cn(
|
||||
'light desktop-title-bar-page fixed inset-0 z-[100] flex flex-col bg-[var(--bg)] text-[var(--text-primary)]',
|
||||
className
|
||||
)}
|
||||
>
|
||||
<DesktopTitleBarLane />
|
||||
<div className='flex flex-1 flex-col items-center justify-center px-8'>
|
||||
<div className='relative mb-16'>
|
||||
<ParticlesVisualization
|
||||
audioLevels={audioLevels}
|
||||
isListening={state === 'listening'}
|
||||
isPlayingAudio={state === 'agent_speaking'}
|
||||
isStreaming={isStreaming}
|
||||
isMuted={isMuted}
|
||||
className='h-80 w-80 md:h-96 md:w-96'
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className='mb-16 flex h-24 items-center justify-center'>
|
||||
{currentTranscript && (
|
||||
<div className='max-w-2xl px-8'>
|
||||
<p className='overflow-hidden text-center text-[var(--text-primary)] text-xl leading-relaxed'>
|
||||
{currentTranscript}
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<p className='mb-8 text-center text-[var(--text-muted)] text-lg'>
|
||||
{getStatusText()}
|
||||
{isMuted && <span className='ml-2 text-[var(--text-muted)] text-sm'>(Muted)</span>}
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div className='px-8 pb-12'>
|
||||
<div className='flex items-center justify-center gap-x-12'>
|
||||
<Button onClick={handleCallEnd} variant='outline' className='size-14 rounded-full p-0'>
|
||||
<Phone className='size-6 rotate-[135deg]' />
|
||||
</Button>
|
||||
|
||||
<Button
|
||||
onClick={toggleMute}
|
||||
variant='outline'
|
||||
disabled={!isInitialized}
|
||||
className={cn(
|
||||
'size-14 rounded-full p-0',
|
||||
isMuted ? 'text-[var(--text-muted)]' : 'text-[var(--text-primary)]'
|
||||
)}
|
||||
>
|
||||
{getButtonContent()}
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -1,2 +1 @@
|
||||
export { useAudioStreaming } from './use-audio-streaming'
|
||||
export { useChatStreaming } from './use-chat-streaming'
|
||||
|
||||
@@ -1,57 +0,0 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { MAX_TTS_TEXT_LENGTH } from '@/lib/api/contracts/media/tts-stream'
|
||||
import { splitForSynthesis } from '@/app/(interfaces)/chat/hooks/use-audio-streaming'
|
||||
|
||||
describe('splitForSynthesis', () => {
|
||||
it('leaves text within the relay cap untouched', () => {
|
||||
expect(splitForSynthesis('Short answer.')).toEqual(['Short answer.'])
|
||||
})
|
||||
|
||||
/**
|
||||
* The caller only sentence-splits on Western `.!?`, so CJK punctuation never
|
||||
* matches and the whole answer arrives as one block. Before splitting, the
|
||||
* relay rejected it and the message played no audio at all.
|
||||
*/
|
||||
it('splits CJK text that never matches the Western sentence split', () => {
|
||||
const text = '这是一个很长的回答。'.repeat(400)
|
||||
expect(text.length).toBeGreaterThan(MAX_TTS_TEXT_LENGTH)
|
||||
|
||||
const chunks = splitForSynthesis(text)
|
||||
|
||||
expect(chunks.length).toBeGreaterThan(1)
|
||||
for (const chunk of chunks) {
|
||||
expect(chunk.length).toBeLessThanOrEqual(MAX_TTS_TEXT_LENGTH)
|
||||
}
|
||||
})
|
||||
|
||||
it('splits a long list that has no terminal punctuation', () => {
|
||||
const text = Array.from({ length: 300 }, (_, i) => `- item number ${i}`).join('\n')
|
||||
expect(text.length).toBeGreaterThan(MAX_TTS_TEXT_LENGTH)
|
||||
|
||||
const chunks = splitForSynthesis(text)
|
||||
|
||||
for (const chunk of chunks) {
|
||||
expect(chunk.length).toBeLessThanOrEqual(MAX_TTS_TEXT_LENGTH)
|
||||
}
|
||||
})
|
||||
|
||||
it('preserves the spoken content across chunks', () => {
|
||||
const text = Array.from({ length: 500 }, (_, i) => `word${i}`).join(' ')
|
||||
|
||||
const chunks = splitForSynthesis(text)
|
||||
|
||||
expect(chunks.join(' ').replace(/\s+/g, ' ')).toBe(text)
|
||||
})
|
||||
|
||||
it('still caps text with no break opportunity at all', () => {
|
||||
const chunks = splitForSynthesis('a'.repeat(MAX_TTS_TEXT_LENGTH * 2 + 5))
|
||||
|
||||
expect(chunks.length).toBe(3)
|
||||
for (const chunk of chunks) {
|
||||
expect(chunk.length).toBeLessThanOrEqual(MAX_TTS_TEXT_LENGTH)
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -1,210 +0,0 @@
|
||||
'use client'
|
||||
|
||||
import { type RefObject, useCallback, useRef, useState } from 'react'
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { DEFAULT_TTS_MODEL_ID, MAX_TTS_TEXT_LENGTH } from '@/lib/api/contracts/media/tts-stream'
|
||||
|
||||
const logger = createLogger('UseAudioStreaming')
|
||||
|
||||
/** Prefer breaking on a boundary this far into the chunk before splitting mid-word. */
|
||||
const MIN_SPLIT_RATIO = 0.6
|
||||
|
||||
/**
|
||||
* Splits text into pieces the TTS relay will accept.
|
||||
*
|
||||
* The caller sentence-splits on Western `.!?` only, so text that never matches
|
||||
* — CJK punctuation, or a list with no terminal punctuation — reaches this hook
|
||||
* as one accumulated block that can exceed the relay's per-request cap. Without
|
||||
* splitting, the relay rejects it and the whole message plays no audio.
|
||||
*/
|
||||
export function splitForSynthesis(text: string, max: number = MAX_TTS_TEXT_LENGTH): string[] {
|
||||
if (text.length <= max) return [text]
|
||||
|
||||
const chunks: string[] = []
|
||||
let rest = text
|
||||
|
||||
while (rest.length > max) {
|
||||
const window = rest.slice(0, max)
|
||||
const boundary = Math.max(
|
||||
window.lastIndexOf(' '),
|
||||
window.lastIndexOf('\n'),
|
||||
window.lastIndexOf('。'),
|
||||
window.lastIndexOf('，'),
|
||||
window.lastIndexOf('、')
|
||||
)
|
||||
const cut = boundary >= max * MIN_SPLIT_RATIO ? boundary + 1 : max
|
||||
const piece = rest.slice(0, cut).trim()
|
||||
if (piece) chunks.push(piece)
|
||||
rest = rest.slice(cut)
|
||||
}
|
||||
|
||||
const tail = rest.trim()
|
||||
if (tail) chunks.push(tail)
|
||||
return chunks
|
||||
}
|
||||
|
||||
declare global {
|
||||
interface Window {
|
||||
webkitAudioContext?: typeof AudioContext
|
||||
}
|
||||
}
|
||||
|
||||
interface AudioStreamingOptions {
|
||||
voiceId: string
|
||||
modelId?: string
|
||||
chatId: string
|
||||
onAudioStart?: () => void
|
||||
onAudioEnd?: () => void
|
||||
onError?: (error: Error) => void
|
||||
}
|
||||
|
||||
interface AudioQueueItem {
|
||||
text: string
|
||||
options: AudioStreamingOptions
|
||||
}
|
||||
|
||||
export function useAudioStreaming(sharedAudioContextRef?: RefObject<AudioContext | null>) {
|
||||
const [isPlayingAudio, setIsPlayingAudio] = useState(false)
|
||||
const localAudioContextRef = useRef<AudioContext | null>(null)
|
||||
const audioContextRef = sharedAudioContextRef || localAudioContextRef
|
||||
const currentSourceRef = useRef<AudioBufferSourceNode | null>(null)
|
||||
const abortControllerRef = useRef<AbortController | null>(null)
|
||||
const audioQueueRef = useRef<AudioQueueItem[]>([])
|
||||
const isProcessingQueueRef = useRef(false)
|
||||
|
||||
const getAudioContext = useCallback(() => {
|
||||
if (!audioContextRef.current) {
|
||||
const AudioContextConstructor = window.AudioContext || window.webkitAudioContext
|
||||
if (!AudioContextConstructor) {
|
||||
throw new Error('AudioContext is not supported in this browser')
|
||||
}
|
||||
audioContextRef.current = new AudioContextConstructor()
|
||||
}
|
||||
return audioContextRef.current
|
||||
}, [])
|
||||
|
||||
const stopAudio = useCallback(() => {
|
||||
abortControllerRef.current?.abort()
|
||||
|
||||
if (currentSourceRef.current) {
|
||||
try {
|
||||
currentSourceRef.current.stop()
|
||||
} catch (e) {
|
||||
// Already stopped
|
||||
}
|
||||
currentSourceRef.current = null
|
||||
}
|
||||
|
||||
audioQueueRef.current = []
|
||||
isProcessingQueueRef.current = false
|
||||
|
||||
setIsPlayingAudio(false)
|
||||
}, [])
|
||||
|
||||
const processAudioQueue = useCallback(async () => {
|
||||
if (isProcessingQueueRef.current || audioQueueRef.current.length === 0) {
|
||||
return
|
||||
}
|
||||
|
||||
isProcessingQueueRef.current = true
|
||||
const item = audioQueueRef.current.shift()
|
||||
|
||||
if (!item) {
|
||||
isProcessingQueueRef.current = false
|
||||
return
|
||||
}
|
||||
|
||||
const { text, options } = item
|
||||
const {
|
||||
voiceId,
|
||||
modelId = DEFAULT_TTS_MODEL_ID,
|
||||
chatId,
|
||||
onAudioStart,
|
||||
onAudioEnd,
|
||||
onError,
|
||||
} = options
|
||||
|
||||
try {
|
||||
const audioContext = getAudioContext()
|
||||
|
||||
if (audioContext.state === 'suspended') {
|
||||
await audioContext.resume()
|
||||
}
|
||||
// boundary-raw-fetch: TTS proxy returns raw audio bytes consumed via response.arrayBuffer() and decoded by AudioContext.decodeAudioData
|
||||
const response = await fetch('/api/proxy/tts/stream', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
text,
|
||||
voiceId,
|
||||
modelId,
|
||||
chatId,
|
||||
}),
|
||||
signal: abortControllerRef.current?.signal,
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text().catch(() => '')
|
||||
throw new Error(errorText || `TTS request failed: ${response.statusText}`)
|
||||
}
|
||||
|
||||
const arrayBuffer = await response.arrayBuffer()
|
||||
const audioBuffer = await audioContext.decodeAudioData(arrayBuffer)
|
||||
|
||||
const source = audioContext.createBufferSource()
|
||||
source.buffer = audioBuffer
|
||||
source.connect(audioContext.destination)
|
||||
source.onended = () => {
|
||||
currentSourceRef.current = null
|
||||
onAudioEnd?.()
|
||||
|
||||
isProcessingQueueRef.current = false
|
||||
|
||||
if (audioQueueRef.current.length === 0) {
|
||||
setIsPlayingAudio(false)
|
||||
}
|
||||
|
||||
setTimeout(() => processAudioQueue(), 0)
|
||||
}
|
||||
|
||||
currentSourceRef.current = source
|
||||
source.start(0)
|
||||
setIsPlayingAudio(true)
|
||||
onAudioStart?.()
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.name !== 'AbortError') {
|
||||
logger.error('Audio streaming error:', error)
|
||||
onError?.(error)
|
||||
}
|
||||
|
||||
isProcessingQueueRef.current = false
|
||||
setTimeout(() => processAudioQueue(), 0)
|
||||
}
|
||||
}, [getAudioContext])
|
||||
|
||||
const streamTextToAudio = useCallback(
|
||||
async (text: string, options: AudioStreamingOptions) => {
|
||||
if (!text.trim()) {
|
||||
return
|
||||
}
|
||||
|
||||
if (!abortControllerRef.current || abortControllerRef.current.signal.aborted) {
|
||||
abortControllerRef.current = new AbortController()
|
||||
}
|
||||
|
||||
for (const piece of splitForSynthesis(text)) {
|
||||
audioQueueRef.current.push({ text: piece, options })
|
||||
}
|
||||
processAudioQueue()
|
||||
},
|
||||
[processAudioQueue]
|
||||
)
|
||||
|
||||
return {
|
||||
isPlayingAudio,
|
||||
streamTextToAudio,
|
||||
stopAudio,
|
||||
}
|
||||
}
|
||||
@@ -344,47 +344,6 @@ describe('useChatStreaming thinking + abort', () => {
|
||||
expect(assistant?.thinking).toBe('real thought')
|
||||
})
|
||||
|
||||
it('TTS audioStreamHandler receives answer text only', async () => {
|
||||
const audioStreamHandler = vi.fn().mockResolvedValue(undefined)
|
||||
|
||||
mockReadSSEEvents.mockImplementation(async (_source, options) => {
|
||||
await options.onEvent({
|
||||
blockId: 'agent-1',
|
||||
event: 'thinking',
|
||||
data: 'secret internal monologue that must not be spoken.',
|
||||
})
|
||||
await options.onEvent({
|
||||
blockId: 'agent-1',
|
||||
chunk: 'Hello world.',
|
||||
})
|
||||
await options.onEvent({
|
||||
event: 'final',
|
||||
data: { success: true, output: {} },
|
||||
})
|
||||
})
|
||||
|
||||
await act(async () => {
|
||||
await handle
|
||||
.latest()
|
||||
.handleStreamedResponse(makeSseResponse(), setMessages, vi.fn(), vi.fn(), {
|
||||
voiceSettings: {
|
||||
isVoiceEnabled: true,
|
||||
voiceId: 'voice-1',
|
||||
autoPlayResponses: true,
|
||||
},
|
||||
audioStreamHandler,
|
||||
})
|
||||
})
|
||||
await flushUiBatch()
|
||||
|
||||
expect(audioStreamHandler).toHaveBeenCalled()
|
||||
for (const call of audioStreamHandler.mock.calls) {
|
||||
expect(String(call[0])).not.toContain('secret')
|
||||
expect(String(call[0])).not.toContain('monologue')
|
||||
}
|
||||
expect(audioStreamHandler.mock.calls.some((c) => String(c[0]).includes('Hello'))).toBe(true)
|
||||
})
|
||||
|
||||
it('stopStreaming preserves thinking and aborts the shared controller', async () => {
|
||||
const abortController = new AbortController()
|
||||
let resolveStream!: () => void
|
||||
|
||||
@@ -69,20 +69,7 @@ function extractFilesFromData(
|
||||
return files
|
||||
}
|
||||
|
||||
interface VoiceSettings {
|
||||
isVoiceEnabled: boolean
|
||||
voiceId: string
|
||||
autoPlayResponses: boolean
|
||||
voiceFirstMode?: boolean
|
||||
textStreamingInVoiceMode?: 'hidden' | 'synced' | 'normal'
|
||||
conversationMode?: boolean
|
||||
}
|
||||
|
||||
export interface StreamingOptions {
|
||||
voiceSettings?: VoiceSettings
|
||||
onAudioStart?: () => void
|
||||
onAudioEnd?: () => void
|
||||
audioStreamHandler?: (text: string) => Promise<void>
|
||||
outputConfigs?: Array<{ blockId: string; path?: string }>
|
||||
/**
|
||||
* Shared AbortController for fetch + SSE body reads. When provided (preferred),
|
||||
@@ -104,9 +91,6 @@ export function useChatStreaming() {
|
||||
const accumulatedTextRef = useRef<string>('')
|
||||
const accumulatedThinkingRef = useRef<string>('')
|
||||
const accumulatedToolCallsRef = useRef<ChatToolCall[]>([])
|
||||
const lastStreamedPositionRef = useRef<number>(0)
|
||||
const audioStreamingActiveRef = useRef<boolean>(false)
|
||||
const lastDisplayedPositionRef = useRef<number>(0)
|
||||
|
||||
const stopStreaming = (setMessages: React.Dispatch<React.SetStateAction<ChatMessage[]>>) => {
|
||||
if (abortControllerRef.current) {
|
||||
@@ -149,9 +133,6 @@ export function useChatStreaming() {
|
||||
accumulatedTextRef.current = ''
|
||||
accumulatedThinkingRef.current = ''
|
||||
accumulatedToolCallsRef.current = []
|
||||
lastStreamedPositionRef.current = 0
|
||||
lastDisplayedPositionRef.current = 0
|
||||
audioStreamingActiveRef.current = false
|
||||
}
|
||||
}
|
||||
|
||||
@@ -172,11 +153,6 @@ export function useChatStreaming() {
|
||||
abortControllerRef.current = new AbortController()
|
||||
}
|
||||
|
||||
const shouldPlayAudio =
|
||||
streamingOptions?.voiceSettings?.isVoiceEnabled &&
|
||||
streamingOptions?.voiceSettings?.autoPlayResponses &&
|
||||
streamingOptions?.audioStreamHandler
|
||||
|
||||
if (!response.body) {
|
||||
setIsLoading(false)
|
||||
setIsStreamingResponse(false)
|
||||
@@ -198,7 +174,6 @@ export function useChatStreaming() {
|
||||
}
|
||||
let accumulatedThinking = ''
|
||||
let isThinkingStreaming = false
|
||||
let lastAudioPosition = 0
|
||||
const toolCallsMap = new Map<string, ChatToolCall>()
|
||||
const toolCallOrder: string[] = []
|
||||
|
||||
@@ -543,9 +518,6 @@ export function useChatStreaming() {
|
||||
accumulatedTextRef.current = ''
|
||||
accumulatedThinkingRef.current = ''
|
||||
accumulatedToolCallsRef.current = []
|
||||
lastStreamedPositionRef.current = 0
|
||||
lastDisplayedPositionRef.current = 0
|
||||
audioStreamingActiveRef.current = false
|
||||
|
||||
terminated = true
|
||||
return true
|
||||
@@ -565,8 +537,6 @@ export function useChatStreaming() {
|
||||
blockTextOrder.splice(orderIndex, 1)
|
||||
}
|
||||
recomputeAccumulatedText()
|
||||
// Spoken audio cannot be unplayed; clamp so slicing stays valid.
|
||||
lastAudioPosition = Math.min(lastAudioPosition, accumulatedText.length)
|
||||
uiDirty = true
|
||||
scheduleUIFlush()
|
||||
}
|
||||
@@ -600,32 +570,6 @@ export function useChatStreaming() {
|
||||
})
|
||||
uiDirty = true
|
||||
scheduleUIFlush()
|
||||
|
||||
if (shouldPlayAudio && streamingOptions?.audioStreamHandler) {
|
||||
const newText = accumulatedText.substring(lastAudioPosition)
|
||||
const sentenceEndings = ['. ', '! ', '? ', '.\n', '!\n', '?\n', '.', '!', '?']
|
||||
let sentenceEnd = -1
|
||||
|
||||
for (const ending of sentenceEndings) {
|
||||
const index = newText.indexOf(ending)
|
||||
if (index > 0) {
|
||||
sentenceEnd = index + ending.length
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if (sentenceEnd > 0) {
|
||||
const sentence = newText.substring(0, sentenceEnd).trim()
|
||||
if (sentence && sentence.length >= 3) {
|
||||
try {
|
||||
await streamingOptions.audioStreamHandler(sentence)
|
||||
lastAudioPosition += sentenceEnd
|
||||
} catch (error) {
|
||||
logger.error('TTS error:', error)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
})
|
||||
@@ -665,25 +609,10 @@ export function useChatStreaming() {
|
||||
}
|
||||
})
|
||||
)
|
||||
if (
|
||||
!wasAborted &&
|
||||
shouldPlayAudio &&
|
||||
streamingOptions?.audioStreamHandler &&
|
||||
accumulatedText.length > lastAudioPosition
|
||||
) {
|
||||
const remainingText = accumulatedText.substring(lastAudioPosition).trim()
|
||||
if (remainingText) {
|
||||
try {
|
||||
await streamingOptions.audioStreamHandler(remainingText)
|
||||
} catch (error) {
|
||||
logger.error('TTS error for remaining text:', error)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
// Stop / timeout abort the shared fetch controller; body read then throws AbortError.
|
||||
// Match chat.tsx + use-audio-streaming: expected cancel, not a hard failure.
|
||||
// Expected cancel, not a hard failure.
|
||||
if (error instanceof Error && error.name === 'AbortError') {
|
||||
logger.info('Stream aborted by user or timeout')
|
||||
settleRunningToolCalls(toolCallsMap, 'cancelled')
|
||||
@@ -718,10 +647,6 @@ export function useChatStreaming() {
|
||||
setTimeout(() => {
|
||||
scrollToBottom()
|
||||
}, 300)
|
||||
|
||||
if (shouldPlayAudio) {
|
||||
streamingOptions?.onAudioEnd?.()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,296 +0,0 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import {
|
||||
createMockRequest,
|
||||
queueTableRows,
|
||||
resetDbChainMock,
|
||||
resetEnvMock,
|
||||
schemaMock,
|
||||
setEnv,
|
||||
} from '@sim/testing'
|
||||
import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
const {
|
||||
mockRecordUsage,
|
||||
mockCheckActorUsageLimits,
|
||||
mockResolveSystemBillingAttribution,
|
||||
mockCheckAttributedUsageLimits,
|
||||
mockToBillingContext,
|
||||
mockEnforceIpRateLimit,
|
||||
mockEnforceChatRateLimit,
|
||||
} = vi.hoisted(() => ({
|
||||
mockRecordUsage: vi.fn(),
|
||||
mockCheckActorUsageLimits: vi.fn(),
|
||||
mockResolveSystemBillingAttribution: vi.fn(),
|
||||
mockCheckAttributedUsageLimits: vi.fn(),
|
||||
mockToBillingContext: vi.fn(),
|
||||
mockEnforceIpRateLimit: vi.fn(),
|
||||
mockEnforceChatRateLimit: vi.fn(),
|
||||
}))
|
||||
|
||||
const SYSTEM_BILLING_ATTRIBUTION = {
|
||||
actorUserId: 'payer-1',
|
||||
workspaceId: 'ws-1',
|
||||
organizationId: 'org-1',
|
||||
billedAccountUserId: 'payer-1',
|
||||
billingEntity: { type: 'organization' as const, id: 'org-1' },
|
||||
billingPeriod: {
|
||||
start: '2026-07-01T00:00:00.000Z',
|
||||
end: '2026-08-01T00:00:00.000Z',
|
||||
},
|
||||
payerSubscription: null,
|
||||
}
|
||||
|
||||
vi.mock('@/lib/billing/core/usage-log', () => ({ recordUsage: mockRecordUsage }))
|
||||
|
||||
vi.mock('@/lib/billing/core/billing-attribution', () => ({
|
||||
resolveSystemBillingAttribution: mockResolveSystemBillingAttribution,
|
||||
checkAttributedUsageLimits: mockCheckAttributedUsageLimits,
|
||||
toBillingContext: mockToBillingContext,
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/billing/calculations/usage-monitor', () => ({
|
||||
checkActorUsageLimits: mockCheckActorUsageLimits,
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/core/rate-limiter/route-helpers', () => ({
|
||||
enforceIpRateLimit: mockEnforceIpRateLimit,
|
||||
enforceChatRateLimit: mockEnforceChatRateLimit,
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/core/security/deployment', () => ({ validateAuthToken: vi.fn(() => false) }))
|
||||
|
||||
import { DEFAULT_TTS_VOICE_ID, MAX_TTS_TEXT_LENGTH } from '@/lib/api/contracts/media/tts-stream'
|
||||
import { POST } from '@/app/api/proxy/tts/stream/route'
|
||||
|
||||
const publicChatRow = {
|
||||
id: 'chat-1',
|
||||
userId: 'owner-1',
|
||||
isActive: true,
|
||||
authType: 'public',
|
||||
password: null,
|
||||
workspaceId: 'ws-1',
|
||||
}
|
||||
|
||||
function validBody(overrides: Record<string, unknown> = {}) {
|
||||
return {
|
||||
text: 'Hello from the deployed chat.',
|
||||
voiceId: DEFAULT_TTS_VOICE_ID,
|
||||
chatId: 'chat-1',
|
||||
...overrides,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Minimal ElevenLabs stub returning a readable audio body. Returns the `cancel`
|
||||
* spy so tests can assert the vendor stream is released when we reject.
|
||||
*/
|
||||
function mockElevenLabsAudio() {
|
||||
const stream = new ReadableStream({
|
||||
start(controller) {
|
||||
controller.enqueue(new Uint8Array([0x49, 0x44, 0x33]))
|
||||
controller.close()
|
||||
},
|
||||
})
|
||||
const cancel = vi.fn(() => stream.cancel())
|
||||
global.fetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
status: 200,
|
||||
statusText: 'OK',
|
||||
body: { getReader: () => stream.getReader(), cancel },
|
||||
// double-cast-allowed: minimal fetch stub for the ElevenLabs stream call
|
||||
}) as unknown as typeof fetch
|
||||
return cancel
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
resetDbChainMock()
|
||||
setEnv({ ELEVENLABS_API_KEY: 'test-key' })
|
||||
mockEnforceIpRateLimit.mockResolvedValue(null)
|
||||
mockEnforceChatRateLimit.mockResolvedValue(null)
|
||||
mockRecordUsage.mockResolvedValue(undefined)
|
||||
mockCheckActorUsageLimits.mockResolvedValue({ isExceeded: false })
|
||||
mockCheckAttributedUsageLimits.mockResolvedValue({ isExceeded: false })
|
||||
mockResolveSystemBillingAttribution.mockResolvedValue(SYSTEM_BILLING_ATTRIBUTION)
|
||||
mockToBillingContext.mockImplementation(
|
||||
(attribution: { billingEntity: { type: 'organization' | 'user'; id: string } }) => ({
|
||||
billingEntity: attribution.billingEntity,
|
||||
billingPeriod: {
|
||||
start: new Date('2026-07-01T00:00:00.000Z'),
|
||||
end: new Date('2026-08-01T00:00:00.000Z'),
|
||||
},
|
||||
})
|
||||
)
|
||||
mockElevenLabsAudio()
|
||||
})
|
||||
|
||||
afterAll(() => {
|
||||
resetDbChainMock()
|
||||
resetEnvMock()
|
||||
})
|
||||
|
||||
describe('POST /api/proxy/tts/stream — spend controls', () => {
|
||||
it('caps text length so one request cannot bill unbounded characters', async () => {
|
||||
const res = await POST(
|
||||
createMockRequest('POST', validBody({ text: 'a'.repeat(MAX_TTS_TEXT_LENGTH + 1) }))
|
||||
)
|
||||
|
||||
expect(res.status).toBe(400)
|
||||
expect(global.fetch).not.toHaveBeenCalled()
|
||||
expect(mockRecordUsage).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('rejects an oversized body before buffering it', async () => {
|
||||
const res = await POST(createMockRequest('POST', validBody({ padding: 'x'.repeat(32 * 1024) })))
|
||||
|
||||
expect(res.status).toBe(413)
|
||||
expect(global.fetch).not.toHaveBeenCalled()
|
||||
expect(mockRecordUsage).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('rejects a voice outside the allowlist so the caller cannot pick a premium voice', async () => {
|
||||
const res = await POST(
|
||||
createMockRequest('POST', validBody({ voiceId: '21m00Tcm4TlvDq8ikWAM' }))
|
||||
)
|
||||
|
||||
expect(res.status).toBe(400)
|
||||
expect(global.fetch).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('rejects a model outside the allowlist so the caller cannot pick the billing model', async () => {
|
||||
const res = await POST(
|
||||
createMockRequest('POST', validBody({ modelId: 'eleven_multilingual_v2' }))
|
||||
)
|
||||
|
||||
expect(res.status).toBe(400)
|
||||
expect(global.fetch).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('throttles per IP before any chat lookup so a flood cannot be amplified into queries', async () => {
|
||||
mockEnforceIpRateLimit.mockResolvedValue(new Response('Rate limit exceeded', { status: 429 }))
|
||||
|
||||
const res = await POST(createMockRequest('POST', validBody()))
|
||||
|
||||
expect(res.status).toBe(429)
|
||||
expect(mockEnforceIpRateLimit).toHaveBeenCalledWith('tts-stream', expect.anything(), {
|
||||
maxTokens: 60,
|
||||
refillRate: 30,
|
||||
refillIntervalMs: 60_000,
|
||||
})
|
||||
expect(mockEnforceChatRateLimit).not.toHaveBeenCalled()
|
||||
expect(global.fetch).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('throttles per chat so many callers cannot drain one public chat', async () => {
|
||||
queueTableRows(schemaMock.chat, [publicChatRow])
|
||||
mockEnforceChatRateLimit.mockResolvedValue(new Response('Rate limit exceeded', { status: 429 }))
|
||||
|
||||
const res = await POST(createMockRequest('POST', validBody()))
|
||||
|
||||
expect(res.status).toBe(429)
|
||||
expect(mockEnforceChatRateLimit).toHaveBeenCalledWith('tts-stream', 'chat-1', {
|
||||
maxTokens: 120,
|
||||
refillRate: 60,
|
||||
refillIntervalMs: 60_000,
|
||||
})
|
||||
expect(global.fetch).not.toHaveBeenCalled()
|
||||
expect(mockRecordUsage).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
|
||||
describe('POST /api/proxy/tts/stream — attribution', () => {
|
||||
it('meters synthesized characters against the chat workspace payer', async () => {
|
||||
queueTableRows(schemaMock.chat, [publicChatRow])
|
||||
const text = 'a'.repeat(1000)
|
||||
|
||||
const res = await POST(createMockRequest('POST', validBody({ text })))
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
expect(mockResolveSystemBillingAttribution).toHaveBeenCalledWith('ws-1')
|
||||
expect(mockCheckAttributedUsageLimits).toHaveBeenCalledWith(SYSTEM_BILLING_ATTRIBUTION)
|
||||
expect(mockRecordUsage).toHaveBeenCalledTimes(1)
|
||||
expect(mockRecordUsage.mock.calls[0][0]).toMatchObject({
|
||||
userId: 'payer-1',
|
||||
workspaceId: 'ws-1',
|
||||
billingEntity: { type: 'organization', id: 'org-1' },
|
||||
})
|
||||
expect(mockRecordUsage.mock.calls[0][0].entries[0]).toMatchObject({
|
||||
category: 'fixed',
|
||||
source: 'voice-output',
|
||||
cost: 0.05,
|
||||
})
|
||||
})
|
||||
|
||||
it('gives each call a unique source reference so equal-length calls are not deduplicated', async () => {
|
||||
const text = 'Same length text.'
|
||||
|
||||
queueTableRows(schemaMock.chat, [publicChatRow])
|
||||
await POST(createMockRequest('POST', validBody({ text })))
|
||||
queueTableRows(schemaMock.chat, [publicChatRow])
|
||||
await POST(createMockRequest('POST', validBody({ text })))
|
||||
|
||||
expect(mockRecordUsage).toHaveBeenCalledTimes(2)
|
||||
const first = mockRecordUsage.mock.calls[0][0].entries[0].sourceReference
|
||||
const second = mockRecordUsage.mock.calls[1][0].entries[0].sourceReference
|
||||
expect(first).toBeDefined()
|
||||
expect(first).not.toBe(second)
|
||||
})
|
||||
|
||||
it('falls back to the chat owner when the workflow has no workspace', async () => {
|
||||
queueTableRows(schemaMock.chat, [{ ...publicChatRow, workspaceId: null }])
|
||||
|
||||
const res = await POST(createMockRequest('POST', validBody()))
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
expect(mockResolveSystemBillingAttribution).not.toHaveBeenCalled()
|
||||
expect(mockCheckActorUsageLimits).toHaveBeenCalledWith('owner-1')
|
||||
expect(mockRecordUsage.mock.calls[0][0]).toMatchObject({ userId: 'owner-1' })
|
||||
})
|
||||
|
||||
it('refuses to spend when the payer is over its usage limit', async () => {
|
||||
queueTableRows(schemaMock.chat, [publicChatRow])
|
||||
mockCheckAttributedUsageLimits.mockResolvedValue({
|
||||
isExceeded: true,
|
||||
message: 'Usage limit exceeded.',
|
||||
})
|
||||
|
||||
const res = await POST(createMockRequest('POST', validBody()))
|
||||
|
||||
expect(res.status).toBe(402)
|
||||
expect(global.fetch).not.toHaveBeenCalled()
|
||||
expect(mockRecordUsage).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('refuses to stream audio it could not record a charge for, releasing the vendor stream', async () => {
|
||||
const cancel = mockElevenLabsAudio()
|
||||
queueTableRows(schemaMock.chat, [publicChatRow])
|
||||
mockRecordUsage.mockRejectedValue(new Error('ledger unavailable'))
|
||||
|
||||
const res = await POST(createMockRequest('POST', validBody()))
|
||||
|
||||
expect(res.status).toBe(500)
|
||||
expect(res.headers.get('Content-Type')).not.toBe('audio/mpeg')
|
||||
expect(cancel).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('rejects an unknown chat without touching the platform key', async () => {
|
||||
queueTableRows(schemaMock.chat, [])
|
||||
|
||||
const res = await POST(createMockRequest('POST', validBody()))
|
||||
|
||||
expect(res.status).toBe(401)
|
||||
expect(global.fetch).not.toHaveBeenCalled()
|
||||
expect(mockRecordUsage).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('does not expose the audio stream to arbitrary origins', async () => {
|
||||
queueTableRows(schemaMock.chat, [publicChatRow])
|
||||
|
||||
const res = await POST(createMockRequest('POST', validBody()))
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
expect(res.headers.get('Access-Control-Allow-Origin')).toBeNull()
|
||||
})
|
||||
})
|
||||
@@ -1,243 +0,0 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { generateId } from '@sim/utils/id'
|
||||
import { type NextRequest, NextResponse } from 'next/server'
|
||||
import { ttsStreamContract } from '@/lib/api/contracts/media/tts-stream'
|
||||
import { parseRequest } from '@/lib/api/server'
|
||||
import { checkActorUsageLimits } from '@/lib/billing/calculations/usage-monitor'
|
||||
import {
|
||||
checkAttributedUsageLimits,
|
||||
resolveSystemBillingAttribution,
|
||||
toBillingContext,
|
||||
} from '@/lib/billing/core/billing-attribution'
|
||||
import { recordUsage } from '@/lib/billing/core/usage-log'
|
||||
import { resolveDeployedChatCaller } from '@/lib/chat/deployed-chat-caller'
|
||||
import { env } from '@/lib/core/config/env'
|
||||
import { getCostMultiplier } from '@/lib/core/config/env-flags'
|
||||
import { enforceChatRateLimit, enforceIpRateLimit } from '@/lib/core/rate-limiter/route-helpers'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
|
||||
const logger = createLogger('ProxyTTSStreamAPI')
|
||||
|
||||
/**
|
||||
* Filters naive floods only: `getClientIp` trusts the leftmost
|
||||
* `X-Forwarded-For` value, which the caller controls, so a deliberate attacker
|
||||
* rotates past this bucket. See {@link TTS_CHAT_RATE_LIMIT}.
|
||||
*
|
||||
* Deployed chat synthesizes sentence by sentence, so a real conversation issues
|
||||
* several requests per answer — hence the generous burst.
|
||||
*/
|
||||
const TTS_IP_RATE_LIMIT = {
|
||||
maxTokens: 60,
|
||||
refillRate: 30,
|
||||
refillIntervalMs: 60 * 1000,
|
||||
} as const
|
||||
|
||||
/**
|
||||
* The load-bearing spend control. Public chats hand their id to every visitor,
|
||||
* so the id alone cannot gate use of the platform ElevenLabs key. This bucket
|
||||
* is keyed on server-held state, bounding total spend per chat regardless of
|
||||
* how many source addresses the traffic claims to come from.
|
||||
*/
|
||||
const TTS_CHAT_RATE_LIMIT = {
|
||||
maxTokens: 120,
|
||||
refillRate: 60,
|
||||
refillIntervalMs: 60 * 1000,
|
||||
} as const
|
||||
|
||||
/**
|
||||
* Published ElevenLabs API rate for Flash/Turbo text-to-speech, in USD per
|
||||
* 1,000 characters. This is the vendor cost; `getCostMultiplier()` applies the
|
||||
* platform markup, matching how other metered sources are priced.
|
||||
*/
|
||||
const TTS_COST_PER_1K_CHARS = 0.05
|
||||
|
||||
/**
|
||||
* The body carries at most `MAX_TTS_TEXT_LENGTH` characters of text plus two
|
||||
* short ids, so a tight cap keeps an anonymous caller from making the route
|
||||
* buffer a large payload before validation runs. Without it the shared default
|
||||
* is 50 MB.
|
||||
*/
|
||||
const MAX_TTS_BODY_BYTES = 16 * 1024
|
||||
|
||||
export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
try {
|
||||
// Throttle per IP before any database work so a flood cannot be amplified into chat lookups.
|
||||
const ipLimited = await enforceIpRateLimit('tts-stream', request, TTS_IP_RATE_LIMIT)
|
||||
if (ipLimited) return ipLimited
|
||||
|
||||
const parsed = await parseRequest(
|
||||
ttsStreamContract,
|
||||
request,
|
||||
{},
|
||||
{
|
||||
maxBodyBytes: MAX_TTS_BODY_BYTES,
|
||||
invalidJsonResponse: () => new NextResponse('Invalid request body', { status: 400 }),
|
||||
validationErrorResponse: (error) => {
|
||||
if (error.issues.some((issue) => issue.path[0] === 'chatId')) {
|
||||
return new NextResponse('chatId is required', { status: 400 })
|
||||
}
|
||||
return new NextResponse('Missing required parameters', { status: 400 })
|
||||
},
|
||||
}
|
||||
)
|
||||
if (!parsed.success) return parsed.response
|
||||
|
||||
const { text, voiceId, modelId, chatId } = parsed.data.body
|
||||
|
||||
const caller = await resolveDeployedChatCaller(request, chatId)
|
||||
if (!caller.authorized) {
|
||||
logger.warn('Chat authentication failed for TTS, chatId:', chatId)
|
||||
return new Response('Unauthorized', { status: 401 })
|
||||
}
|
||||
|
||||
const chatLimited = await enforceChatRateLimit('tts-stream', chatId, TTS_CHAT_RATE_LIMIT)
|
||||
if (chatLimited) return chatLimited
|
||||
|
||||
// Anonymous deployed chats have no human request actor, so the workspace payer is charged.
|
||||
const workspaceId = caller.workspaceId ?? undefined
|
||||
const billingAttribution = workspaceId
|
||||
? await resolveSystemBillingAttribution(workspaceId)
|
||||
: undefined
|
||||
const actorUserId = billingAttribution?.actorUserId ?? caller.ownerId
|
||||
|
||||
const usageCheck = billingAttribution
|
||||
? await checkAttributedUsageLimits(billingAttribution)
|
||||
: await checkActorUsageLimits(actorUserId)
|
||||
if (usageCheck.isExceeded) {
|
||||
return new Response(usageCheck.message || 'Usage limit exceeded.', { status: 402 })
|
||||
}
|
||||
|
||||
const apiKey = env.ELEVENLABS_API_KEY
|
||||
if (!apiKey) {
|
||||
logger.error('ELEVENLABS_API_KEY not configured on server')
|
||||
return new Response('ElevenLabs service not configured', { status: 503 })
|
||||
}
|
||||
|
||||
const query = new URLSearchParams({ output_format: 'mp3_44100_128' })
|
||||
const endpoint = `https://api.elevenlabs.io/v1/text-to-speech/${voiceId}/stream?${query.toString()}`
|
||||
|
||||
const response = await fetch(endpoint, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Accept: 'audio/mpeg',
|
||||
'Content-Type': 'application/json',
|
||||
'xi-api-key': apiKey,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
text,
|
||||
model_id: modelId,
|
||||
voice_settings: {
|
||||
stability: 0.5,
|
||||
similarity_boost: 0.8,
|
||||
style: 0.0,
|
||||
use_speaker_boost: false,
|
||||
},
|
||||
apply_text_normalization: 'auto',
|
||||
}),
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
logger.error(`Failed to generate Stream TTS: ${response.status} ${response.statusText}`)
|
||||
return new Response(`Failed to generate TTS: ${response.status} ${response.statusText}`, {
|
||||
status: response.status,
|
||||
})
|
||||
}
|
||||
|
||||
if (!response.body) {
|
||||
logger.error('No response body received from ElevenLabs')
|
||||
return new Response('No audio stream received', { status: 422 })
|
||||
}
|
||||
|
||||
/**
|
||||
* Meter once ElevenLabs has accepted the request — the characters are billed
|
||||
* to us at that point regardless of whether the client drains the stream.
|
||||
*
|
||||
* `sourceReference` must be unique per call. `usage_log.event_key` is
|
||||
* unique and inserts conflict-do-nothing, and the key is derived from the
|
||||
* entry's stable fields — without this, two synthesis calls of equal length
|
||||
* in the same workspace would collide and the second would silently go
|
||||
* unbilled. Each call is a separate charge from ElevenLabs, so each needs
|
||||
* its own row rather than being deduplicated. `generateId` rather than
|
||||
* `generateRequestId`, whose fallback truncates to 8 characters.
|
||||
*
|
||||
* A ledger failure fails the request rather than streaming anyway: the
|
||||
* caller is anonymous, so serving audio we could not charge for is exactly
|
||||
* the unmetered spend this route exists to prevent. The vendor call is
|
||||
* already paid for at this point, but the caller gains nothing from it, so
|
||||
* there is no incentive to farm ledger outages.
|
||||
*
|
||||
* No threshold settlement here: it runs per metered event elsewhere and is
|
||||
* far too heavy for a per-sentence realtime path. The workflow execution
|
||||
* that produced this text already settles the payer.
|
||||
*/
|
||||
try {
|
||||
await recordUsage({
|
||||
userId: actorUserId,
|
||||
workspaceId,
|
||||
...(billingAttribution ? toBillingContext(billingAttribution) : {}),
|
||||
entries: [
|
||||
{
|
||||
category: 'fixed',
|
||||
source: 'voice-output',
|
||||
description: `Voice output (${text.length} characters)`,
|
||||
cost: (text.length / 1000) * TTS_COST_PER_1K_CHARS * getCostMultiplier(),
|
||||
sourceReference: `voice-output:${chatId}:${generateId()}`,
|
||||
},
|
||||
],
|
||||
})
|
||||
} catch (err) {
|
||||
logger.error('Failed to record voice output usage, refusing to stream:', err)
|
||||
// Release the open vendor stream; nothing will drain it once we reject.
|
||||
await response.body?.cancel().catch(() => {})
|
||||
return new Response('Unable to record usage for this request', { status: 500 })
|
||||
}
|
||||
|
||||
const { readable, writable } = new TransformStream({
|
||||
transform(chunk, controller) {
|
||||
controller.enqueue(chunk)
|
||||
},
|
||||
flush(controller) {
|
||||
controller.terminate()
|
||||
},
|
||||
})
|
||||
|
||||
const writer = writable.getWriter()
|
||||
const reader = response.body.getReader()
|
||||
|
||||
;(async () => {
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read()
|
||||
if (done) {
|
||||
await writer.close()
|
||||
break
|
||||
}
|
||||
writer.write(value).catch(logger.error)
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error('Error during Stream streaming:', error)
|
||||
await writer.abort(error)
|
||||
}
|
||||
})()
|
||||
|
||||
return new Response(readable, {
|
||||
headers: {
|
||||
'Content-Type': 'audio/mpeg',
|
||||
'Transfer-Encoding': 'chunked',
|
||||
'Cache-Control': 'no-cache, no-store, must-revalidate',
|
||||
Pragma: 'no-cache',
|
||||
Expires: '0',
|
||||
'X-Content-Type-Options': 'nosniff',
|
||||
Connection: 'keep-alive',
|
||||
'X-Accel-Buffering': 'no',
|
||||
'X-Stream-Type': 'real-time',
|
||||
},
|
||||
})
|
||||
} catch (error) {
|
||||
logger.error('Error in Stream TTS:', error)
|
||||
|
||||
return new Response('Internal Server Error', {
|
||||
status: 500,
|
||||
})
|
||||
}
|
||||
})
|
||||
@@ -4,10 +4,8 @@
|
||||
import {
|
||||
authMockFns,
|
||||
createMockRequest,
|
||||
queueTableRows,
|
||||
resetDbChainMock,
|
||||
resetEnvMock,
|
||||
schemaMock,
|
||||
setEnv,
|
||||
} from '@sim/testing'
|
||||
import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
@@ -17,7 +15,6 @@ const {
|
||||
mockCheckActorUsageLimits,
|
||||
mockVerifyWorkspaceMembership,
|
||||
mockResolveBillingAttribution,
|
||||
mockResolveSystemBillingAttribution,
|
||||
mockCheckAttributedUsageLimits,
|
||||
mockToBillingContext,
|
||||
mockCheckAndBillPayerOverageThreshold,
|
||||
@@ -26,30 +23,15 @@ const {
|
||||
mockCheckActorUsageLimits: vi.fn(),
|
||||
mockVerifyWorkspaceMembership: vi.fn(),
|
||||
mockResolveBillingAttribution: vi.fn(),
|
||||
mockResolveSystemBillingAttribution: vi.fn(),
|
||||
mockCheckAttributedUsageLimits: vi.fn(),
|
||||
mockToBillingContext: vi.fn(),
|
||||
mockCheckAndBillPayerOverageThreshold: vi.fn(),
|
||||
}))
|
||||
|
||||
const SYSTEM_BILLING_ATTRIBUTION = {
|
||||
actorUserId: 'owner-after-transfer',
|
||||
workspaceId: 'ws-1',
|
||||
organizationId: 'org-after-transfer',
|
||||
billedAccountUserId: 'owner-after-transfer',
|
||||
billingEntity: { type: 'organization' as const, id: 'org-after-transfer' },
|
||||
billingPeriod: {
|
||||
start: '2026-07-01T00:00:00.000Z',
|
||||
end: '2026-08-01T00:00:00.000Z',
|
||||
},
|
||||
payerSubscription: null,
|
||||
}
|
||||
|
||||
vi.mock('@/lib/billing/core/usage-log', () => ({ recordUsage: mockRecordUsage }))
|
||||
|
||||
vi.mock('@/lib/billing/core/billing-attribution', () => ({
|
||||
resolveBillingAttribution: mockResolveBillingAttribution,
|
||||
resolveSystemBillingAttribution: mockResolveSystemBillingAttribution,
|
||||
checkAttributedUsageLimits: mockCheckAttributedUsageLimits,
|
||||
toBillingContext: mockToBillingContext,
|
||||
}))
|
||||
@@ -72,21 +54,10 @@ vi.mock('@/lib/core/rate-limiter', () => ({
|
||||
},
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/core/security/deployment', () => ({ validateAuthToken: vi.fn(() => false) }))
|
||||
|
||||
import { POST } from '@/app/api/speech/token/route'
|
||||
|
||||
const mockGetSession = authMockFns.mockGetSession
|
||||
|
||||
const publicChatRow = {
|
||||
id: 'chat-1',
|
||||
userId: 'owner-1',
|
||||
isActive: true,
|
||||
authType: 'public',
|
||||
password: null,
|
||||
workspaceId: 'ws-1',
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
resetDbChainMock()
|
||||
@@ -102,7 +73,6 @@ beforeEach(() => {
|
||||
billingEntity: { type: 'organization', id: 'org-1' },
|
||||
})
|
||||
)
|
||||
mockResolveSystemBillingAttribution.mockResolvedValue(SYSTEM_BILLING_ATTRIBUTION)
|
||||
mockToBillingContext.mockImplementation(
|
||||
(attribution: { billingEntity: { type: 'organization' | 'user'; id: string } }) => ({
|
||||
billingEntity: attribution.billingEntity,
|
||||
@@ -140,7 +110,6 @@ describe('POST /api/speech/token — usage attribution', () => {
|
||||
actorUserId: 'member-1',
|
||||
workspaceId: 'ws-1',
|
||||
})
|
||||
expect(mockResolveSystemBillingAttribution).not.toHaveBeenCalled()
|
||||
expect(mockCheckAndBillPayerOverageThreshold).toHaveBeenCalledWith({
|
||||
type: 'organization',
|
||||
id: 'org-1',
|
||||
@@ -156,44 +125,8 @@ describe('POST /api/speech/token — usage attribution', () => {
|
||||
expect(mockRecordUsage).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('deployed chat: uses one atomic system actor and payer snapshot', async () => {
|
||||
queueTableRows(schemaMock.chat, [publicChatRow])
|
||||
|
||||
const res = await POST(createMockRequest('POST', { chatId: 'chat-1' }))
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
expect(mockGetSession).not.toHaveBeenCalled()
|
||||
expect(mockResolveSystemBillingAttribution).toHaveBeenCalledWith('ws-1')
|
||||
expect(mockResolveBillingAttribution).not.toHaveBeenCalled()
|
||||
expect(mockCheckAttributedUsageLimits).toHaveBeenCalledWith(SYSTEM_BILLING_ATTRIBUTION)
|
||||
expect(mockToBillingContext).toHaveBeenCalledWith(SYSTEM_BILLING_ATTRIBUTION)
|
||||
expect(mockRecordUsage.mock.calls[0][0]).toMatchObject({
|
||||
userId: 'owner-after-transfer',
|
||||
workspaceId: 'ws-1',
|
||||
billingEntity: { type: 'organization', id: 'org-after-transfer' },
|
||||
})
|
||||
expect(mockCheckAndBillPayerOverageThreshold).toHaveBeenCalledWith({
|
||||
type: 'organization',
|
||||
id: 'org-after-transfer',
|
||||
})
|
||||
})
|
||||
|
||||
it('deployed chat: uses the chat owner only when no workspace exists', async () => {
|
||||
queueTableRows(schemaMock.chat, [{ ...publicChatRow, workspaceId: null }])
|
||||
|
||||
const res = await POST(createMockRequest('POST', { chatId: 'chat-1' }))
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
expect(mockResolveSystemBillingAttribution).not.toHaveBeenCalled()
|
||||
expect(mockResolveBillingAttribution).not.toHaveBeenCalled()
|
||||
expect(mockRecordUsage.mock.calls[0][0]).toMatchObject({
|
||||
userId: 'owner-1',
|
||||
})
|
||||
expect(mockRecordUsage.mock.calls[0][0].workspaceId).toBeUndefined()
|
||||
})
|
||||
|
||||
it('rejects an oversized body before any auth/billing work runs', async () => {
|
||||
const oversizedBody = { chatId: 'x'.repeat(64 * 1024) }
|
||||
const oversizedBody = { workspaceId: 'x'.repeat(64 * 1024) }
|
||||
const res = await POST(createMockRequest('POST', oversizedBody))
|
||||
|
||||
expect(res.status).toBe(413)
|
||||
|
||||
@@ -10,16 +10,13 @@ import {
|
||||
type BillingAttributionSnapshot,
|
||||
checkAttributedUsageLimits,
|
||||
resolveBillingAttribution,
|
||||
resolveSystemBillingAttribution,
|
||||
toBillingContext,
|
||||
} from '@/lib/billing/core/billing-attribution'
|
||||
import { recordUsage } from '@/lib/billing/core/usage-log'
|
||||
import { checkAndBillPayerOverageThreshold } from '@/lib/billing/threshold-billing'
|
||||
import { resolveDeployedChatCaller } from '@/lib/chat/deployed-chat-caller'
|
||||
import { env } from '@/lib/core/config/env'
|
||||
import { getCostMultiplier, isBillingEnabled } from '@/lib/core/config/env-flags'
|
||||
import { RateLimiter } from '@/lib/core/rate-limiter'
|
||||
import { getClientIp } from '@/lib/core/utils/request'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
import { verifyWorkspaceMembership } from '@/app/api/workflows/utils'
|
||||
|
||||
@@ -31,7 +28,6 @@ const ELEVENLABS_TOKEN_URL = 'https://api.elevenlabs.io/v1/single-use-token/real
|
||||
|
||||
const VOICE_SESSION_COST_PER_MIN = 0.008
|
||||
const WORKSPACE_SESSION_MAX_MINUTES = 3
|
||||
const CHAT_SESSION_MAX_MINUTES = 1
|
||||
|
||||
const STT_TOKEN_RATE_LIMIT = {
|
||||
maxTokens: 30,
|
||||
@@ -40,7 +36,7 @@ const STT_TOKEN_RATE_LIMIT = {
|
||||
} as const
|
||||
|
||||
/**
|
||||
* This body only ever carries an optional chatId/workspaceId string, so a
|
||||
* This body only ever carries an optional workspaceId string, so a
|
||||
* tight cap keeps an unauthenticated caller from forcing a large in-memory
|
||||
* allocation before the auth checks below run.
|
||||
*/
|
||||
@@ -57,57 +53,34 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
const parsedBody = await parseOptionalJsonBody(request, MAX_SPEECH_TOKEN_BODY_BYTES)
|
||||
if (!parsedBody.success) return parsedBody.response
|
||||
const body = speechTokenBodySchema.safeParse(parsedBody.data ?? {})
|
||||
const chatId =
|
||||
body.success && typeof body.data.chatId === 'string' ? body.data.chatId : undefined
|
||||
|
||||
let actorUserId: string | undefined
|
||||
let workspaceId: string | undefined
|
||||
let billingAttribution: BillingAttributionSnapshot | undefined
|
||||
|
||||
if (chatId) {
|
||||
const chatAuth = await resolveDeployedChatCaller(request, chatId)
|
||||
if (!chatAuth.authorized) {
|
||||
return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
|
||||
}
|
||||
/**
|
||||
* Anonymous deployed chats have no human request actor, so resolve the
|
||||
* system actor and immutable workspace payer together.
|
||||
*/
|
||||
workspaceId = chatAuth.workspaceId ?? undefined
|
||||
if (workspaceId) {
|
||||
billingAttribution = await resolveSystemBillingAttribution(workspaceId)
|
||||
actorUserId = billingAttribution.actorUserId
|
||||
} else {
|
||||
actorUserId = chatAuth.ownerId
|
||||
}
|
||||
} else {
|
||||
const session = await getSession()
|
||||
if (!session?.user?.id) {
|
||||
return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
|
||||
}
|
||||
actorUserId = session.user.id
|
||||
/**
|
||||
* Editor voice accepts only a workspace the caller belongs to, preventing
|
||||
* client-supplied IDs from misattributing or bypassing member usage.
|
||||
*/
|
||||
const requestedWorkspaceId =
|
||||
body.success && typeof body.data.workspaceId === 'string'
|
||||
? body.data.workspaceId
|
||||
: undefined
|
||||
if (requestedWorkspaceId) {
|
||||
const permission = await verifyWorkspaceMembership(session.user.id, requestedWorkspaceId)
|
||||
if (permission) workspaceId = requestedWorkspaceId
|
||||
}
|
||||
/**
|
||||
* Editor voice is workspace-scoped so every charge has a payer and member
|
||||
* cap attribution.
|
||||
*/
|
||||
if (!workspaceId) {
|
||||
return NextResponse.json({ error: 'Workspace context is required.' }, { status: 400 })
|
||||
}
|
||||
const session = await getSession()
|
||||
if (!session?.user?.id) {
|
||||
return NextResponse.json({ error: 'Unauthorized' }, { status: 401 })
|
||||
}
|
||||
const actorUserId = session.user.id
|
||||
/**
|
||||
* Editor voice accepts only a workspace the caller belongs to, preventing
|
||||
* client-supplied IDs from misattributing or bypassing member usage.
|
||||
*/
|
||||
const requestedWorkspaceId =
|
||||
body.success && typeof body.data.workspaceId === 'string' ? body.data.workspaceId : undefined
|
||||
if (requestedWorkspaceId) {
|
||||
const permission = await verifyWorkspaceMembership(session.user.id, requestedWorkspaceId)
|
||||
if (permission) workspaceId = requestedWorkspaceId
|
||||
}
|
||||
/**
|
||||
* Editor voice is workspace-scoped so every charge has a payer and member
|
||||
* cap attribution.
|
||||
*/
|
||||
if (!workspaceId) {
|
||||
return NextResponse.json({ error: 'Workspace context is required.' }, { status: 400 })
|
||||
}
|
||||
|
||||
if (!billingAttribution && actorUserId && workspaceId) {
|
||||
if (!billingAttribution) {
|
||||
billingAttribution = await resolveBillingAttribution({
|
||||
actorUserId,
|
||||
workspaceId,
|
||||
@@ -115,11 +88,10 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
}
|
||||
|
||||
if (isBillingEnabled) {
|
||||
const rateLimitKey = chatId
|
||||
? `stt-token:chat:${chatId}:${getClientIp(request)}`
|
||||
: `stt-token:user:${actorUserId}`
|
||||
|
||||
const rateCheck = await rateLimiter.checkRateLimitDirect(rateLimitKey, STT_TOKEN_RATE_LIMIT)
|
||||
const rateCheck = await rateLimiter.checkRateLimitDirect(
|
||||
`stt-token:user:${actorUserId}`,
|
||||
STT_TOKEN_RATE_LIMIT
|
||||
)
|
||||
if (!rateCheck.allowed) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Voice input rate limit exceeded. Please try again later.' },
|
||||
@@ -133,20 +105,28 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
}
|
||||
}
|
||||
|
||||
if (actorUserId) {
|
||||
const usageCheck = billingAttribution
|
||||
? await checkAttributedUsageLimits(billingAttribution)
|
||||
: await checkActorUsageLimits(actorUserId)
|
||||
if (usageCheck.isExceeded) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error:
|
||||
usageCheck.message || 'Usage limit exceeded. Please upgrade your plan to continue.',
|
||||
scope: usageCheck.scope,
|
||||
},
|
||||
{ status: 402 }
|
||||
)
|
||||
}
|
||||
/**
|
||||
* Read-then-act, as every metered route in the repo does: concurrent calls
|
||||
* can pass against the same balance and overshoot the payer's limit. Making
|
||||
* this atomic needs a reservation primitive in `lib/billing` that fixes the
|
||||
* class everywhere, not a one-off here.
|
||||
*
|
||||
* Accepted as bounded rather than eliminated: this route is session-gated
|
||||
* and rate limited per user, so the overshoot is a knowable ceiling against
|
||||
* an identified payer.
|
||||
*/
|
||||
const usageCheck = billingAttribution
|
||||
? await checkAttributedUsageLimits(billingAttribution)
|
||||
: await checkActorUsageLimits(actorUserId)
|
||||
if (usageCheck.isExceeded) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
error:
|
||||
usageCheck.message || 'Usage limit exceeded. Please upgrade your plan to continue.',
|
||||
scope: usageCheck.scope,
|
||||
},
|
||||
{ status: 402 }
|
||||
)
|
||||
}
|
||||
|
||||
const apiKey = env.ELEVENLABS_API_KEY
|
||||
@@ -172,31 +152,28 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
|
||||
const data = await response.json()
|
||||
|
||||
if (actorUserId) {
|
||||
const maxMinutes = chatId ? CHAT_SESSION_MAX_MINUTES : WORKSPACE_SESSION_MAX_MINUTES
|
||||
const sessionCost = VOICE_SESSION_COST_PER_MIN * maxMinutes
|
||||
const sessionCost = VOICE_SESSION_COST_PER_MIN * WORKSPACE_SESSION_MAX_MINUTES
|
||||
|
||||
try {
|
||||
await recordUsage({
|
||||
userId: actorUserId,
|
||||
workspaceId,
|
||||
...(billingAttribution ? toBillingContext(billingAttribution) : {}),
|
||||
entries: [
|
||||
{
|
||||
category: 'fixed',
|
||||
source: 'voice-input',
|
||||
description: `Voice input session (${maxMinutes} min)`,
|
||||
cost: sessionCost * getCostMultiplier(),
|
||||
sourceReference: `voice-input:${hashVoiceToken(data.token)}`,
|
||||
},
|
||||
],
|
||||
})
|
||||
if (billingAttribution) {
|
||||
await checkAndBillPayerOverageThreshold(billingAttribution.billingEntity)
|
||||
}
|
||||
} catch (err) {
|
||||
logger.warn('Failed to record voice input usage, continuing:', err)
|
||||
try {
|
||||
await recordUsage({
|
||||
userId: actorUserId,
|
||||
workspaceId,
|
||||
...(billingAttribution ? toBillingContext(billingAttribution) : {}),
|
||||
entries: [
|
||||
{
|
||||
category: 'fixed',
|
||||
source: 'voice-input',
|
||||
description: `Voice input session (${WORKSPACE_SESSION_MAX_MINUTES} min)`,
|
||||
cost: sessionCost * getCostMultiplier(),
|
||||
sourceReference: `voice-input:${hashVoiceToken(data.token)}`,
|
||||
},
|
||||
],
|
||||
})
|
||||
if (billingAttribution) {
|
||||
await checkAndBillPayerOverageThreshold(billingAttribution.billingEntity)
|
||||
}
|
||||
} catch (err) {
|
||||
logger.warn('Failed to record voice input usage, continuing:', err)
|
||||
}
|
||||
|
||||
return NextResponse.json({ token: data.token })
|
||||
|
||||
@@ -1,37 +0,0 @@
|
||||
import { useQuery } from '@tanstack/react-query'
|
||||
import { requestJson } from '@/lib/api/client/request'
|
||||
import type { ContractJsonResponse } from '@/lib/api/contracts'
|
||||
import { getVoiceSettingsContract } from '@/lib/api/contracts'
|
||||
|
||||
/**
|
||||
* Query key factory for voice settings queries
|
||||
*/
|
||||
export const VOICE_SETTINGS_STALE_TIME = 5 * 60 * 1000
|
||||
|
||||
export const voiceSettingsKeys = {
|
||||
all: ['voiceSettings'] as const,
|
||||
availability: () => [...voiceSettingsKeys.all, 'availability'] as const,
|
||||
}
|
||||
|
||||
type VoiceSettingsResponse = ContractJsonResponse<typeof getVoiceSettingsContract>
|
||||
|
||||
async function fetchVoiceSettings(signal?: AbortSignal): Promise<VoiceSettingsResponse> {
|
||||
try {
|
||||
return await requestJson(getVoiceSettingsContract, { signal })
|
||||
} catch {
|
||||
return { sttAvailable: false }
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads the server-side voice configuration so clients can conditionally
|
||||
* enable voice input. Returns `{ sttAvailable: false }` on failure rather
|
||||
* than throwing, since STT is an optional capability.
|
||||
*/
|
||||
export function useVoiceSettings() {
|
||||
return useQuery({
|
||||
queryKey: voiceSettingsKeys.availability(),
|
||||
queryFn: ({ signal }) => fetchVoiceSettings(signal),
|
||||
staleTime: VOICE_SETTINGS_STALE_TIME,
|
||||
})
|
||||
}
|
||||
@@ -1,2 +1 @@
|
||||
export * from '@/lib/api/contracts/media/speech'
|
||||
export * from '@/lib/api/contracts/media/tts-stream'
|
||||
|
||||
@@ -3,7 +3,6 @@ import { defineRouteContract } from '@/lib/api/contracts/types'
|
||||
|
||||
export const speechTokenBodySchema = z
|
||||
.object({
|
||||
chatId: z.string().optional(),
|
||||
/** Editor/workspace voice: the workspace the session user is recording in. */
|
||||
workspaceId: z.string().optional(),
|
||||
})
|
||||
|
||||
@@ -1,37 +0,0 @@
|
||||
import { z } from 'zod'
|
||||
import { defineRouteContract } from '@/lib/api/contracts/types'
|
||||
|
||||
/** Default ElevenLabs voice (Jessica) — Flash v2.5-optimized. */
|
||||
export const DEFAULT_TTS_VOICE_ID = 'cgSgspJ2msm6clMCkdW9'
|
||||
|
||||
export const DEFAULT_TTS_MODEL_ID = 'eleven_flash_v2_5'
|
||||
|
||||
/**
|
||||
* Voices and models the deployed-chat relay may spend the platform ElevenLabs
|
||||
* key on. Anonymous visitors of a public chat can reach the relay, so these are
|
||||
* allowlists rather than free-form strings — otherwise the caller chooses the
|
||||
* (possibly premium or cloned) voice and the billing model.
|
||||
*/
|
||||
const ALLOWED_TTS_VOICE_IDS = [DEFAULT_TTS_VOICE_ID] as const
|
||||
const ALLOWED_TTS_MODEL_IDS = [DEFAULT_TTS_MODEL_ID] as const
|
||||
|
||||
/**
|
||||
* ElevenLabs bills per character, so an uncapped `text` is an uncapped charge.
|
||||
* Chat TTS synthesizes the streamed answer sentence by sentence, keeping real
|
||||
* requests far below this ceiling.
|
||||
*/
|
||||
export const MAX_TTS_TEXT_LENGTH = 2000
|
||||
|
||||
export const ttsStreamBodySchema = z.object({
|
||||
text: z.string().min(1).max(MAX_TTS_TEXT_LENGTH),
|
||||
voiceId: z.enum(ALLOWED_TTS_VOICE_IDS),
|
||||
modelId: z.enum(ALLOWED_TTS_MODEL_IDS).optional().default(DEFAULT_TTS_MODEL_ID),
|
||||
chatId: z.string().min(1),
|
||||
})
|
||||
|
||||
export const ttsStreamContract = defineRouteContract({
|
||||
method: 'POST',
|
||||
path: '/api/proxy/tts/stream',
|
||||
body: ttsStreamBodySchema,
|
||||
response: { mode: 'stream' },
|
||||
})
|
||||
@@ -1,71 +0,0 @@
|
||||
import { db } from '@sim/db'
|
||||
import { chat, workflow } from '@sim/db/schema'
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { and, eq, isNull } from 'drizzle-orm'
|
||||
import type { NextRequest } from 'next/server'
|
||||
import { validateAuthToken } from '@/lib/core/security/deployment'
|
||||
|
||||
const logger = createLogger('DeployedChatCaller')
|
||||
|
||||
export type DeployedChatCaller =
|
||||
| { authorized: false }
|
||||
| { authorized: true; ownerId: string; workspaceId: string | null }
|
||||
|
||||
/**
|
||||
* Resolves whether a caller may act on a deployed chat, and who pays for it.
|
||||
*
|
||||
* Anonymous visitors of a public chat are authorized, so callers that spend a
|
||||
* platform credential must meter against the returned payer rather than treat
|
||||
* authorization as permission to spend freely.
|
||||
*
|
||||
* Shared by every deployed-chat surface that bills a vendor call, so the gate
|
||||
* and the payer are resolved together and cannot drift apart per route.
|
||||
*/
|
||||
export async function resolveDeployedChatCaller(
|
||||
request: NextRequest,
|
||||
chatId: string
|
||||
): Promise<DeployedChatCaller> {
|
||||
try {
|
||||
const rows = await db
|
||||
.select({
|
||||
userId: chat.userId,
|
||||
isActive: chat.isActive,
|
||||
authType: chat.authType,
|
||||
password: chat.password,
|
||||
workspaceId: workflow.workspaceId,
|
||||
})
|
||||
.from(chat)
|
||||
.leftJoin(workflow, eq(workflow.id, chat.workflowId))
|
||||
.where(and(eq(chat.id, chatId), isNull(chat.archivedAt)))
|
||||
.limit(1)
|
||||
|
||||
if (rows.length === 0 || !rows[0].isActive) {
|
||||
logger.warn('Chat not found, archived or inactive', { chatId })
|
||||
return { authorized: false }
|
||||
}
|
||||
|
||||
const chatData = rows[0]
|
||||
const authorized = {
|
||||
authorized: true,
|
||||
ownerId: chatData.userId,
|
||||
workspaceId: chatData.workspaceId,
|
||||
} as const
|
||||
|
||||
if (chatData.authType === 'public') {
|
||||
return authorized
|
||||
}
|
||||
|
||||
const authCookie = request.cookies.get(`chat_auth_${chatId}`)
|
||||
if (
|
||||
authCookie &&
|
||||
validateAuthToken(authCookie.value, chatId, chatData.authType, chatData.password)
|
||||
) {
|
||||
return authorized
|
||||
}
|
||||
|
||||
return { authorized: false }
|
||||
} catch (error) {
|
||||
logger.error('Error resolving deployed chat caller', { chatId, error })
|
||||
return { authorized: false }
|
||||
}
|
||||
}
|
||||
@@ -183,7 +183,7 @@ export const env = createEnv({
|
||||
COHERE_API_KEY_1: z.string().min(1).optional(), // Primary Cohere API key for rotation
|
||||
COHERE_API_KEY_2: z.string().min(1).optional(), // Additional Cohere API key for load balancing
|
||||
COHERE_API_KEY_3: z.string().min(1).optional(), // Additional Cohere API key for load balancing
|
||||
ELEVENLABS_API_KEY: z.string().min(1).optional(), // ElevenLabs API key for text-to-speech in deployed chat
|
||||
ELEVENLABS_API_KEY: z.string().min(1).optional(), // ElevenLabs API key for workspace speech-to-text
|
||||
SERPER_API_KEY: z.string().min(1).optional(), // Serper API key for online search
|
||||
EXA_API_KEY: z.string().min(1).optional(), // Exa AI API key for enhanced online search
|
||||
BLACKLISTED_PROVIDERS: z.string().optional(), // Comma-separated provider IDs to hide (e.g., "openai,anthropic")
|
||||
|
||||
@@ -20,8 +20,3 @@ export function getClientIp(request: { headers: { get(name: string): string | nu
|
||||
'unknown'
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* No-operation function for use as default callback
|
||||
*/
|
||||
export const noop = () => {}
|
||||
|
||||
@@ -4,7 +4,6 @@ export const ELEVENLABS_WS_URL = 'wss://api.elevenlabs.io/v1/speech-to-text/real
|
||||
export const SAMPLE_RATE = 16000
|
||||
export const CHUNK_SEND_INTERVAL_MS = 250
|
||||
export const MAX_SESSION_MS = 3 * 60 * 1000
|
||||
export const MAX_CHAT_SESSION_MS = 1 * 60 * 1000
|
||||
|
||||
/**
|
||||
* Whether a speech-to-text provider is configured.
|
||||
|
||||
Reference in New Issue
Block a user