fix(stage-tamagotchi): incorrect state managed for tamagotchi (desktop) hearing module

2026-08-31 01:13:48 +08:00 · 2026-03-27 17:58:45 +08:00
parent 9d983bb2ea
commit 2a9ae94239
3 changed files with 162 additions and 69 deletions
@@ -12,7 +12,6 @@ const show = defineModel('show', { type: Boolean, default: false })

 const settingsAudioDeviceStore = useSettingsAudioDevice()
 const { enabled, selectedAudioInput, stream, audioInputs } = storeToRefs(settingsAudioDeviceStore)
-const { startStream, stopStream } = settingsAudioDeviceStore

 const getMediaAccessStatus = useElectronEventaInvoke(electron.systemPreferences.getMediaAccessStatus)
 const { state: mediaAccessStatus, execute: refreshMediaAccessStatus } = useAsyncState(() => getMediaAccessStatus(['microphone']), 'not-determined')
@@ -20,13 +19,25 @@ const { state: mediaAccessStatus, execute: refreshMediaAccessStatus } = useAsync
 const { audioContext, initialize, dispose, pause } = useAudioContextFromStream(stream)
 const { volumeLevel, startAnalyzer, stopAnalyzer } = useAudioAnalyzer()

-watch(enabled, (val) => {
-  if (val) {
-    startStream()
-    initialize().then(() => startAnalyzer(audioContext.value!))
+// NOTICE: Do not call `startStream()` / `stopStream()` from this component.
+//
+// `useSettingsAudioDevice()` already owns the mic stream lifecycle via the persisted `enabled` state.
+// We previously toggled the stream here as well, which introduced a second lifecycle controller: the
+// dialog could recreate the MediaStream while the page-level transcription pipeline still believed
+// the old session was active.
+//
+// That produced the "VAD still works, but no transcript arrives" failure after retoggling the mic.
+//
+// This component should only react to the current stream to drive analyzer UI state.
+watch([enabled, stream], ([isEnabled, currentStream]) => {
+  if (isEnabled && currentStream) {
+    initialize().then(() => {
+      if (audioContext.value)
+        return startAnalyzer(audioContext.value)
+    })
  }
  else {
-    stopStream()
+    stopAnalyzer()
    pause()
  }
 }, { immediate: true })
@@ -240,6 +240,7 @@ const { init: initVAD, dispose: disposeVAD, start: startVAD, loaded: vadLoaded }
 })

 let stopOnStopRecord: (() => void) | undefined
+const audioInteractionStarting = ref(false)

 // Caption overlay broadcast channel
 type CaptionChannelEvent
@@ -247,6 +248,37 @@ type CaptionChannelEvent
    | { type: 'caption-assistant', text: string }
 const { post: postCaption } = useBroadcastChannel<CaptionChannelEvent, CaptionChannelEvent>({ name: 'airi-caption-overlay' })

+function handleStreamingSentenceEnd(delta: string) {
+  console.info('[Main Page] Received transcription delta:', delta)
+  const finalText = delta
+  if (!finalText || !finalText.trim()) {
+    return
+  }
+
+  postCaption({ type: 'caption-speaker', text: finalText })
+
+  void (async () => {
+    try {
+      const provider = await providersStore.getProviderInstance(activeChatProvider.value)
+      if (!provider || !activeChatModel.value) {
+        console.warn('[Main Page] No provider or model available, skipping chat send')
+        return
+      }
+
+      console.info('[Main Page] Sending transcription to chat:', finalText)
+      await chatStore.ingest(finalText, { model: activeChatModel.value, chatProvider: provider as ChatProvider })
+    }
+    catch (err) {
+      console.error('[Main Page] Failed to send chat from voice:', err)
+    }
+  })()
+}
+
+function handleStreamingSpeechEnd(text: string) {
+  console.info('[Main Page] Speech ended, final text:', text)
+  postCaption({ type: 'caption-speaker', text })
+}
+
 async function handleSpeechStart() {
  if (shouldUseStreamInput.value) {
    console.info('Speech detected - transcription session should already be active')
@@ -266,6 +298,19 @@ async function handleSpeechEnd() {
 }

 async function startAudioInteraction() {
+  if (audioInteractionStarting.value)
+    return
+
+  // NOTICE: `stopOnStopRecord` only tracks whether the non-stream recording hook was registered.
+  //
+  // It does NOT guarantee that the current realtime transcription session is still attached to the
+  // latest `MediaStream`. We previously used it as a generic "already started" guard, which broke
+  // the hearing-config retoggle path: the mic stream was recreated, VAD restarted on the new stream,
+  // but `transcribeForMediaStream()` never reattached so speech was detected without any transcript.
+  //
+  // Keep the startup guard scoped to "startup in progress" only, and let stream changes restart the
+  // transcription binding when a new stream arrives.
+  audioInteractionStarting.value = true
  try {
    console.info('[Main Page] Starting audio interaction...')

@@ -291,35 +336,8 @@ async function startAudioInteraction() {

      // Use sentence deltas for live captions and speech end for final text.
      await transcribeForMediaStream(stream.value, {
-        onSentenceEnd: (delta) => {
-          console.info('[Main Page] Received transcription delta:', delta)
-          const finalText = delta
-          if (!finalText || !finalText.trim()) {
-            return
-          }
-
-          postCaption({ type: 'caption-speaker', text: finalText })
-
-          void (async () => {
-            try {
-              const provider = await providersStore.getProviderInstance(activeChatProvider.value)
-              if (!provider || !activeChatModel.value) {
-                console.warn('[Main Page] No provider or model available, skipping chat send')
-                return
-              }
-
-              console.info('[Main Page] Sending transcription to chat:', finalText)
-              await chatStore.ingest(finalText, { model: activeChatModel.value, chatProvider: provider as ChatProvider })
-            }
-            catch (err) {
-              console.error('[Main Page] Failed to send chat from voice:', err)
-            }
-          })()
-        },
-        onSpeechEnd: (text) => {
-          console.info('[Main Page] Speech ended, final text:', text)
-          postCaption({ type: 'caption-speaker', text })
-        },
+        onSentenceEnd: handleStreamingSentenceEnd,
+        onSpeechEnd: handleStreamingSpeechEnd,
      })

      console.info('[Main Page] Streaming transcription started successfully')
@@ -332,45 +350,58 @@ async function startAudioInteraction() {
      })
    }

-    // Hook once
-    stopOnStopRecord = onStopRecord(async (recording) => {
-      if (shouldUseStreamInput.value)
-        return
-
-      const text = await transcribeForRecording(recording)
-      if (!text || !text.trim())
-        return
-
-      // Update caption overlay speaker text via BroadcastChannel
-      postCaption({ type: 'caption-speaker', text })
-
-      try {
-        const provider = await providersStore.getProviderInstance(activeChatProvider.value)
-        if (!provider || !activeChatModel.value)
+    // NOTICE: This hook is only for record-then-transcribe providers.
+    //
+    // Streaming providers use the active `MediaStream` directly, so this callback must not be treated
+    // as proof that a realtime session is alive. Future refactors should keep recorder-hook bookkeeping
+    // separate from stream transcription state, otherwise mic/device re-toggles can leave VAD active
+    // but transcription detached.
+    //
+    // Hook once for non-streaming providers.
+    if (!stopOnStopRecord) {
+      stopOnStopRecord = onStopRecord(async (recording) => {
+        if (shouldUseStreamInput.value)
          return

-        await chatStore.ingest(text, { model: activeChatModel.value, chatProvider: provider as ChatProvider })
-      }
-      catch (err) {
-        console.error('Failed to send chat from voice:', err)
-      }
-    })
+        const text = await transcribeForRecording(recording)
+        if (!text || !text.trim())
+          return
+
+        // Update caption overlay speaker text via BroadcastChannel
+        postCaption({ type: 'caption-speaker', text })
+
+        try {
+          const provider = await providersStore.getProviderInstance(activeChatProvider.value)
+          if (!provider || !activeChatModel.value)
+            return
+
+          await chatStore.ingest(text, { model: activeChatModel.value, chatProvider: provider as ChatProvider })
+        }
+        catch (err) {
+          console.error('Failed to send chat from voice:', err)
+        }
+      })
+    }
  }
  catch (e) {
    console.error('Audio interaction init failed:', e)
  }
+  finally {
+    audioInteractionStarting.value = false
+  }
 }

 function stopAudioInteraction() {
  tryCatch(() => {
    stopOnStopRecord?.()
    stopOnStopRecord = undefined
+    audioInteractionStarting.value = false
    void stopStreamingTranscription(true)
    disposeVAD()
  })
 }

-watch([enabled, stream], async ([val]) => {
+watch(enabled, async (val) => {
  console.info('[Main Page] Audio enabled changed:', val, 'stream available:', !!stream.value)
  if (val) {
    await askPermission()
@@ -395,6 +426,18 @@ onUnmounted(() => {
  stopAudioInteraction()
 })

+watch(stream, async (currentStream) => {
+  if (!enabled.value || !currentStream || audioInteractionStarting.value)
+    return
+
+  // NOTICE: The controls-island mic toggle and device changes can replace the underlying MediaStream
+  // without reloading the page. When that happens, VAD may successfully restart against the new stream,
+  // but any existing transcription transport is still bound to the old one. Always allow the page to
+  // re-run `startAudioInteraction()` for a newly available stream unless startup is already underway.
+  console.info('[Main Page] Stream became available, ensuring audio interaction is started')
+  await startAudioInteraction()
+})
+
 watch([stream, () => vadLoaded.value], async ([s, loaded]) => {
  if (enabled.value && loaded && s) {
    try {
@@ -2,7 +2,7 @@ import type { TranscriptionProviderWithExtraOptions } from '@xsai-ext/providers/
 import type { WithUnknown } from '@xsai/shared'
 import type { StreamTranscriptionResult, StreamTranscriptionOptions as XSAIStreamTranscriptionOptions } from '@xsai/stream-transcription'

-import { tryCatch } from '@moeru/std'
+import { errorMessageFrom, tryCatch } from '@moeru/std'
 import { useLocalStorageManualReset } from '@proj-airi/stage-shared/composables'
 import { refManualReset } from '@vueuse/core'
 import { generateTranscription } from '@xsai/generate-transcription'
@@ -16,7 +16,7 @@ import { streamAliyunTranscription } from '../providers/aliyun/stream-transcript
 import { streamWebSpeechAPITranscription } from '../providers/web-speech-api'

 function errorMessage(err: unknown): string {
-  const msg = err instanceof Error ? err.message : String(err)
+  const msg = errorMessageFrom(err) ?? String(err)
  // Browsers hide the real reason (CORS, timeout, DNS, …) behind this generic string.
  if (msg === 'Failed to fetch' || msg === 'Load failed') {
    return `${msg} — check the browser console (Network tab) for the exact reason (e.g. CORS, network timeout, DNS failure).`
@@ -24,6 +24,33 @@ function errorMessage(err: unknown): string {
  return msg
 }

+// NOTICE: Realtime transcription intentionally uses `AbortError` as a control-flow signal when the
+// current stream session is being stopped on purpose.
+//
+// This happens in `stopStreamingTranscription()`,
+// which aborts the session with one of the DOMException messages below when the user disables the mic,
+// the page tears down audio interaction, callbacks are intentionally rebound, or the idle timeout closes
+// an inactive stream. Those cases should not be surfaced as provider failures because the session was
+// explicitly asked to stop. If a future abort is noisy or unexpected, inspect the abort source first:
+// `stopStreamingTranscription()` in this file is the primary origin, and provider-specific teardown
+// bridges such as `packages/stage-ui/src/stores/providers/aliyun/stream-transcription.ts` propagate the
+// same reason through the transport. Only treat an abort as "expected" if it is one of these known
+// shutdown paths; any other `AbortError` should still be investigated as a real lifecycle bug or a
+// provider/runtime failure.
+function isExpectedStreamStopError(err: unknown): boolean {
+  return err instanceof DOMException
+    && err.name === 'AbortError'
+    && (err.message === 'Stopped' || err.message === 'Aborted' || err.message === 'Closed' || err.message === 'Idle timeout')
+}
+
+function haveStreamingCallbacksChanged(
+  previous: { onSentenceEnd?: (delta: string) => void, onSpeechEnd?: (text: string) => void } | undefined,
+  next: { onSentenceEnd?: (delta: string) => void, onSpeechEnd?: (text: string) => void },
+): boolean {
+  return (next.onSentenceEnd !== undefined && next.onSentenceEnd !== previous?.onSentenceEnd)
+    || (next.onSpeechEnd !== undefined && next.onSpeechEnd !== previous?.onSpeechEnd)
+}
+
 export interface StreamTranscriptionFileInputOptions extends Omit<XSAIStreamTranscriptionOptions, 'file' | 'fileName'> {
  file: Blob
  fileName?: string
@@ -357,6 +384,9 @@ export const useHearingSpeechInputPipeline = defineStore('modules:hearing:speech
          return text
        }
        catch (err) {
+          if (isExpectedStreamStopError(err))
+            return
+
          error.value = errorMessage(err)
          console.error('Error getting transcription result:', error.value)
        }
@@ -402,6 +432,9 @@ export const useHearingSpeechInputPipeline = defineStore('modules:hearing:speech
        return text
      }
      catch (err) {
+        if (isExpectedStreamStopError(err))
+          return
+
        error.value = errorMessage(err)
        console.error('Error generating transcription:', error.value)
      }
@@ -460,15 +493,15 @@ export const useHearingSpeechInputPipeline = defineStore('modules:hearing:speech
        // Check if session already exists and reuse it
        const existingSession = streamingSession.value
        if (existingSession && existingSession.providerId === 'browser-web-speech-api') {
+          const nextCallbacks = {
+            onSentenceEnd: options?.onSentenceEnd,
+            onSpeechEnd: options?.onSpeechEnd,
+          }
          // For Web Speech API, if callbacks are provided and different, we need to restart
          // because recognition instance callbacks are set once and can't be changed
-          // However, if no new callbacks are provided, we can just reuse the session
-          const hasNewCallbacks = !!(options?.onSentenceEnd || options?.onSpeechEnd)
+          const hasNewCallbacks = haveStreamingCallbacksChanged(existingSession.callbacks, nextCallbacks)

          if (hasNewCallbacks) {
-            // We need to restart to use new callbacks, but only if they're actually different
-            // Since we can't compare functions, we'll just always restart if new callbacks are provided
-            // This ensures callbacks are always up-to-date
            console.info('Web Speech API: New callbacks provided, restarting session to use them')
            await stopStreamingTranscription(false, existingSession.providerId)
            // Continue to create new session below
@@ -580,7 +613,8 @@ export const useHearingSpeechInputPipeline = defineStore('modules:hearing:speech
              }
            }
            catch (err) {
-              console.error('Error reading text stream:', err)
+              if (!isExpectedStreamStopError(err))
+                console.error('Error reading text stream:', err)
            }
          })()
        }
@@ -600,9 +634,10 @@ export const useHearingSpeechInputPipeline = defineStore('modules:hearing:speech
      // require restarting the session to create a new reader.
      const existingSession = streamingSession.value
      if (existingSession) {
-        const hasNewCallbacks
-          = options?.onSentenceEnd !== undefined
-            || options?.onSpeechEnd !== undefined
+        const hasNewCallbacks = haveStreamingCallbacksChanged(existingSession.callbacks, {
+          onSentenceEnd: options?.onSentenceEnd,
+          onSpeechEnd: options?.onSpeechEnd,
+        })

        if (hasNewCallbacks) {
          console.info('[Hearing Pipeline] New callbacks provided, restarting session')
@@ -699,7 +734,8 @@ export const useHearingSpeechInputPipeline = defineStore('modules:hearing:speech
            }
          }
          catch (err) {
-            console.error('Error reading text stream:', err)
+            if (!isExpectedStreamStopError(err))
+              console.error('Error reading text stream:', err)
          }
          finally {
            // Use captured callbacks to avoid cross-session leakage
@@ -709,6 +745,9 @@ export const useHearingSpeechInputPipeline = defineStore('modules:hearing:speech
      }
    }
    catch (err) {
+      if (isExpectedStreamStopError(err))
+        return
+
      error.value = errorMessage(err)
      console.error('Error generating transcription:', error.value)
    }