fix(stage-ui): let optional modules switch off without losing settings (#3174)

This commit is contained in:
jim139129
2026-10-28 17:38:03 +08:00
committed by GitHub
parent 672fd6693a
commit 5ea6aa50a3
31 changed files with 758 additions and 66 deletions
@@ -671,6 +671,8 @@ pages:
description: Configure 3D VRM models and settings
scene: Scene
modules:
availability:
none: None
beat_sync:
description: Vibe with beats from the audio source
title: Beat Sync
+2
View File
@@ -21,6 +21,8 @@ mobile-tools:
menu: Menu
menu-unread: Menu, {count} unread notifications
chat:
audio:
hearing-disabled: '[Audio recording: transcription is disabled.]'
provider-configuration:
title: Chat provider isn't configured
description: Choose and validate a provider in Settings before sending a message.
@@ -632,6 +632,8 @@ pages:
description: 配置 3D VRM 模型
scene: 场景
modules:
availability:
none: 无
beat_sync:
description: 随音源节拍律动
title: 同步音律
@@ -21,6 +21,8 @@ mobile-tools:
menu: 菜单
menu-unread: 菜单,{count} 条未读通知
chat:
audio:
hearing-disabled: '[语音录音:转写已停用。]'
provider-configuration:
title: 尚未正确配置聊天 Provider
description: 请先前往设置页面选择并验证一个 Provider,再发送消息。
@@ -1,6 +1,7 @@
import en from '@proj-airi/i18n/locales/en'
import { pushToTalkTriggerKey } from '@proj-airi/stage-ui/composables/push-to-talk-trigger'
import { useHearingStore } from '@proj-airi/stage-ui/stores/modules/hearing'
import { createPinia, disposePinia } from 'pinia'
import { afterEach, describe, expect, it, vi } from 'vitest'
import { cleanup, render } from 'vitest-browser-vue'
@@ -49,6 +50,24 @@ async function openInputModeSelect(screen: Awaited<ReturnType<typeof renderHeari
}
describe('hearing overview', () => {
// https://github.com/moeru-ai/airi/issues/3153
// ROOT CAUSE:
// The overview lacked a module switch. The switch now preserves the selected provider and model.
it('issue #3153: switches hearing off and on without clearing its selection', async () => {
localStorage.setItem('settings/hearing/active-provider', 'browser-web-speech-api')
localStorage.setItem('settings/hearing/active-model', 'web-speech-api')
const screen = await renderHearing(true)
const hearing = useHearingStore(pinias[pinias.length - 1])
const toggle = screen.getByRole('switch', { name: 'Hearing', exact: true })
await expect.element(toggle).toBeChecked()
await toggle.click()
expect(hearing.configured).toBe(false)
expect(hearing.activeTranscriptionProvider).toBe('browser-web-speech-api')
expect(hearing.activeTranscriptionModel).toBe('web-speech-api')
await toggle.click()
expect(hearing.configured).toBe(true)
})
it('links to the speech recognition and wake words pages', async () => {
const screen = await renderHearing(true)
@@ -65,6 +65,12 @@ const pages = computed(() => [
<template>
<div :class="['flex flex-col gap-6']">
<SettingsCard>
<FieldCheckbox
v-model="hearingStore.enabled"
:label="t('settings.pages.modules.hearing.title')"
/>
</SettingsCard>
<RippleGrid :items="pages" :columns="{ default: 1, sm: 2 }">
<template #item="{ item: page }">
<IconStatusItem
@@ -1,8 +1,10 @@
import en from '@proj-airi/i18n/locales/en'
import { useHearingStore } from '@proj-airi/stage-ui/stores/modules/hearing'
import { createPinia, disposePinia } from 'pinia'
import { afterEach, describe, expect, it, vi } from 'vitest'
import { cleanup, render } from 'vitest-browser-vue'
import { page } from 'vitest/browser'
import { createI18n } from 'vue-i18n'
import { createMemoryHistory, createRouter } from 'vue-router'
@@ -36,6 +38,32 @@ async function renderTranscriber() {
}
describe('speech recognition settings', () => {
// https://github.com/moeru-ai/airi/issues/3153
// ROOT CAUSE:
// Provider radio cards had no disabled choice. None now disables Hearing and retains the provider and model.
it.each(['provider', 'switch'] as const)('issue #3153: disables recognition through the %s control without deleting the provider or model', async (control) => {
localStorage.setItem('settings/hearing/active-provider', 'browser-web-speech-api')
localStorage.setItem('settings/hearing/active-model', 'web-speech-api')
const screen = await renderTranscriber()
const hearing = useHearingStore(pinias[pinias.length - 1])
expect(hearing.configured).toBe(true)
const noneCard = screen.getByRole('radio', { name: 'None', exact: true }).element().closest('label')!
const toggle = screen.getByRole('switch', { name: 'Hearing', exact: true })
if (control === 'switch')
await toggle.click()
else
await page.elementLocator(noneCard).click()
expect(hearing.configured).toBe(false)
expect(hearing.activeTranscriptionProvider).toBe('browser-web-speech-api')
expect(hearing.activeTranscriptionModel).toBe('web-speech-api')
await expect.element(screen.getByRole('radio', { name: 'None', exact: true })).toBeChecked()
await expect.element(toggle).not.toBeChecked()
await toggle.click()
expect(hearing.configured).toBe(true)
expect(hearing.activeTranscriptionModel).toBe('web-speech-api')
})
it('shows the microphone, the providers, and the playground without the overview controls', async () => {
const screen = await renderTranscriber()
@@ -25,6 +25,14 @@ import { moduleModelSearchKey, moduleProviderSearchKey } from '../module-model-s
const { t } = useI18n()
const hearingStore = useHearingStore()
const selectedProvider = computed({
get: () => hearingStore.enabled ? hearingStore.activeTranscriptionProvider : '',
set: (provider: string) => {
hearingStore.enabled = !!provider
if (provider)
hearingStore.activeTranscriptionProvider = provider
},
})
const {
activeTranscriptionProvider,
activeTranscriptionModel,
@@ -130,6 +138,8 @@ const isSpeech = computed(() => inputState.value?.phase === 'capturing')
const isSpeechVolume = isSpeech
async function setupAudioMonitoring() {
if (!hearingStore.enabled)
return
const stopped = stopAudioMonitoring()
const startup = new AbortController()
monitoringStartup = startup
@@ -287,7 +297,7 @@ provide(hearingProviderViewContextKey, {
// Sync OpenAI Compatible model from provider config
function syncOpenAICompatibleSettings() {
if (activeTranscriptionProvider.value !== 'openai-compatible-audio-transcription')
if (!hearingStore.enabled || activeTranscriptionProvider.value !== 'openai-compatible-audio-transcription')
return
const providerConfig = providerStore.getProviderConfig(activeTranscriptionProvider.value)
@@ -311,7 +321,9 @@ watch([selectedAudioInput, useVADModel], async () => {
await setupAudioMonitoring()
})
watch(activeTranscriptionProvider, async (provider) => {
watch(selectedProvider, async (provider, _previous, onCleanup) => {
let cancelled = false
onCleanup(() => cancelled = true)
const shouldRestartMonitoring = isMonitoring.value
if (shouldRestartMonitoring) {
@@ -320,11 +332,15 @@ watch(activeTranscriptionProvider, async (provider) => {
}
clearPlaygroundSegments()
error.value = ''
vadModelError.value = ''
if (!provider)
return
await hearingStore.loadModelsForProvider(provider)
if (cancelled)
return
syncOpenAICompatibleSettings()
const models = providerModels.value
@@ -348,6 +364,7 @@ onUnmounted(() => {
<template>
<div :class="['flex flex-col gap-6 md:flex-row']">
<SettingsCard :class="['md:w-[40%]']">
<FieldCheckbox v-model="hearingStore.enabled" :label="t('settings.pages.modules.hearing.title')" />
<div :class="['flex flex-col gap-4']">
<MicrophonePicker
v-model="selectedAudioInput"
@@ -375,15 +392,21 @@ onUnmounted(() => {
See also: https://stackoverflow.com/a/33737340
-->
<fieldset
v-if="moduleTranscriptionProvidersMetadata.length > 0"
:class="['flex flex-row gap-4 min-w-0 overflow-x-auto scroll-smooth']"
role="radiogroup"
>
<RadioCardSimple
id="hearing-none"
v-model="selectedProvider"
name="provider"
value=""
:title="t('settings.pages.modules.availability.none')"
/>
<RadioCardSimple
v-for="metadata in moduleTranscriptionProvidersMetadata"
:id="metadata.id"
:key="metadata.id"
v-model="activeTranscriptionProvider"
v-model="selectedProvider"
:data-settings-search="moduleProviderSearchKey('transcription', metadata.id)"
name="provider"
:value="metadata.id"
@@ -407,7 +430,7 @@ onUnmounted(() => {
/>
</RouterLink>
</fieldset>
<div v-else>
<div v-if="moduleTranscriptionProvidersMetadata.length === 0">
<RouterLink
to="/settings/providers"
:class="[
@@ -430,11 +453,11 @@ onUnmounted(() => {
<component
:is="activeProviderHearingView"
v-if="activeProviderHearingView"
v-if="hearingStore.enabled && activeProviderHearingView"
/>
<!-- Model selection section -->
<div v-if="activeTranscriptionProvider">
<div v-if="selectedProvider">
<div :class="['flex flex-col gap-4']">
<div>
<h2 :class="['text-lg md:text-2xl']">
@@ -514,7 +537,7 @@ onUnmounted(() => {
</div>
<!-- Confidence threshold (only for non-streaming providers) -->
<div v-if="!supportsStreamInput" :class="['border-t border-neutral-200 pt-4 dark:border-neutral-700']">
<div v-if="hearingStore.enabled && !supportsStreamInput" :class="['border-t border-neutral-200 pt-4 dark:border-neutral-700']">
<div :class="['mb-4']">
<h2 :class="['text-lg text-neutral-500 md:text-2xl dark:text-neutral-500']">
{{ t('settings.pages.modules.hearing.sections.section.confidence-threshold.title') }}
@@ -558,7 +581,7 @@ onUnmounted(() => {
</p>
<ErrorContainer
v-if="error || transcriptionPipelineError"
v-if="hearingStore.enabled && (error || transcriptionPipelineError)"
:title="t('settings.pages.modules.hearing.sections.section.playground.error-title')"
:error="error || transcriptionPipelineError"
:class="['mb-4']"
@@ -568,7 +591,7 @@ onUnmounted(() => {
<Button
:class="['mb-4', 'w-full']"
data-testid="hearing-playground-monitor-toggle"
:disabled="!activeTranscriptionProvider"
:disabled="!selectedProvider"
@click="toggleMonitoring"
>
{{ isMonitoring
@@ -38,7 +38,7 @@ const providersStore = useProviderStore()
const providerStore = useProviderConfigStore()
const speechStore = useSpeechStore()
const airiCardStore = useAiriCardStore()
const { allAudioSpeechProvidersMetadata, moduleSpeechProvidersMetadata } = storeToRefs(providersStore)
const { moduleSpeechProvidersMetadata } = storeToRefs(providersStore)
const {
activeSpeechProvider,
activeSpeechModel,
@@ -75,6 +75,18 @@ const audioUrl = ref('')
const audioPlayer = ref<HTMLAudioElement | null>(null)
const errorMessage = ref('')
let lastOfficialTtsExposureKey = ''
let previewCancellation = new AbortController()
watch(() => speechStore.enabled, (enabled) => {
if (enabled) {
previewCancellation = new AbortController()
}
else {
previewCancellation.abort()
stopTestAudio()
errorMessage.value = ''
}
}, { immediate: true, flush: 'sync' })
const STREAMING_MODEL_OPTION_PREFIX = 'streaming:'
@@ -92,20 +104,20 @@ const selectableSpeechSources = computed(() => {
}))
return [
{
id: 'speech-noop',
providerId: 'speech-noop',
title: t('settings.pages.modules.availability.none'),
description: undefined,
},
...configuredSources,
...allAudioSpeechProvidersMetadata.value
.filter(metadata => metadata.id === 'speech-noop')
.map(metadata => ({
id: metadata.id,
providerId: metadata.id,
title: metadata.localizedName || 'Unknown',
description: metadata.localizedDescription,
})),
]
})
const displayedSpeechSource = computed({
get: () => {
if (!speechStore.enabled)
return 'speech-noop'
if (activeSpeechProvider.value === OFFICIAL_SPEECH_STREAMING_PROVIDER_ID)
return OFFICIAL_SPEECH_PROVIDER_ID
return activeSpeechProvider.value
@@ -299,9 +311,12 @@ async function selectSpeechVoice(voiceId: string | undefined) {
/** Persists the selection only after the leader commits its provider and model. */
async function selectSpeechSource(sourceId: string) {
const selection = await speechStore.selectProviderModel(sourceId, '')
if (!selection)
const selection = await speechStore.selectSource(sourceId)
if (!selection) {
stopTestAudio()
errorMessage.value = ''
return
}
const providerId = providerStore.providers[sourceId]?.definitionId || sourceId
// Use this command's receipt: another selection can reach the store before
// this caller resumes, but must not relabel this analytics event.
@@ -310,7 +325,8 @@ async function selectSpeechSource(sourceId: string) {
tts_model_id: selection.model || 'unknown',
source: 'settings',
})
await persistSelection()
// A follower can receive the action receipt before its state snapshot. Persist the committed selection.
await airiCardStore.updateActiveCardSpeech(selection)
}
/** Resolves the displayed model option before committing it in the leader. */
@@ -348,7 +364,7 @@ function trackOfficialTtsExposure(providerId = activeSpeechProvider.value, model
/** Applies provider defaults in the leader without a follower state proposal. */
async function syncOpenAICompatibleSettings() {
if (activeSpeechProvider.value !== 'openai-compatible-audio-speech')
if (!speechStore.enabled || activeSpeechProvider.value !== 'openai-compatible-audio-speech')
return
const providerConfig = providerStore.getProviderConfig(activeSpeechProvider.value)
@@ -412,13 +428,18 @@ async function persistSelection() {
// Function to generate speech
async function generateTestSpeech() {
if (!speechStore.enabled)
return
const signal = previewCancellation.signal
if (!testText.value.trim() && !useSSML.value)
return
if (useSSML.value && !ssmlText.value.trim())
return
const provider = await providersStore.getProviderInstance(activeSpeechProvider.value) as SpeechProviderWithExtraOptions<string, any>
const provider = await providersStore.getProviderInstance<SpeechProviderWithExtraOptions<string, Record<string, unknown>>>(activeSpeechProvider.value)
if (signal.aborted)
return
if (!provider) {
console.error('Failed to initialize speech provider')
return
@@ -504,8 +525,11 @@ async function generateTestSpeech() {
),
input: speechRequest.input,
voice: voice.id,
abortSignal: signal,
})
signal.throwIfAborted()
// Convert the response to a blob and create an object URL
audioUrl.value = URL.createObjectURL(new Blob([response]))
if (isOfficialTtsProvider(previewProvider)) {
@@ -520,7 +544,7 @@ async function generateTestSpeech() {
// Play the audio
setTimeout(() => {
if (audioPlayer.value) {
if (!signal.aborted && audioPlayer.value) {
void audioPlayer.value.play()
.then(() => {
trackVoicePreviewPlayed({
@@ -535,6 +559,8 @@ async function generateTestSpeech() {
}, 100)
}
catch (error) {
if (signal.aborted)
return
console.error('Error generating speech:', error)
errorMessage.value = errorMessageFrom(error) || 'An unknown error occurred'
}
@@ -559,9 +585,8 @@ function stopTestAudio() {
// Clean up when component is unmounted
onUnmounted(() => {
if (audioUrl.value) {
URL.revokeObjectURL(audioUrl.value)
}
previewCancellation.abort()
stopTestAudio()
})
function updateCustomVoiceName(value: string | undefined) {
@@ -615,6 +640,7 @@ async function handleDeleteProvider(providerId: string) {
<ErrorContainer v-if="errorMessage" :error="errorMessage" />
<div flex="~ col md:row gap-6">
<SettingsCard class="md:w-[40%]">
<FieldCheckbox v-model="speechStore.enabled" :label="t('settings.pages.modules.speech.title')" />
<div flex="~ col gap-4">
<div>
<h2 class="text-lg text-neutral-500 md:text-2xl dark:text-neutral-400">
@@ -686,7 +712,7 @@ async function handleDeleteProvider(providerId: string) {
</div>
<!-- Model selection section -->
<div v-if="activeSpeechProvider && activeSpeechProvider !== 'speech-noop'">
<div v-if="speechStore.enabled && activeSpeechProvider && activeSpeechProvider !== 'speech-noop'">
<div flex="~ col gap-4">
<div>
<h2 class="text-lg md:text-2xl">
@@ -779,7 +805,7 @@ async function handleDeleteProvider(providerId: string) {
</div>
<!-- Voice Configuration Section -->
<div v-if="activeSpeechProvider && activeSpeechProvider !== 'speech-noop'">
<div v-if="speechStore.enabled && activeSpeechProvider && activeSpeechProvider !== 'speech-noop'">
<div flex="~ col gap-4">
<div>
<h2 class="text-lg text-neutral-500 md:text-2xl dark:text-neutral-400">
@@ -967,8 +993,8 @@ async function handleDeleteProvider(providerId: string) {
<button
border="neutral-800 dark:neutral-200 solid 2" transition="border duration-250 ease-in-out"
rounded-lg px-4 text="neutral-100 dark:neutral-900" py-2 text-sm
:disabled="isGenerating || (!testText.trim() && !useSSML) || (useSSML && !ssmlText.trim()) || !activeSpeechVoice"
:class="{ 'opacity-50 cursor-not-allowed': isGenerating || (!testText.trim() && !useSSML) || (useSSML && !ssmlText.trim()) || !activeSpeechVoice }"
:disabled="!speechStore.enabled || isGenerating || (!testText.trim() && !useSSML) || (useSSML && !ssmlText.trim()) || !activeSpeechVoice"
:class="{ 'opacity-50 cursor-not-allowed': !speechStore.enabled || isGenerating || (!testText.trim() && !useSSML) || (useSSML && !ssmlText.trim()) || !activeSpeechVoice }"
bg="neutral-700 dark:neutral-300" @click="generateTestSpeech"
>
<div flex="~ row" items-center gap-2>
@@ -0,0 +1,51 @@
import en from '@proj-airi/i18n/locales/en'
import { useVisionStore } from '@proj-airi/stage-ui/stores/modules/vision'
import { createPinia, disposePinia } from 'pinia'
import { expect, it, onTestFinished } from 'vitest'
import { render } from 'vitest-browser-vue'
import { page } from 'vitest/browser'
import { createI18n } from 'vue-i18n'
import { createMemoryHistory, createRouter } from 'vue-router'
import VisionPage from './vision.vue'
import 'virtual:uno.css'
// https://github.com/moeru-ai/airi/issues/3153
// ROOT CAUSE:
// Provider cards lacked a disabled choice. None now remains available even without configured providers.
it.each(['provider', 'switch'] as const)('issue #3153: disables vision through the %s control even when the selected provider is unavailable', async (control) => {
localStorage.clear()
localStorage.setItem('settings/vision/active-provider', 'unavailable-vision')
localStorage.setItem('settings/vision/active-model', 'saved-model')
const pinia = createPinia()
const router = createRouter({ history: createMemoryHistory(), routes: [{ path: '/:path(.*)*', component: { template: '<div />' } }] })
await router.push('/settings/modules/vision')
const view = render(VisionPage, {
global: { plugins: [pinia, router, createI18n({ legacy: false, locale: 'en', messages: { en } })] },
})
onTestFinished(() => {
view.unmount()
disposePinia(pinia)
localStorage.clear()
})
const noneCard = view.getByRole('radio', { name: 'None', exact: true }).element().closest('label')!
const toggle = view.getByRole('switch', { name: 'Vision', exact: true })
if (control === 'switch')
await toggle.click()
else
await page.elementLocator(noneCard).click()
await expect.element(view.getByRole('radio', { name: 'None', exact: true })).toBeChecked()
const vision = useVisionStore(pinia)
expect(vision.enabled).toBe(false)
expect(vision.configured).toBe(false)
expect(vision.activeProvider).toBe('unavailable-vision')
expect(vision.activeModel).toBe('saved-model')
await expect.element(toggle).not.toBeChecked()
await toggle.click()
expect(vision.enabled).toBe(true)
expect(vision.activeProvider).toBe('unavailable-vision')
expect(vision.activeModel).toBe('saved-model')
})
@@ -56,9 +56,15 @@ watch(activeProvider, async (provider) => {
}, { immediate: true })
async function selectProvider(provider: string) {
if (!provider) {
visionStore.enabled = false
return
}
await airiCardStore.selectActiveCardVisionProvider(provider)
}
const selectedProvider = computed(() => visionStore.enabled ? activeProvider.value : '')
async function persistSelection() {
await airiCardStore.updateActiveCardVision({ provider: activeProvider.value, model: activeModel.value })
}
@@ -106,6 +112,7 @@ function formatRelativeTime(timestamp: number | null) {
<template>
<div :class="['flex', 'flex-col', 'gap-6']">
<SettingsCard>
<FieldCheckbox v-model="visionStore.enabled" :label="t('settings.pages.modules.vision.title')" />
<FieldCheckbox
v-model="useForChat"
:label="t('stage.chat.images.use-vision')"
@@ -130,16 +137,23 @@ function formatRelativeTime(timestamp: number | null) {
</div>
<div :class="['max-w-full']">
<fieldset
v-if="moduleVisionProvidersMetadata.length > 0"
:class="['flex', 'min-w-0', 'flex-row', 'gap-4', 'overflow-x-auto', 'scroll-smooth']"
role="radiogroup"
>
<RadioCardSimple
id="vision-none"
:model-value="selectedProvider"
name="provider"
value=""
:title="t('settings.pages.modules.availability.none')"
@update:model-value="selectProvider"
/>
<RadioCardSimple
v-for="metadata in moduleVisionProvidersMetadata"
:id="metadata.id"
:key="metadata.id"
:data-settings-search="moduleProviderSearchKey('vision', metadata.id)"
:model-value="activeProvider"
:model-value="selectedProvider"
name="provider"
:value="metadata.id"
:title="metadata.localizedName || 'Unknown'"
@@ -215,7 +229,7 @@ function formatRelativeTime(timestamp: number | null) {
/>
</RouterLink>
</fieldset>
<div v-else>
<div v-if="moduleVisionProvidersMetadata.length === 0">
<RouterLink
to="/settings/providers#vision"
:class="[
@@ -251,7 +265,7 @@ function formatRelativeTime(timestamp: number | null) {
</div>
</SettingsCard>
<SettingsCard v-if="activeProvider && supportsModelListing">
<SettingsCard v-if="selectedProvider && supportsModelListing">
<div :class="['flex', 'flex-col', 'gap-4']">
<div>
<h2 :class="['text-lg', 'md:text-2xl']">
@@ -309,7 +323,7 @@ function formatRelativeTime(timestamp: number | null) {
</div>
</SettingsCard>
<SettingsCard v-else-if="activeProvider && !supportsModelListing">
<SettingsCard v-else-if="selectedProvider && !supportsModelListing">
<div :class="['flex', 'flex-col', 'gap-4']">
<div>
<h2 :class="['text-lg', 'text-neutral-500', 'md:text-2xl', 'dark:text-neutral-400']">
+12
View File
@@ -2,6 +2,18 @@
Shared core for stage
## Optional modules
Provider lists expose `None` for optional features. Switches control module activation.
Consciousness remains required. Minecraft runs as an external service and keeps its own lifecycle.
Hearing, Speech, and Vision persist their enable states separately from provider and model selections.
Disabling these modules preserves credentials and selections. Re-enabling restores their use.
Their synchronized stores replicate the enable states across desktop windows.
Hearing cancels active transcription and rejects results from its previous enabled period.
The voice host also stops listening and dictation. Text chat can continue without transcribing stored recordings.
Disabling Speech stops its producers and playback. Text generation and tool work continue until the chat turn ends.
## Experimental features
Register flags in `libs/feature-flags.ts`. Read their state through `useFeatureFlagsStore().isEnabled(key)`.
@@ -19,7 +19,7 @@ const microphoneError = computed(() => snapshot.value.microphone?.error)
const transcript = computed(() => snapshot.value.input?.text)
const isTranscribing = computed(() => snapshot.value.input?.phase === 'finalizing')
const { volumeLevel } = useAudioAnalyzer()
const { configured } = storeToRefs(useHearingStore())
const { configured, enabled: hearingEnabled } = storeToRefs(useHearingStore())
const issue = computed(() => {
// Device failure prevents all input, so it takes precedence over provider feedback.
if (microphoneError.value)
@@ -53,7 +53,7 @@ const label = computed(() => {
<template>
<StatusCapsule
v-if="enabled || issue || isTranscribing"
v-if="hearingEnabled && (enabled || issue || isTranscribing)"
:align="align"
:data-status="state"
:tone="issue ? 'error' : 'neutral'"
@@ -6,6 +6,8 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { createApp } from 'vue'
import { useAiriCardStore } from '../stores/modules/airi-card'
import { useHearingStore } from '../stores/modules/hearing'
import { useSpeechStore } from '../stores/modules/speech'
import { useProviderConfigStore } from '../stores/providers/config'
import { useProviderStore } from '../stores/providers/provider'
import { useSettingsNeedsAttention } from './use-settings-needs-attention'
@@ -76,6 +78,29 @@ describe('useSettingsNeedsAttention', () => {
afterEach(() => vi.unstubAllGlobals())
// https://github.com/moeru-ai/airi/issues/3153
// ROOT CAUSE:
// Saved providers counted as active. Disabled modules now leave the provider usage list without deleting shared settings.
it('issue #3153: removes disabled module warnings while retaining another module that uses the same provider', async () => {
const cards = useAiriCardStore()
await cards.initialize()
const provider = 'removed-provider'
const characterId = await cards.addCard(cardWithSpeech(provider), 'import')
const hearing = useHearingStore()
hearing.activeTranscriptionProvider = provider
await waitForProviderRegistry()
const { moduleSelections } = useSettingsNeedsAttention()
expect(moduleSelections.value.some(selection => selection.key === 'hearing')).toBe(true)
hearing.enabled = false
expect(moduleSelections.value.some(selection => selection.key === 'hearing')).toBe(false)
expect(moduleSelections.value.some(selection => selection.key === `${characterId}:speech`)).toBe(true)
expect(hearing.activeTranscriptionProvider).toBe(provider)
useSpeechStore().enabled = false
expect(moduleSelections.value.some(selection => selection.key === `${characterId}:speech`)).toBe(false)
})
// ROOT CAUSE:
//
// The added flag of a provider stays after an update removes the provider.
@@ -324,13 +324,13 @@ export function useSettingsNeedsAttention() {
const modulesByProvider = computed(() => {
const selections: Array<readonly [provider: string, module: Module]> = [
[consciousnessStore.activeProvider, 'consciousness'],
[speechStore.activeSpeechProvider, 'speech'],
[hearingStore.activeTranscriptionProvider, 'hearing'],
[visionStore.activeProvider, 'vision'],
[speechStore.enabled ? speechStore.activeSpeechProvider : '', 'speech'],
[hearingStore.enabled ? hearingStore.activeTranscriptionProvider : '', 'hearing'],
[visionStore.enabled ? visionStore.activeProvider : '', 'vision'],
...[...cardStore.characterModules.values()].flatMap(modules => [
[modules.consciousness.provider, 'consciousness'],
[modules.speech.provider, 'speech'],
[modules.vision.provider, 'vision'],
[speechStore.enabled ? modules.speech.provider : '', 'speech'],
[visionStore.enabled ? modules.vision.provider : '', 'vision'],
] as const),
]
const result = new Map<string, Set<Module>>()
@@ -417,9 +417,9 @@ export function useSettingsNeedsAttention() {
...[...cardStore.characterModules].flatMap(([characterId, modules]) => CHARACTER_MODULES.map(module => ({
characterId: characterId as string | undefined,
module: module as Module,
provider: modules[module].provider,
provider: (module === 'vision' && !visionStore.enabled) || (module === 'speech' && !speechStore.enabled) ? '' : modules[module].provider,
}))),
{ characterId: undefined, module: 'hearing' as Module, provider: hearingStore.activeTranscriptionProvider },
{ characterId: undefined, module: 'hearing' as Module, provider: hearingStore.enabled ? hearingStore.activeTranscriptionProvider : '' },
]
return selections
.filter(({ provider }) => provider && provider !== 'speech-noop')
@@ -504,6 +504,8 @@ export function useSettingsNeedsAttention() {
* A provider that the speech module cannot use shows as its own item instead.
*/
const speechProblems = computed(() => characterSpeech.value.flatMap((speech): Problem[] => {
if (!speechStore.enabled)
return []
const missing: MissingSpeechField[] = speech.simulated ? ['voice'] : speech.providerUsable ? speech.missing : []
const fieldsKey = speechFieldsKeyOf(missing)
if (!fieldsKey)
@@ -539,6 +541,8 @@ export function useSettingsNeedsAttention() {
* Push to Talk and dictation explain it when the person presses them, so they do not show here.
*/
const hearingProblems = computed((): Problem[] => {
if (!hearingStore.enabled)
return []
const provider = hearingStore.activeTranscriptionProvider
const { microphoneOn, mode, configured, simulated } = voiceInput.value
if (!simulated) {
@@ -107,6 +107,24 @@ describe('useVisionInference', () => {
const toolImage = { imageDataUrl: 'data:image/png;base64,Zm9v', workloadId: 'tool:image' as const }
it('cancels an active read on disable and blocks later provider calls', async () => {
const releases = holdEachRead()
const { runVisionInference, lastText } = useVisionInference()
const read = runVisionInference(toolImage)
const cancelled = expect(read).rejects.toMatchObject({ name: 'AbortError' })
await vi.advanceTimersByTimeAsync(1)
expect(stream).toHaveBeenCalledOnce()
useVisionStore().enabled = false
expect(stream.mock.calls[0][3]?.abortSignal?.aborted).toBe(true)
releases[0]()
await cancelled
await expect(runVisionInference(toolImage)).rejects.toMatchObject({ name: 'AbortError' })
expect(stream).toHaveBeenCalledOnce()
expect(useVisionActivityStore().inferenceCount).toBe(0)
expect(lastText.value).toBe('')
})
it('reads one image at a time for a provider that answers one read at a time', async () => {
// ROOT CAUSE:
//
@@ -6,7 +6,7 @@ import type { VisionWorkloadId } from './use-vision-workloads'
import { errorMessageFrom } from '@moeru/std'
import { DEFAULT_CONCURRENT_VISION_READS } from '@proj-airi/provider-inference'
import { storeToRefs } from 'pinia'
import { ref } from 'vue'
import { ref, watch } from 'vue'
import { useLLM } from '../../stores/ai/chat-llm/llm'
import { reportActivity, useVisionActivityStore, useVisionStore } from '../../stores/modules/vision'
@@ -57,6 +57,7 @@ export function useVisionInference() {
async function describeImage(providerId: string, modelId: string, input: VisionInferenceInput) {
const thinking = ollamaThinkingEnabled.value
const provider = await providersStore.getChatProviderInstance(providerId)
input.abortSignal?.throwIfAborted()
const workload = getVisionWorkload(input.workloadId)
const prompt = input.promptOverride ?? workload.prompt
const { url } = parseDataUrl(input.imageDataUrl)
@@ -112,6 +113,24 @@ export function useVisionInference() {
}
async function runVisionInference(input: VisionInferenceInput) {
if (!visionStore.enabled)
throw new DOMException('Vision is disabled', 'AbortError')
const cancellation = new AbortController()
const stop = watch(() => visionStore.enabled, (enabled) => {
if (!enabled)
cancellation.abort(new DOMException('Vision is disabled', 'AbortError'))
}, { flush: 'sync' })
const signal = AbortSignal.any([cancellation.signal, ...(input.abortSignal ? [input.abortSignal] : [])])
try {
return await readImage({ ...input, abortSignal: signal })
}
finally {
stop()
}
}
async function readImage(input: VisionInferenceInput) {
const providerId = input.selection?.provider ?? activeProvider.value
const modelId = input.selection?.model ?? activeModel.value
if (!providerId || !modelId)
@@ -133,7 +152,9 @@ export function useVisionInference() {
}
try {
lastText.value = await describeImage(providerId, modelId, input)
const text = await describeImage(providerId, modelId, input)
input.abortSignal?.throwIfAborted()
lastText.value = text
}
catch (error) {
if (!input.abortSignal?.aborted)
@@ -18,7 +18,7 @@ export function useVoiceListening() {
const devices = useSettingsAudioDevice()
const hearing = useHearingStore()
watch(() => devices.enabled && listensContinuously(hearing.inputMode), (listen) => {
watch(() => hearing.enabled && devices.enabled && listensContinuously(hearing.inputMode), (listen) => {
if (listen)
voice.startListening()
else
@@ -48,7 +48,7 @@ export function useVoicePushToTalk(options: VoicePushToTalkOptions = {}) {
const sessions = useChatSessionStore()
return useVoiceHold({
enabled: () => devices.enabled && hearing.inputMode === 'push-to-talk',
enabled: () => hearing.enabled && devices.enabled && hearing.inputMode === 'push-to-talk',
sessionId: () => sessions.activeSessionId,
begin: (sessionId) => {
if (!hearing.configured) {
+6 -1
View File
@@ -781,10 +781,15 @@ export const useChatStore = defineStore('chat', () => {
}
if (!text) {
if (!hearing.enabled) {
// Stored recordings stay intact. A text-only model can continue the conversation while recognition is disabled.
turn.content[index] = { type: 'text', text: t('stage.chat.audio.hearing-disabled') }
continue
}
// The recording belongs to this turn. A voice input of another turn can run at the same time.
transcriber ??= hearing.createTranscriber(undefined, { traceParent: turnSpan })
if (!hearing.configured)
throw new Error('Configure a transcription provider to send audio to this model')
transcriber ??= hearing.createTranscriber(undefined, { traceParent: turnSpan })
const blob = new Blob([new Uint8Array(decodeBase64(part.data))], { type: part.format === 'mp3' ? 'audio/mpeg' : 'audio/wav' })
let completed = false
@@ -648,6 +648,9 @@ export const useAiriCardStore = defineStore('airi-card', () => {
async function selectActiveCardVisionProvider(provider: string) {
await pendingAuthenticationSetup
const vision = useVisionStore()
vision.enabled = true
if (vision.activeProvider === provider)
return true
vision.activeProvider = provider
vision.resetModelSelection()
await vision.loadModelsForProvider(provider)
@@ -29,6 +29,23 @@ vi.mock('vue-i18n', () => ({
}))
describe('official provider module defaults', () => {
it('keeps explicit None selections disabled when authentication applies defaults', async () => {
const hearing = useHearingStore()
const vision = useVisionStore()
const speech = useSpeechStore()
hearing.enabled = false
vision.enabled = false
speech.enabled = false
speech.activeSpeechProvider = 'speech-noop'
await configureAsDefaultsIfEmpty()
expect(hearing.configured).toBe(false)
expect(vision.configured).toBe(false)
expect(speech.configured).toBe(false)
expect(speech.activeSpeechProvider).toBe('speech-noop')
})
beforeEach(() => {
setActivePinia(createPinia())
vi.stubGlobal('fetch', vi.fn(async () => new Response(JSON.stringify({
@@ -35,7 +35,7 @@ export async function configureAsDefaultsIfEmpty(): Promise<boolean> {
const usesOfficialConsciousness = consciousnessStore.activeProvider === officialModuleDefaults.consciousness.provider
const needsHearingDefault = !hearingStore.activeTranscriptionProvider
const usesOfficialHearing = hearingStore.activeTranscriptionProvider === officialModuleDefaults.hearing.provider
const needsSpeechDefault = !speechStore.activeSpeechProvider || speechStore.activeSpeechProvider === 'speech-noop'
const needsSpeechDefault = speechStore.enabled && (!speechStore.activeSpeechProvider || speechStore.activeSpeechProvider === 'speech-noop')
const usesOfficialSpeech = speechStore.activeSpeechProvider === officialModuleDefaults.speech.provider
const needsVisionDefault = !visionStore.activeProvider
const usesOfficialVision = visionStore.activeProvider === officialModuleDefaults.vision.provider
@@ -8,6 +8,7 @@ import { createI18n } from 'vue-i18n'
import HearingStatus from '../../components/scenarios/status/hearing-status.vue'
import { getSpeechBusContext, voiceSnapshotChanged } from '../../services/speech/bus'
import { useHearingStore } from './hearing'
const cleanups: (() => void)[] = []
afterEach(() => {
@@ -24,4 +25,6 @@ it('shows final transcription work and reports a provider failure through the ho
await expect.element(view.getByRole('status')).toHaveTextContent('Transcribing')
context.emit(voiceSnapshotChanged, { connected: true, drafts: [], error: 'Provider timed out', input: { requestId: 'recording', sessionId: 'alice', phase: 'settled', text: 'recognized words', segments: [{ id: 'speech-0', text: 'recognized words', final: true }] } })
await expect.element(view.getByRole('status')).toHaveTextContent('Could not transcribe audio')
useHearingStore(pinia).enabled = false
await expect.element(view.getByRole('status')).not.toBeInTheDocument()
})
@@ -16,7 +16,7 @@ import { useLocalStorageManualReset } from '@proj-airi/stage-shared/composables'
import { refManualReset } from '@vueuse/core'
import { generateTranscription } from '@xsai/generate-transcription'
import { defineStore, storeToRefs } from 'pinia'
import { computed, ref, toRaw, watch } from 'vue'
import { computed, onScopeDispose, ref, toRaw, watch } from 'vue'
import { useAnalytics } from '../../composables/use-analytics'
import { createHearingTranscriber } from '../../libs/audio/hearing-transcriber'
@@ -324,6 +324,17 @@ export const useHearingStore = defineStore('hearing-store', () => {
// Pinia synchronization owns live cross-window state. localStorage only
// loads and saves durable values for this synchronized store.
const persistenceOptions = { listenToStorageChanges: false }
const enabled = useLocalStorageManualReset('settings/hearing/enabled', true, persistenceOptions)
// Each enabled period owns its requests. Disabling also invalidates transcribers captured by pending recordings.
let recognition = new AbortController()
watch(enabled, (value) => {
if (value)
recognition = new AbortController()
else
recognition.abort(new DOMException('Hearing is disabled', 'AbortError'))
}, { immediate: true, flush: 'sync' })
onScopeDispose(() => recognition.abort())
// State
const activeTranscriptionProvider = useLocalStorageManualReset('settings/hearing/active-provider', '', persistenceOptions)
@@ -387,7 +398,7 @@ export const useHearingStore = defineStore('hearing-store', () => {
}
const configured = computed(() => {
if (!activeTranscriptionProvider.value)
if (!enabled.value || !activeTranscriptionProvider.value)
return false
// Web Speech API doesn't strictly need a model selected (it has a default)
@@ -407,6 +418,7 @@ export const useHearingStore = defineStore('hearing-store', () => {
})
function resetState() {
enabled.reset()
activeTranscriptionProvider.reset()
activeTranscriptionModel.reset()
activeCustomModelName.reset()
@@ -430,6 +442,8 @@ export const useHearingStore = defineStore('hearing-store', () => {
format?: 'json' | 'verbose_json',
options?: HearingTranscriptionInvokeOptions,
): Promise<HearingTranscriptionResult> {
const signal = AbortSignal.any([recognition.signal, ...(options?.signal ? [options.signal] : [])])
signal.throwIfAborted()
const normalizedInput = (input instanceof File ? { file: input } : input ?? {}) as {
file?: File
fileName?: string
@@ -462,7 +476,7 @@ export const useHearingStore = defineStore('hearing-store', () => {
try {
if (features.supportsStreamOutput && streamExecutor) {
const request = { ...provider.transcription(model, options?.providerOptions), abortSignal: options?.signal }
const request = { ...provider.transcription(model, options?.providerOptions), abortSignal: signal }
// Recorder files contain encoded media. Sherpaw accepts only the raw PCM16 stream from the live pipeline.
if (providerId === SHERPAW_TRANSCRIPTION_PROVIDER_ID && normalizedInput.file && !normalizedInput.inputAudioStream) {
@@ -522,12 +536,14 @@ export const useHearingStore = defineStore('hearing-store', () => {
const useVerboseJson = !format && threshold > CONFIDENCE_THRESHOLD_DISABLED
const response = await generateTranscription({
...provider.transcription(model, options?.providerOptions),
abortSignal: options?.signal,
abortSignal: signal,
file: normalizedInput.file,
fileName: resolveTranscriptionFileName(normalizedInput.file, normalizedInput.fileName),
responseFormat: useVerboseJson ? 'verbose_json' : format,
})
signal.throwIfAborted()
if (useVerboseJson) {
if (response.segments) {
verboseJsonNotSupported.value = false
@@ -561,6 +577,8 @@ export const useHearingStore = defineStore('hearing-store', () => {
/** Freezes Hearing settings for one request. The controller supplies media and owns cancellation. */
function createTranscriber(providerId = activeTranscriptionProvider.value, trace: HearingTranscriberTraceOptions = {}): StreamingTranscriber {
const recognitionSignal = recognition.signal
recognitionSignal.throwIfAborted()
const config = structuredClone(toRaw(providerStore.getProviderConfig(providerId) ?? {}))
const model = resolveActiveTranscriptionModel(activeTranscriptionModel.value, config)
const threshold = confidenceThreshold.value
@@ -573,7 +591,7 @@ export const useHearingStore = defineStore('hearing-store', () => {
upload = 'media-stream'
else if (providerId === SHERPAW_TRANSCRIPTION_PROVIDER_ID || features.supportsStreamInput)
upload = 'pcm'
return createHearingTranscriber(async (audio, signal) => {
const transcriber = createHearingTranscriber(async (audio, signal) => {
const setupError = resolveActiveTranscriptionProviderError(providerId)
if (setupError)
throw new Error(setupError)
@@ -625,9 +643,16 @@ export const useHearingStore = defineStore('hearing-store', () => {
throw error
}
}, trace)
return {
transcribe: request => transcriber.transcribe({
...request,
signal: AbortSignal.any([request.signal, recognitionSignal]),
}),
}
}
return {
enabled,
activeTranscriptionProvider,
activeTranscriptionModel,
availableProvidersMetadata,
@@ -0,0 +1,154 @@
import type { PcmBlock } from '@proj-airi/pipelines-audio'
import type { LeadershipMode, SyncedPiniaRuntime } from 'pinia-plugin-synced'
import { generateTranscription } from '@xsai/generate-transcription'
import { createPinia, disposePinia } from 'pinia'
import { createSyncedPiniaPlugin } from 'pinia-plugin-synced'
import { afterEach, expect, it, vi } from 'vitest'
import { createApp, nextTick } from 'vue'
import { createI18n } from 'vue-i18n'
import { OFFICIAL_SPEECH_PROVIDER_ID, OFFICIAL_SPEECH_STREAMING_PROVIDER_ID } from '../../libs/providers/providers/official'
import { useProviderConfigStore } from '../providers/config'
import { useHearingStore } from './hearing'
import { useSpeechStore } from './speech'
import { useVisionStore } from './vision/store'
vi.mock('@xsai/generate-transcription', () => ({ generateTranscription: vi.fn() }))
const contexts: { pinia: ReturnType<typeof createPinia>, runtime?: SyncedPiniaRuntime, unmount: () => void }[] = []
function createContext(namespace?: string, leadership?: LeadershipMode) {
const pinia = createPinia()
const runtime = namespace ? createSyncedPiniaPlugin({ namespace, leadership, callTimeout: 1000 }) : undefined
if (runtime)
pinia.use(runtime.plugin)
let hearing!: ReturnType<typeof useHearingStore>
let vision!: ReturnType<typeof useVisionStore>
let speech!: ReturnType<typeof useSpeechStore>
const app = createApp({
setup() {
hearing = useHearingStore(pinia)
vision = useVisionStore(pinia)
speech = useSpeechStore(pinia)
return () => null
},
}).use(createI18n({ legacy: false, locale: 'en', messages: { en: {} }, missingWarn: false, fallbackWarn: false })).use(pinia)
app.mount(document.createElement('div'))
contexts.push({ pinia, runtime, unmount: () => app.unmount() })
return { pinia, runtime, hearing, vision, speech }
}
afterEach(() => {
for (const { pinia, runtime, unmount } of contexts.splice(0)) {
unmount()
runtime?.dispose()
disposePinia(pinia)
}
localStorage.clear()
vi.mocked(generateTranscription).mockReset()
})
// https://github.com/moeru-ai/airi/issues/3153
// ROOT CAUSE:
// Provider selection also implied activation. Persisted enable states now retain selections while disabling the modules.
it('issue #3153: retains disabled modules and their settings after reopening', async () => {
const first = createContext()
first.hearing.activeTranscriptionProvider = 'browser-web-speech-api'
first.hearing.activeTranscriptionModel = 'web-speech-api'
first.vision.activeProvider = 'vision-ollama'
first.vision.activeModel = 'saved-vision-model'
first.hearing.enabled = false
first.vision.enabled = false
first.speech.enabled = false
await nextTick()
const reopened = createContext()
expect(reopened.hearing.enabled).toBe(false)
expect(reopened.hearing.configured).toBe(false)
expect(reopened.hearing.activeTranscriptionProvider).toBe('browser-web-speech-api')
expect(reopened.hearing.activeTranscriptionModel).toBe('web-speech-api')
expect(reopened.vision.enabled).toBe(false)
expect(reopened.vision.configured).toBe(false)
expect(reopened.vision.activeModel).toBe('saved-vision-model')
expect(reopened.speech.enabled).toBe(false)
expect(reopened.speech.configured).toBe(false)
reopened.hearing.enabled = true
reopened.vision.enabled = true
expect(reopened.hearing.configured).toBe(true)
expect(reopened.vision.configured).toBe(true)
})
it('replicates disabled states without a follower proposal', async () => {
const namespace = `optional-modules:${crypto.randomUUID()}`
const leader = createContext(namespace, 'leader-only')
const follower = createContext(namespace, 'follower-only')
await vi.waitFor(() => expect(follower.runtime?.getLeaderId()).toBe(leader.runtime?.participantId))
let leaderMutations = 0
let followerMutations = 0
for (const store of [leader.hearing, leader.vision, leader.speech])
store.$subscribe(() => leaderMutations++, { flush: 'sync' })
for (const store of [follower.hearing, follower.vision, follower.speech])
store.$subscribe(() => followerMutations++, { flush: 'sync' })
leader.hearing.enabled = false
leader.vision.enabled = false
leader.speech.enabled = false
await vi.waitFor(() => expect(follower.hearing.enabled).toBe(false))
await vi.waitFor(() => expect(follower.vision.enabled).toBe(false))
await vi.waitFor(() => expect(follower.speech.enabled).toBe(false))
await new Promise(resolve => setTimeout(resolve, 50))
expect(leaderMutations).toBe(3)
expect(followerMutations).toBe(3)
})
// https://github.com/moeru-ai/airi/issues/3153
// ROOT CAUSE:
// The official source card represents two providers. Re-enabling must retain the saved streaming provider and model.
it('issue #3153: restores the streaming selection through the official source card', async () => {
const { speech } = createContext()
speech.activeSpeechProvider = OFFICIAL_SPEECH_STREAMING_PROVIDER_ID
speech.activeSpeechModel = 'saved-streaming-model'
speech.activeSpeechVoiceId = 'saved-voice'
await speech.selectSource('speech-noop')
expect(speech.enabled).toBe(false)
const selection = await speech.selectSource(OFFICIAL_SPEECH_PROVIDER_ID)
expect(speech.enabled).toBe(true)
expect(selection).toEqual({ provider: OFFICIAL_SPEECH_STREAMING_PROVIDER_ID, model: 'saved-streaming-model', voice_id: 'saved-voice' })
})
// https://github.com/moeru-ai/airi/issues/3153
// ROOT CAUSE:
// Recordings captured a provider without a module lifetime. Disabling now aborts that lifetime and rejects late results.
it('issue #3153: cancels an active transcription and rejects a captured transcriber after re-enabling', async () => {
const { hearing, pinia } = createContext()
const provider = 'openai-compatible-audio-transcription'
await useProviderConfigStore(pinia).ensureProvider(provider, provider, { apiKey: 'saved-key', baseUrl: 'https://asr.invalid/v1/' })
hearing.activeTranscriptionProvider = provider
hearing.activeTranscriptionModel = 'whisper-1'
const pending = Promise.withResolvers<Awaited<ReturnType<typeof generateTranscription>>>()
vi.mocked(generateTranscription).mockReturnValue(pending.promise)
const transcriber = hearing.createTranscriber()
const audio = new ReadableStream<PcmBlock>({
start(controller) {
controller.enqueue({ range: { sourceId: 'recording', startFrame: 0, endFrame: 160 }, sampleRate: 16000, channels: [new Float32Array(160)] })
controller.close()
},
})
const reader = transcriber.transcribe({ audio, signal: new AbortController().signal }).getReader()
const read = reader.read().catch(error => error)
await vi.waitFor(() => expect(generateTranscription).toHaveBeenCalledTimes(1))
const request = vi.mocked(generateTranscription).mock.calls[0][0]
hearing.enabled = false
expect(request.abortSignal?.aborted).toBe(true)
expect(await read).toMatchObject({ name: 'AbortError' })
expect(() => hearing.createTranscriber()).toThrow()
pending.resolve({ text: 'late words', duration: 0, language: 'en', segments: [], words: [] })
await expect(reader.read()).rejects.toMatchObject({ name: 'AbortError' })
expect(useProviderConfigStore(pinia).getProviderConfig(provider)?.apiKey).toBe('saved-key')
hearing.enabled = true
const stale = transcriber.transcribe({ audio: new ReadableStream<PcmBlock>(), signal: new AbortController().signal }).getReader()
await expect(stale.read()).rejects.toMatchObject({ name: 'AbortError' })
expect(generateTranscription).toHaveBeenCalledTimes(1)
})
@@ -3,9 +3,11 @@ import type { Component } from 'vue'
import en from '@proj-airi/i18n/locales/en'
import { MotionPlugin } from '@vueuse/motion'
import { generateSpeech } from '@xsai/generate-speech'
import { createPinia, disposePinia } from 'pinia'
import { createSyncedPiniaPlugin } from 'pinia-plugin-synced'
import { afterEach, expect, it, vi } from 'vitest'
import { page } from 'vitest/browser'
import { createApp, h } from 'vue'
import { createI18n } from 'vue-i18n'
import { createMemoryHistory, createRouter } from 'vue-router'
@@ -21,6 +23,7 @@ import { useSpeechStore } from './speech'
// Analytics delivery is external IO. Exercise the real page and stores while
// recording its outgoing payload instead of sending product events.
vi.mock('../../libs/product-signals/client', { spy: true })
vi.mock('@xsai/generate-speech', { spy: true })
const cleanups: Array<() => void> = []
@@ -61,6 +64,81 @@ afterEach(() => {
localStorage.clear()
})
// https://github.com/moeru-ai/airi/issues/3153
// ROOT CAUSE:
// None replaced the selected speech configuration. The source action now retains it and commits activation in the leader.
it.each(['provider', 'switch'] as const)('issue #3153: disables speech with the %s control from a follower and restores the saved model and voice', async (control) => {
localStorage.clear()
vi.stubGlobal('fetch', vi.fn<typeof fetch>(async () => Response.json({ voices: [], data: [] })))
const namespace = `speech-disabled:${crypto.randomUUID()}`
const leader = mountRenderer(namespace)
await vi.waitFor(() => expect(leader.runtime.isLeader()).toBe(true))
const provider = 'openai-compatible-audio-speech'
await useProviderConfigStore(leader.pinia).ensureProvider(provider, provider, { apiKey: 'key', baseUrl: 'https://voices.invalid/v1/' })
await useProviderStore(leader.pinia).forceProviderConfigured(provider)
await leader.speech.selectProviderModel(provider, 'tts-1', 'alloy')
const follower = mountRenderer(namespace, SpeechSettings)
await vi.waitFor(() => expect(follower.speech.activeSpeechVoiceId).toBe('alloy'))
const view = page.elementLocator(follower.container)
const toggle = view.getByRole('switch', { name: 'Speech', exact: true })
const none = view.getByRole('radio', { name: 'None', exact: true })
await expect.element(toggle).toBeChecked()
if (control === 'switch')
await toggle.click()
else
await page.elementLocator(none.element().closest('label')!).click()
await vi.waitFor(() => expect(leader.speech.enabled).toBe(false))
expect(leader.speech.configured).toBe(false)
expect(leader.speech.activeSpeechProvider).toBe(provider)
expect(leader.speech.activeSpeechModel).toBe('tts-1')
expect(leader.speech.activeSpeechVoiceId).toBe('alloy')
// The leader commits before the follower renders its snapshot. Wait for the
// displayed state before clicking a radio that was previously selected.
await expect.element(toggle).not.toBeChecked()
await expect.element(none).toBeChecked()
if (control === 'switch') {
await toggle.click()
}
else {
const savedProvider = follower.container.querySelector<HTMLInputElement>(`input[value="${provider}"]`)!
await expect.element(page.elementLocator(savedProvider)).not.toBeChecked()
await page.elementLocator(savedProvider.closest('label')!).click()
}
await vi.waitFor(() => expect(leader.speech.enabled).toBe(true))
await expect.element(toggle).toBeChecked()
expect(follower.speech.activeSpeechProvider).toBe(provider)
expect(leader.speech.activeSpeechModel).toBe('tts-1')
expect(leader.speech.activeSpeechVoiceId).toBe('alloy')
})
// https://github.com/moeru-ai/airi/issues/3153
// ROOT CAUSE:
// The preview bypassed module cancellation. Disabling now aborts its request and rejects late audio.
it('issue #3153: cancels a speech preview when another window disables speech', async () => {
localStorage.clear()
vi.stubGlobal('fetch', vi.fn<typeof fetch>(async () => Response.json({ voices: [], data: [] })))
const pending = Promise.withResolvers<ArrayBuffer>()
vi.mocked(generateSpeech).mockReturnValue(pending.promise)
const namespace = `speech-preview-disabled:${crypto.randomUUID()}`
const leader = mountRenderer(namespace)
await vi.waitFor(() => expect(leader.runtime.isLeader()).toBe(true))
const provider = 'openai-compatible-audio-speech'
await useProviderConfigStore(leader.pinia).ensureProvider(provider, provider, { apiKey: 'key', baseUrl: 'https://voices.invalid/v1/' })
await useProviderStore(leader.pinia).forceProviderConfigured(provider)
await leader.speech.selectProviderModel(provider, 'tts-1', 'alloy')
const follower = mountRenderer(namespace, SpeechSettings)
const preview = () => Array.from(follower.container.querySelectorAll('button')).find(button => button.textContent?.includes('Test Voice'))!
await vi.waitFor(() => expect(preview()?.disabled).toBe(false))
preview().click()
await vi.waitFor(() => expect(generateSpeech).toHaveBeenCalledTimes(1))
const signal = vi.mocked(generateSpeech).mock.calls[0][0].abortSignal
await leader.speech.selectSource('speech-noop')
await vi.waitFor(() => expect(signal?.aborted).toBe(true))
pending.resolve(new ArrayBuffer(0))
await vi.waitFor(() => expect(preview()?.disabled).toBe(true))
expect(follower.container.querySelector('audio')).toBeNull()
})
// https://github.com/moeru-ai/airi/actions/runs/34348745853/job/102456521103
// ROOT CAUSE: Page initialization and provider watchers awaited model RPCs
// without handling transport disposal. Passing assertions hid a teardown rejection.
@@ -136,7 +214,7 @@ it('reports the committed provider and model after a settings-page click', async
tts_model_id: 'v1',
source: 'settings',
}))
expect(follower.speech.activeSpeechProvider).toBe('microsoft-speech')
await vi.waitFor(() => expect(follower.speech.activeSpeechProvider).toBe('microsoft-speech'))
})
// https://github.com/moeru-ai/airi/pull/2490#discussion_r3967708960
+40 -5
View File
@@ -1,5 +1,5 @@
import type { SpeechOutput, TurnRef } from '@proj-airi/core-agent'
import type { SpeechProviderWithExtraOptions } from '@xsai-ext/providers/utils'
import type { SpeechProvider, SpeechProviderWithExtraOptions } from '@xsai-ext/providers/utils'
import type {} from 'pinia-plugin-synced'
import type { AiriExtension } from '../../types/airiCard'
@@ -148,6 +148,16 @@ export const useSpeechStore = defineStore('speech', () => {
// State
const activeSpeechProvider = useLocalStorageManualReset<string>('settings/speech/active-provider', 'speech-noop', persistenceOptions)
const enabled = useLocalStorageManualReset('settings/speech/enabled', true, persistenceOptions)
// Disabling invalidates synthesis captured by an open response, including requests that have not started yet.
let synthesis = new AbortController()
watch(enabled, (value) => {
if (value)
synthesis = new AbortController()
else
synthesis.abort(new DOMException('Speech is disabled', 'AbortError'))
}, { immediate: true, flush: 'sync' })
onScopeDispose(() => synthesis.abort())
const activeSpeechModel = useLocalStorageManualReset<string>('settings/speech/active-model', '', persistenceOptions)
const activeSpeechVoiceId = useLocalStorageManualReset<string>('settings/speech/voice', '', persistenceOptions)
const activeSpeechVoice = refManualReset<VoiceInfo | undefined>(undefined)
@@ -469,6 +479,17 @@ export const useSpeechStore = defineStore('speech', () => {
clearVoiceSelection()
}
/** Selects a source in the leader. None disables speech, and selecting the saved source restores its model and voice. */
async function selectSource(provider: string) {
enabled.value = provider !== 'speech-noop'
if (!enabled.value)
return
const restoresStreamingSource = provider === OFFICIAL_SPEECH_PROVIDER_ID && activeSpeechProvider.value === OFFICIAL_SPEECH_STREAMING_PROVIDER_ID
if (activeSpeechProvider.value !== provider && !restoresStreamingSource)
await selectProviderModel(provider, '')
return { provider: activeSpeechProvider.value, model: activeSpeechModel.value, voice_id: activeSpeechVoiceId.value }
}
/** Commits an explicit selection in the leader before watchers request its catalog. An omitted voice preserves an unchanged selection. */
async function selectProviderModel(provider: string, model: string, voiceId?: string) {
if (disposed)
@@ -581,17 +602,19 @@ export const useSpeechStore = defineStore('speech', () => {
* @returns ArrayBuffer containing the audio data
*/
async function speech(
provider: SpeechProviderWithExtraOptions<string, any>,
provider: SpeechProvider | SpeechProviderWithExtraOptions<string, Record<string, unknown>>,
model: string,
input: string,
voice: string,
providerConfig: Record<string, any> = {},
providerConfig: Record<string, unknown> = {},
analytics: SpeechAnalytics = {
trigger: 'manual',
source: 'manual_preview',
voice_type: resolveVoiceType(voice),
},
): Promise<ArrayBuffer> {
const signal = synthesis.signal
signal.throwIfAborted()
const requestProviderConfig = activeSpeechProvider.value === OFFICIAL_SPEECH_PROVIDER_ID
|| activeSpeechProvider.value === OFFICIAL_SPEECH_STREAMING_PROVIDER_ID
? withAiriTtsAnalytics(providerConfig, analytics)
@@ -600,13 +623,18 @@ export const useSpeechStore = defineStore('speech', () => {
...provider.speech(model, requestProviderConfig),
input,
voice,
abortSignal: signal,
})
signal.throwIfAborted()
return response
}
/** Captures one response's provider, voice, model, and settings before any asynchronous work. */
function createOutput(turn: TurnRef, selection: AiriExtension['modules']['speech'], output: Pick<SpeechOutput, 'playback' | 'onPlaybackStart' | 'onPlaybackEnd' | 'onSpecial'>, audioContext: AudioContext): SpeechOutput {
if (!enabled.value)
return { ...output, synthesize: async () => null }
const synthesisSignal = synthesis.signal
const providerId = selection.provider
const config = structuredClone(toRaw(providerStore.getProviderConfig(providerId) ?? {}))
const selectedVoice = availableVoices.value[providerId]?.find(voice => voice.id === selection.voice_id)
@@ -633,12 +661,13 @@ export const useSpeechStore = defineStore('speech', () => {
ttsVoiceType: voiceType,
bufferEntireSession: resource.startsWith('seed-tts-2.0') || resource.startsWith('seed-icl-2.0'),
extraBody: { api_resource_id: resource, audio: { sample_rate: 24000, bit_rate: 64000 } },
}, text, signal) }
}, text, AbortSignal.any([signal, synthesisSignal])) }
}
if (!model || !voiceId)
throw new Error('Speech requires a model and voice')
const provider = providersStore.getProviderInstance<SpeechProviderWithExtraOptions<string, Record<string, unknown>>>(providerId)
return { ...output, language, synthesize: async (request, signal) => {
signal = AbortSignal.any([signal, synthesisSignal])
signal.throwIfAborted()
const client = await provider
signal.throwIfAborted()
@@ -647,6 +676,7 @@ export const useSpeechStore = defineStore('speech', () => {
? { ...config, extraBody: { ...(config.extraBody as Record<string, unknown> | undefined), airi_analytics: { trigger: 'auto', source: 'chat_auto_tts', voice_type: voiceType, turn_id: turn.turnId } } }
: config
const audio = await generateSpeech({ ...client.speech(model, settings), input, voice: voiceId, abortSignal: signal })
signal.throwIfAborted()
return new Blob([audio])
} }
}
@@ -727,6 +757,8 @@ export const useSpeechStore = defineStore('speech', () => {
}
const configured = computed(() => {
if (!enabled.value)
return false
if (activeSpeechProvider.value === 'speech-noop')
return false
@@ -754,6 +786,7 @@ export const useSpeechStore = defineStore('speech', () => {
/** Resets shared settings in the leader and rejects catalog results started before this reset. */
async function resetSettings() {
enabled.reset()
// Invalidate request ownership before the reset publishes new settings.
cancelCatalogRequests()
activeSpeechProvider.reset()
@@ -774,6 +807,7 @@ export const useSpeechStore = defineStore('speech', () => {
return {
// State
configured,
enabled,
activeSpeechProvider,
activeSpeechModel,
activeSpeechVoice,
@@ -803,6 +837,7 @@ export const useSpeechStore = defineStore('speech', () => {
loadVoiceCatalog,
invalidateVoiceCatalogs,
selectProviderModel,
selectSource,
ensureActiveSpeechVoice,
getVoicesForProvider,
ensureStreamingDefaultModel,
@@ -815,7 +850,7 @@ export const useSpeechStore = defineStore('speech', () => {
}
}, {
synced: {
actions: ['loadVoiceCatalog', 'invalidateVoiceCatalogs', 'selectProviderModel', 'ensureActiveSpeechVoice', 'resetSettings'],
actions: ['loadVoiceCatalog', 'invalidateVoiceCatalogs', 'selectProviderModel', 'selectSource', 'ensureActiveSpeechVoice', 'resetSettings'],
state: true,
},
})
@@ -59,7 +59,7 @@ export const useVisionOrchestratorStore = defineStore('vision-orchestrator', ()
const lastWorkloadId = ref<VisionWorkloadId>('screen:interpret')
async function processCapture(payload: VisionCapturePayload) {
if (!activeProvider.value || !activeModel.value) {
if (!visionStore.configured) {
const configurationError = new Error('Vision model is not configured')
recordError(configurationError)
throw configurationError
@@ -13,6 +13,7 @@ export const useVisionStore = defineStore('vision', () => {
// Pinia synchronization owns live cross-window state. localStorage only
// loads and saves durable values for this synchronized store.
const persistenceOptions = { listenToStorageChanges: false }
const enabled = useLocalStorageManualReset('settings/vision/enabled', true, persistenceOptions)
const activeProvider = useLocalStorageManualReset('settings/vision/active-provider', '', persistenceOptions)
const activeModel = useLocalStorageManualReset('settings/vision/active-model', '', persistenceOptions)
@@ -49,7 +50,7 @@ export const useVisionStore = defineStore('vision', () => {
})
const configured = computed(() => {
return !!activeProvider.value && !!activeModel.value
return enabled.value && !!activeProvider.value && !!activeModel.value
})
function resetModelSelection() {
@@ -73,6 +74,7 @@ export const useVisionStore = defineStore('vision', () => {
}
function resetState() {
enabled.reset()
useForChat.reset()
useForToolImages.reset()
activeProvider.reset()
@@ -80,6 +82,7 @@ export const useVisionStore = defineStore('vision', () => {
}
return {
enabled,
useForChat,
useForToolImages,
activeProvider,
@@ -0,0 +1,91 @@
import type { SpeechOutput } from '@proj-airi/core-agent'
import { defineInvoke } from '@moeru/eventa'
import { Playback } from '@proj-airi/pipelines-audio'
import { createPinia, disposePinia } from 'pinia'
import { afterEach, expect, it, vi } from 'vitest'
import { createApp } from 'vue'
import { createI18n } from 'vue-i18n'
import { getSpeechBusContext, voiceSpeechCommand } from '../services/speech/bus'
import { useLlmStreamingControlStore } from './ai/chat-llm/streaming-control'
import { useChatStore } from './chat'
import { useSpeechStore } from './modules/speech'
import { useVoiceStore } from './voice'
const cleanups: (() => void)[] = []
afterEach(() => {
cleanups.splice(0).reverse().forEach(cleanup => cleanup())
vi.restoreAllMocks()
localStorage.clear()
})
// https://github.com/moeru-ai/airi/issues/3153
// ROOT CAUSE:
// Cancelling a response also cancels chat generation and tool handlers. Disabling speech now cancels only its producers and playback.
it.each(['local', 'remote'] as const)('issue #3153: disabling %s speech preserves the chat turn and its tool handlers', async (source) => {
const pinia = createPinia()
let voice!: ReturnType<typeof useVoiceStore>
const app = createApp({ setup() {
voice = useVoiceStore()
return () => null
} }).use(pinia).use(createI18n({ legacy: false, locale: 'en', messages: { en: {} }, missingWarn: false, fallbackWarn: false }))
const container = document.createElement('div')
app.mount(container)
cleanups.push(() => {
app.unmount()
disposePinia(pinia)
})
const settings = useSpeechStore(pinia)
settings.enabled = true
const cancelChat = vi.spyOn(useChatStore(pinia), 'cancelTurn')
const control = useLlmStreamingControlStore(pinia)
const turn = { sessionId: 'alice', turnId: `speech-disabled-${source}` }
const generation = control.beginTurn({ turnId: turn.turnId })
const tool = vi.fn()
generation.on({ name: 'test.continue', prompt: 'Continue the task.' }, tool)
const stopped = vi.fn(async () => ({ throughMs: 10 }))
const played = vi.fn(() => ({ done: new Promise<{ throughMs: number }>(() => {}), stop: stopped }))
let synthesisSignal: AbortSignal | undefined
const output: SpeechOutput = {
playback: new Playback({ nowMs: () => 0, play: played }),
stream: async (_text, signal) => {
synthesisSignal = signal
return new ReadableStream({ start(controller) {
controller.enqueue({ audio: new Blob(['speech']), text: 'Hello.' })
} })
},
}
cleanups.push(voice.connectOutput(() => output))
const command = defineInvoke(getSpeechBusContext(), voiceSpeechCommand)
const producer = { originId: 'chat-window', producerId: 'answer', turn }
if (source === 'local') {
voice.startResponse(turn)
await voice.getSpeech(turn)!.write('Hello.')
}
else {
await command({ ...producer, type: 'open', purpose: 'answer' })
await command({ ...producer, type: 'text', value: 'Hello.' })
}
await expect.poll(() => played.mock.calls.length).toBe(1)
settings.enabled = false
expect(cancelChat).not.toHaveBeenCalled()
expect(voice.activeTurns).toEqual([turn])
await expect.poll(() => synthesisSignal?.aborted).toBe(true)
await expect.poll(() => stopped.mock.calls.length).toBe(1)
await control.dispatchWith('<|CALL ["test.continue"]|>', { turnId: turn.turnId })
expect(tool).toHaveBeenCalledOnce()
if (source === 'local')
expect(await voice.getSpeech(turn)!.write('Later text.')).toEqual({ status: 'closed' })
else
expect(await command({ ...producer, type: 'text', value: 'Later text.' })).toEqual({ status: 'closed' })
expect(await voice.finishResponse(turn)).toBe('finished')
expect(voice.activeTurns).toEqual([])
expect(cancelChat).not.toHaveBeenCalled()
await expect(generation.done).resolves.toEqual({ type: 'completed' })
})
+24 -1
View File
@@ -29,6 +29,7 @@ import { useAudioContext, useSpeakingStore } from './audio'
import { useChatStore } from './chat'
import { useChatSessionStore } from './chat/session-store'
import { startsInputOnSpeech, useHearingStore } from './modules/hearing'
import { useSpeechStore } from './modules/speech'
import { useSettingsAudioDevice } from './settings/audio-device'
import { useSpeakerTrackerStore } from './speaker-tracker'
import { useVoiceCaptureLeaseStore } from './voice-capture-lease'
@@ -52,6 +53,7 @@ const VOICE_END_SILENCE = { pauseMs: 1200, graceMs: 3000 } as const
export const useVoiceStore = defineStore('voice', () => {
const devices = useSettingsAudioDevice()
const hearing = useHearingStore()
const speechSettings = useSpeechStore()
const chat = useChatStore()
const sessions = useChatSessionStore()
const voiceMessages = useVoiceMessagesStore()
@@ -165,6 +167,16 @@ export const useVoiceStore = defineStore('voice', () => {
watch([state, transcript, frontDraftId, error, microphoneStream, microphoneEnabled, microphoneError, () => wakeWordDetection.preparation, () => wakeWordDetection.error], publishSnapshot)
watch(drafts, publishSnapshot, { deep: true })
// The host cancels dictation as well as continuous input. Remote snapshots trigger only local runtime cleanup.
watch(() => hearing.enabled, (enabled) => {
error.value = undefined
if (enabled)
return
void stopListening()
presentedInput = undefined
publishSnapshot()
}, { flush: 'sync' })
function beginInput(options: BeginSpeechInput) {
error.value = undefined
return controller.beginInput(options)
@@ -172,7 +184,7 @@ export const useVoiceStore = defineStore('voice', () => {
/** Begins an input that a control ends. Returns nothing while another window holds a capture lease. */
function beginManual(sessionId: string) {
if (captureLease.held)
if (!hearing.enabled || captureLease.held)
return undefined
return beginInput({ sessionId, interruptTurns: activeTurns.value.filter(turn => turn.sessionId === sessionId), start: { kind: 'after-silence' }, cause: 'control' })
}
@@ -316,6 +328,17 @@ export const useVoiceStore = defineStore('voice', () => {
const producers = new Map<string, SpeechStream>()
const opened = new Set<string>()
const stops = [
// Disabling speech stops local and remote producers. The response stays open for text generation and tools.
watch(() => speechSettings.enabled, (enabled) => {
if (enabled)
return
for (const { response, speech } of responses.values()) {
void response.playback.stop({ fadeMs: 0 }).catch(report)
speech?.cancel('Speech is disabled')
}
for (const speech of producers.values())
speech.cancel('Speech is disabled')
}, { flush: 'sync' }),
voiceMessages.connect(),
captureLease.connect(),
// Automatic inputs stop at the `enabled` check of the voice activity plugin. Manual inputs do not, so end them here.