mirror of
https://github.com/moeru-ai/airi.git
synced 2026-10-30 19:34:10 +08:00
fix(stage-ui): let optional modules switch off without losing settings (#3174)
This commit is contained in:
@@ -671,6 +671,8 @@ pages:
|
||||
description: Configure 3D VRM models and settings
|
||||
scene: Scene
|
||||
modules:
|
||||
availability:
|
||||
none: None
|
||||
beat_sync:
|
||||
description: Vibe with beats from the audio source
|
||||
title: Beat Sync
|
||||
|
||||
@@ -21,6 +21,8 @@ mobile-tools:
|
||||
menu: Menu
|
||||
menu-unread: Menu, {count} unread notifications
|
||||
chat:
|
||||
audio:
|
||||
hearing-disabled: '[Audio recording: transcription is disabled.]'
|
||||
provider-configuration:
|
||||
title: Chat provider isn't configured
|
||||
description: Choose and validate a provider in Settings before sending a message.
|
||||
|
||||
@@ -632,6 +632,8 @@ pages:
|
||||
description: 配置 3D VRM 模型
|
||||
scene: 场景
|
||||
modules:
|
||||
availability:
|
||||
none: 无
|
||||
beat_sync:
|
||||
description: 随音源节拍律动
|
||||
title: 同步音律
|
||||
|
||||
@@ -21,6 +21,8 @@ mobile-tools:
|
||||
menu: 菜单
|
||||
menu-unread: 菜单,{count} 条未读通知
|
||||
chat:
|
||||
audio:
|
||||
hearing-disabled: '[语音录音:转写已停用。]'
|
||||
provider-configuration:
|
||||
title: 尚未正确配置聊天 Provider
|
||||
description: 请先前往设置页面选择并验证一个 Provider,再发送消息。
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import en from '@proj-airi/i18n/locales/en'
|
||||
|
||||
import { pushToTalkTriggerKey } from '@proj-airi/stage-ui/composables/push-to-talk-trigger'
|
||||
import { useHearingStore } from '@proj-airi/stage-ui/stores/modules/hearing'
|
||||
import { createPinia, disposePinia } from 'pinia'
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import { cleanup, render } from 'vitest-browser-vue'
|
||||
@@ -49,6 +50,24 @@ async function openInputModeSelect(screen: Awaited<ReturnType<typeof renderHeari
|
||||
}
|
||||
|
||||
describe('hearing overview', () => {
|
||||
// https://github.com/moeru-ai/airi/issues/3153
|
||||
// ROOT CAUSE:
|
||||
// The overview lacked a module switch. The switch now preserves the selected provider and model.
|
||||
it('issue #3153: switches hearing off and on without clearing its selection', async () => {
|
||||
localStorage.setItem('settings/hearing/active-provider', 'browser-web-speech-api')
|
||||
localStorage.setItem('settings/hearing/active-model', 'web-speech-api')
|
||||
const screen = await renderHearing(true)
|
||||
const hearing = useHearingStore(pinias[pinias.length - 1])
|
||||
const toggle = screen.getByRole('switch', { name: 'Hearing', exact: true })
|
||||
await expect.element(toggle).toBeChecked()
|
||||
await toggle.click()
|
||||
expect(hearing.configured).toBe(false)
|
||||
expect(hearing.activeTranscriptionProvider).toBe('browser-web-speech-api')
|
||||
expect(hearing.activeTranscriptionModel).toBe('web-speech-api')
|
||||
await toggle.click()
|
||||
expect(hearing.configured).toBe(true)
|
||||
})
|
||||
|
||||
it('links to the speech recognition and wake words pages', async () => {
|
||||
const screen = await renderHearing(true)
|
||||
|
||||
|
||||
@@ -65,6 +65,12 @@ const pages = computed(() => [
|
||||
|
||||
<template>
|
||||
<div :class="['flex flex-col gap-6']">
|
||||
<SettingsCard>
|
||||
<FieldCheckbox
|
||||
v-model="hearingStore.enabled"
|
||||
:label="t('settings.pages.modules.hearing.title')"
|
||||
/>
|
||||
</SettingsCard>
|
||||
<RippleGrid :items="pages" :columns="{ default: 1, sm: 2 }">
|
||||
<template #item="{ item: page }">
|
||||
<IconStatusItem
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
import en from '@proj-airi/i18n/locales/en'
|
||||
|
||||
import { useHearingStore } from '@proj-airi/stage-ui/stores/modules/hearing'
|
||||
import { createPinia, disposePinia } from 'pinia'
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import { cleanup, render } from 'vitest-browser-vue'
|
||||
import { page } from 'vitest/browser'
|
||||
import { createI18n } from 'vue-i18n'
|
||||
import { createMemoryHistory, createRouter } from 'vue-router'
|
||||
|
||||
@@ -36,6 +38,32 @@ async function renderTranscriber() {
|
||||
}
|
||||
|
||||
describe('speech recognition settings', () => {
|
||||
// https://github.com/moeru-ai/airi/issues/3153
|
||||
// ROOT CAUSE:
|
||||
// Provider radio cards had no disabled choice. None now disables Hearing and retains the provider and model.
|
||||
it.each(['provider', 'switch'] as const)('issue #3153: disables recognition through the %s control without deleting the provider or model', async (control) => {
|
||||
localStorage.setItem('settings/hearing/active-provider', 'browser-web-speech-api')
|
||||
localStorage.setItem('settings/hearing/active-model', 'web-speech-api')
|
||||
const screen = await renderTranscriber()
|
||||
const hearing = useHearingStore(pinias[pinias.length - 1])
|
||||
|
||||
expect(hearing.configured).toBe(true)
|
||||
const noneCard = screen.getByRole('radio', { name: 'None', exact: true }).element().closest('label')!
|
||||
const toggle = screen.getByRole('switch', { name: 'Hearing', exact: true })
|
||||
if (control === 'switch')
|
||||
await toggle.click()
|
||||
else
|
||||
await page.elementLocator(noneCard).click()
|
||||
expect(hearing.configured).toBe(false)
|
||||
expect(hearing.activeTranscriptionProvider).toBe('browser-web-speech-api')
|
||||
expect(hearing.activeTranscriptionModel).toBe('web-speech-api')
|
||||
await expect.element(screen.getByRole('radio', { name: 'None', exact: true })).toBeChecked()
|
||||
await expect.element(toggle).not.toBeChecked()
|
||||
await toggle.click()
|
||||
expect(hearing.configured).toBe(true)
|
||||
expect(hearing.activeTranscriptionModel).toBe('web-speech-api')
|
||||
})
|
||||
|
||||
it('shows the microphone, the providers, and the playground without the overview controls', async () => {
|
||||
const screen = await renderTranscriber()
|
||||
|
||||
|
||||
@@ -25,6 +25,14 @@ import { moduleModelSearchKey, moduleProviderSearchKey } from '../module-model-s
|
||||
const { t } = useI18n()
|
||||
|
||||
const hearingStore = useHearingStore()
|
||||
const selectedProvider = computed({
|
||||
get: () => hearingStore.enabled ? hearingStore.activeTranscriptionProvider : '',
|
||||
set: (provider: string) => {
|
||||
hearingStore.enabled = !!provider
|
||||
if (provider)
|
||||
hearingStore.activeTranscriptionProvider = provider
|
||||
},
|
||||
})
|
||||
const {
|
||||
activeTranscriptionProvider,
|
||||
activeTranscriptionModel,
|
||||
@@ -130,6 +138,8 @@ const isSpeech = computed(() => inputState.value?.phase === 'capturing')
|
||||
const isSpeechVolume = isSpeech
|
||||
|
||||
async function setupAudioMonitoring() {
|
||||
if (!hearingStore.enabled)
|
||||
return
|
||||
const stopped = stopAudioMonitoring()
|
||||
const startup = new AbortController()
|
||||
monitoringStartup = startup
|
||||
@@ -287,7 +297,7 @@ provide(hearingProviderViewContextKey, {
|
||||
|
||||
// Sync OpenAI Compatible model from provider config
|
||||
function syncOpenAICompatibleSettings() {
|
||||
if (activeTranscriptionProvider.value !== 'openai-compatible-audio-transcription')
|
||||
if (!hearingStore.enabled || activeTranscriptionProvider.value !== 'openai-compatible-audio-transcription')
|
||||
return
|
||||
|
||||
const providerConfig = providerStore.getProviderConfig(activeTranscriptionProvider.value)
|
||||
@@ -311,7 +321,9 @@ watch([selectedAudioInput, useVADModel], async () => {
|
||||
await setupAudioMonitoring()
|
||||
})
|
||||
|
||||
watch(activeTranscriptionProvider, async (provider) => {
|
||||
watch(selectedProvider, async (provider, _previous, onCleanup) => {
|
||||
let cancelled = false
|
||||
onCleanup(() => cancelled = true)
|
||||
const shouldRestartMonitoring = isMonitoring.value
|
||||
|
||||
if (shouldRestartMonitoring) {
|
||||
@@ -320,11 +332,15 @@ watch(activeTranscriptionProvider, async (provider) => {
|
||||
}
|
||||
|
||||
clearPlaygroundSegments()
|
||||
error.value = ''
|
||||
vadModelError.value = ''
|
||||
|
||||
if (!provider)
|
||||
return
|
||||
|
||||
await hearingStore.loadModelsForProvider(provider)
|
||||
if (cancelled)
|
||||
return
|
||||
syncOpenAICompatibleSettings()
|
||||
|
||||
const models = providerModels.value
|
||||
@@ -348,6 +364,7 @@ onUnmounted(() => {
|
||||
<template>
|
||||
<div :class="['flex flex-col gap-6 md:flex-row']">
|
||||
<SettingsCard :class="['md:w-[40%]']">
|
||||
<FieldCheckbox v-model="hearingStore.enabled" :label="t('settings.pages.modules.hearing.title')" />
|
||||
<div :class="['flex flex-col gap-4']">
|
||||
<MicrophonePicker
|
||||
v-model="selectedAudioInput"
|
||||
@@ -375,15 +392,21 @@ onUnmounted(() => {
|
||||
See also: https://stackoverflow.com/a/33737340
|
||||
-->
|
||||
<fieldset
|
||||
v-if="moduleTranscriptionProvidersMetadata.length > 0"
|
||||
:class="['flex flex-row gap-4 min-w-0 overflow-x-auto scroll-smooth']"
|
||||
role="radiogroup"
|
||||
>
|
||||
<RadioCardSimple
|
||||
id="hearing-none"
|
||||
v-model="selectedProvider"
|
||||
name="provider"
|
||||
value=""
|
||||
:title="t('settings.pages.modules.availability.none')"
|
||||
/>
|
||||
<RadioCardSimple
|
||||
v-for="metadata in moduleTranscriptionProvidersMetadata"
|
||||
:id="metadata.id"
|
||||
:key="metadata.id"
|
||||
v-model="activeTranscriptionProvider"
|
||||
v-model="selectedProvider"
|
||||
:data-settings-search="moduleProviderSearchKey('transcription', metadata.id)"
|
||||
name="provider"
|
||||
:value="metadata.id"
|
||||
@@ -407,7 +430,7 @@ onUnmounted(() => {
|
||||
/>
|
||||
</RouterLink>
|
||||
</fieldset>
|
||||
<div v-else>
|
||||
<div v-if="moduleTranscriptionProvidersMetadata.length === 0">
|
||||
<RouterLink
|
||||
to="/settings/providers"
|
||||
:class="[
|
||||
@@ -430,11 +453,11 @@ onUnmounted(() => {
|
||||
|
||||
<component
|
||||
:is="activeProviderHearingView"
|
||||
v-if="activeProviderHearingView"
|
||||
v-if="hearingStore.enabled && activeProviderHearingView"
|
||||
/>
|
||||
|
||||
<!-- Model selection section -->
|
||||
<div v-if="activeTranscriptionProvider">
|
||||
<div v-if="selectedProvider">
|
||||
<div :class="['flex flex-col gap-4']">
|
||||
<div>
|
||||
<h2 :class="['text-lg md:text-2xl']">
|
||||
@@ -514,7 +537,7 @@ onUnmounted(() => {
|
||||
</div>
|
||||
|
||||
<!-- Confidence threshold (only for non-streaming providers) -->
|
||||
<div v-if="!supportsStreamInput" :class="['border-t border-neutral-200 pt-4 dark:border-neutral-700']">
|
||||
<div v-if="hearingStore.enabled && !supportsStreamInput" :class="['border-t border-neutral-200 pt-4 dark:border-neutral-700']">
|
||||
<div :class="['mb-4']">
|
||||
<h2 :class="['text-lg text-neutral-500 md:text-2xl dark:text-neutral-500']">
|
||||
{{ t('settings.pages.modules.hearing.sections.section.confidence-threshold.title') }}
|
||||
@@ -558,7 +581,7 @@ onUnmounted(() => {
|
||||
</p>
|
||||
|
||||
<ErrorContainer
|
||||
v-if="error || transcriptionPipelineError"
|
||||
v-if="hearingStore.enabled && (error || transcriptionPipelineError)"
|
||||
:title="t('settings.pages.modules.hearing.sections.section.playground.error-title')"
|
||||
:error="error || transcriptionPipelineError"
|
||||
:class="['mb-4']"
|
||||
@@ -568,7 +591,7 @@ onUnmounted(() => {
|
||||
<Button
|
||||
:class="['mb-4', 'w-full']"
|
||||
data-testid="hearing-playground-monitor-toggle"
|
||||
:disabled="!activeTranscriptionProvider"
|
||||
:disabled="!selectedProvider"
|
||||
@click="toggleMonitoring"
|
||||
>
|
||||
{{ isMonitoring
|
||||
|
||||
@@ -38,7 +38,7 @@ const providersStore = useProviderStore()
|
||||
const providerStore = useProviderConfigStore()
|
||||
const speechStore = useSpeechStore()
|
||||
const airiCardStore = useAiriCardStore()
|
||||
const { allAudioSpeechProvidersMetadata, moduleSpeechProvidersMetadata } = storeToRefs(providersStore)
|
||||
const { moduleSpeechProvidersMetadata } = storeToRefs(providersStore)
|
||||
const {
|
||||
activeSpeechProvider,
|
||||
activeSpeechModel,
|
||||
@@ -75,6 +75,18 @@ const audioUrl = ref('')
|
||||
const audioPlayer = ref<HTMLAudioElement | null>(null)
|
||||
const errorMessage = ref('')
|
||||
let lastOfficialTtsExposureKey = ''
|
||||
let previewCancellation = new AbortController()
|
||||
|
||||
watch(() => speechStore.enabled, (enabled) => {
|
||||
if (enabled) {
|
||||
previewCancellation = new AbortController()
|
||||
}
|
||||
else {
|
||||
previewCancellation.abort()
|
||||
stopTestAudio()
|
||||
errorMessage.value = ''
|
||||
}
|
||||
}, { immediate: true, flush: 'sync' })
|
||||
|
||||
const STREAMING_MODEL_OPTION_PREFIX = 'streaming:'
|
||||
|
||||
@@ -92,20 +104,20 @@ const selectableSpeechSources = computed(() => {
|
||||
}))
|
||||
|
||||
return [
|
||||
{
|
||||
id: 'speech-noop',
|
||||
providerId: 'speech-noop',
|
||||
title: t('settings.pages.modules.availability.none'),
|
||||
description: undefined,
|
||||
},
|
||||
...configuredSources,
|
||||
...allAudioSpeechProvidersMetadata.value
|
||||
.filter(metadata => metadata.id === 'speech-noop')
|
||||
.map(metadata => ({
|
||||
id: metadata.id,
|
||||
providerId: metadata.id,
|
||||
title: metadata.localizedName || 'Unknown',
|
||||
description: metadata.localizedDescription,
|
||||
})),
|
||||
]
|
||||
})
|
||||
|
||||
const displayedSpeechSource = computed({
|
||||
get: () => {
|
||||
if (!speechStore.enabled)
|
||||
return 'speech-noop'
|
||||
if (activeSpeechProvider.value === OFFICIAL_SPEECH_STREAMING_PROVIDER_ID)
|
||||
return OFFICIAL_SPEECH_PROVIDER_ID
|
||||
return activeSpeechProvider.value
|
||||
@@ -299,9 +311,12 @@ async function selectSpeechVoice(voiceId: string | undefined) {
|
||||
|
||||
/** Persists the selection only after the leader commits its provider and model. */
|
||||
async function selectSpeechSource(sourceId: string) {
|
||||
const selection = await speechStore.selectProviderModel(sourceId, '')
|
||||
if (!selection)
|
||||
const selection = await speechStore.selectSource(sourceId)
|
||||
if (!selection) {
|
||||
stopTestAudio()
|
||||
errorMessage.value = ''
|
||||
return
|
||||
}
|
||||
const providerId = providerStore.providers[sourceId]?.definitionId || sourceId
|
||||
// Use this command's receipt: another selection can reach the store before
|
||||
// this caller resumes, but must not relabel this analytics event.
|
||||
@@ -310,7 +325,8 @@ async function selectSpeechSource(sourceId: string) {
|
||||
tts_model_id: selection.model || 'unknown',
|
||||
source: 'settings',
|
||||
})
|
||||
await persistSelection()
|
||||
// A follower can receive the action receipt before its state snapshot. Persist the committed selection.
|
||||
await airiCardStore.updateActiveCardSpeech(selection)
|
||||
}
|
||||
|
||||
/** Resolves the displayed model option before committing it in the leader. */
|
||||
@@ -348,7 +364,7 @@ function trackOfficialTtsExposure(providerId = activeSpeechProvider.value, model
|
||||
|
||||
/** Applies provider defaults in the leader without a follower state proposal. */
|
||||
async function syncOpenAICompatibleSettings() {
|
||||
if (activeSpeechProvider.value !== 'openai-compatible-audio-speech')
|
||||
if (!speechStore.enabled || activeSpeechProvider.value !== 'openai-compatible-audio-speech')
|
||||
return
|
||||
|
||||
const providerConfig = providerStore.getProviderConfig(activeSpeechProvider.value)
|
||||
@@ -412,13 +428,18 @@ async function persistSelection() {
|
||||
|
||||
// Function to generate speech
|
||||
async function generateTestSpeech() {
|
||||
if (!speechStore.enabled)
|
||||
return
|
||||
const signal = previewCancellation.signal
|
||||
if (!testText.value.trim() && !useSSML.value)
|
||||
return
|
||||
|
||||
if (useSSML.value && !ssmlText.value.trim())
|
||||
return
|
||||
|
||||
const provider = await providersStore.getProviderInstance(activeSpeechProvider.value) as SpeechProviderWithExtraOptions<string, any>
|
||||
const provider = await providersStore.getProviderInstance<SpeechProviderWithExtraOptions<string, Record<string, unknown>>>(activeSpeechProvider.value)
|
||||
if (signal.aborted)
|
||||
return
|
||||
if (!provider) {
|
||||
console.error('Failed to initialize speech provider')
|
||||
return
|
||||
@@ -504,8 +525,11 @@ async function generateTestSpeech() {
|
||||
),
|
||||
input: speechRequest.input,
|
||||
voice: voice.id,
|
||||
abortSignal: signal,
|
||||
})
|
||||
|
||||
signal.throwIfAborted()
|
||||
|
||||
// Convert the response to a blob and create an object URL
|
||||
audioUrl.value = URL.createObjectURL(new Blob([response]))
|
||||
if (isOfficialTtsProvider(previewProvider)) {
|
||||
@@ -520,7 +544,7 @@ async function generateTestSpeech() {
|
||||
|
||||
// Play the audio
|
||||
setTimeout(() => {
|
||||
if (audioPlayer.value) {
|
||||
if (!signal.aborted && audioPlayer.value) {
|
||||
void audioPlayer.value.play()
|
||||
.then(() => {
|
||||
trackVoicePreviewPlayed({
|
||||
@@ -535,6 +559,8 @@ async function generateTestSpeech() {
|
||||
}, 100)
|
||||
}
|
||||
catch (error) {
|
||||
if (signal.aborted)
|
||||
return
|
||||
console.error('Error generating speech:', error)
|
||||
errorMessage.value = errorMessageFrom(error) || 'An unknown error occurred'
|
||||
}
|
||||
@@ -559,9 +585,8 @@ function stopTestAudio() {
|
||||
|
||||
// Clean up when component is unmounted
|
||||
onUnmounted(() => {
|
||||
if (audioUrl.value) {
|
||||
URL.revokeObjectURL(audioUrl.value)
|
||||
}
|
||||
previewCancellation.abort()
|
||||
stopTestAudio()
|
||||
})
|
||||
|
||||
function updateCustomVoiceName(value: string | undefined) {
|
||||
@@ -615,6 +640,7 @@ async function handleDeleteProvider(providerId: string) {
|
||||
<ErrorContainer v-if="errorMessage" :error="errorMessage" />
|
||||
<div flex="~ col md:row gap-6">
|
||||
<SettingsCard class="md:w-[40%]">
|
||||
<FieldCheckbox v-model="speechStore.enabled" :label="t('settings.pages.modules.speech.title')" />
|
||||
<div flex="~ col gap-4">
|
||||
<div>
|
||||
<h2 class="text-lg text-neutral-500 md:text-2xl dark:text-neutral-400">
|
||||
@@ -686,7 +712,7 @@ async function handleDeleteProvider(providerId: string) {
|
||||
</div>
|
||||
|
||||
<!-- Model selection section -->
|
||||
<div v-if="activeSpeechProvider && activeSpeechProvider !== 'speech-noop'">
|
||||
<div v-if="speechStore.enabled && activeSpeechProvider && activeSpeechProvider !== 'speech-noop'">
|
||||
<div flex="~ col gap-4">
|
||||
<div>
|
||||
<h2 class="text-lg md:text-2xl">
|
||||
@@ -779,7 +805,7 @@ async function handleDeleteProvider(providerId: string) {
|
||||
</div>
|
||||
|
||||
<!-- Voice Configuration Section -->
|
||||
<div v-if="activeSpeechProvider && activeSpeechProvider !== 'speech-noop'">
|
||||
<div v-if="speechStore.enabled && activeSpeechProvider && activeSpeechProvider !== 'speech-noop'">
|
||||
<div flex="~ col gap-4">
|
||||
<div>
|
||||
<h2 class="text-lg text-neutral-500 md:text-2xl dark:text-neutral-400">
|
||||
@@ -967,8 +993,8 @@ async function handleDeleteProvider(providerId: string) {
|
||||
<button
|
||||
border="neutral-800 dark:neutral-200 solid 2" transition="border duration-250 ease-in-out"
|
||||
rounded-lg px-4 text="neutral-100 dark:neutral-900" py-2 text-sm
|
||||
:disabled="isGenerating || (!testText.trim() && !useSSML) || (useSSML && !ssmlText.trim()) || !activeSpeechVoice"
|
||||
:class="{ 'opacity-50 cursor-not-allowed': isGenerating || (!testText.trim() && !useSSML) || (useSSML && !ssmlText.trim()) || !activeSpeechVoice }"
|
||||
:disabled="!speechStore.enabled || isGenerating || (!testText.trim() && !useSSML) || (useSSML && !ssmlText.trim()) || !activeSpeechVoice"
|
||||
:class="{ 'opacity-50 cursor-not-allowed': !speechStore.enabled || isGenerating || (!testText.trim() && !useSSML) || (useSSML && !ssmlText.trim()) || !activeSpeechVoice }"
|
||||
bg="neutral-700 dark:neutral-300" @click="generateTestSpeech"
|
||||
>
|
||||
<div flex="~ row" items-center gap-2>
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
import en from '@proj-airi/i18n/locales/en'
|
||||
|
||||
import { useVisionStore } from '@proj-airi/stage-ui/stores/modules/vision'
|
||||
import { createPinia, disposePinia } from 'pinia'
|
||||
import { expect, it, onTestFinished } from 'vitest'
|
||||
import { render } from 'vitest-browser-vue'
|
||||
import { page } from 'vitest/browser'
|
||||
import { createI18n } from 'vue-i18n'
|
||||
import { createMemoryHistory, createRouter } from 'vue-router'
|
||||
|
||||
import VisionPage from './vision.vue'
|
||||
|
||||
import 'virtual:uno.css'
|
||||
|
||||
// https://github.com/moeru-ai/airi/issues/3153
|
||||
// ROOT CAUSE:
|
||||
// Provider cards lacked a disabled choice. None now remains available even without configured providers.
|
||||
it.each(['provider', 'switch'] as const)('issue #3153: disables vision through the %s control even when the selected provider is unavailable', async (control) => {
|
||||
localStorage.clear()
|
||||
localStorage.setItem('settings/vision/active-provider', 'unavailable-vision')
|
||||
localStorage.setItem('settings/vision/active-model', 'saved-model')
|
||||
const pinia = createPinia()
|
||||
const router = createRouter({ history: createMemoryHistory(), routes: [{ path: '/:path(.*)*', component: { template: '<div />' } }] })
|
||||
await router.push('/settings/modules/vision')
|
||||
const view = render(VisionPage, {
|
||||
global: { plugins: [pinia, router, createI18n({ legacy: false, locale: 'en', messages: { en } })] },
|
||||
})
|
||||
onTestFinished(() => {
|
||||
view.unmount()
|
||||
disposePinia(pinia)
|
||||
localStorage.clear()
|
||||
})
|
||||
|
||||
const noneCard = view.getByRole('radio', { name: 'None', exact: true }).element().closest('label')!
|
||||
const toggle = view.getByRole('switch', { name: 'Vision', exact: true })
|
||||
if (control === 'switch')
|
||||
await toggle.click()
|
||||
else
|
||||
await page.elementLocator(noneCard).click()
|
||||
await expect.element(view.getByRole('radio', { name: 'None', exact: true })).toBeChecked()
|
||||
const vision = useVisionStore(pinia)
|
||||
expect(vision.enabled).toBe(false)
|
||||
expect(vision.configured).toBe(false)
|
||||
expect(vision.activeProvider).toBe('unavailable-vision')
|
||||
expect(vision.activeModel).toBe('saved-model')
|
||||
await expect.element(toggle).not.toBeChecked()
|
||||
await toggle.click()
|
||||
expect(vision.enabled).toBe(true)
|
||||
expect(vision.activeProvider).toBe('unavailable-vision')
|
||||
expect(vision.activeModel).toBe('saved-model')
|
||||
})
|
||||
@@ -56,9 +56,15 @@ watch(activeProvider, async (provider) => {
|
||||
}, { immediate: true })
|
||||
|
||||
async function selectProvider(provider: string) {
|
||||
if (!provider) {
|
||||
visionStore.enabled = false
|
||||
return
|
||||
}
|
||||
await airiCardStore.selectActiveCardVisionProvider(provider)
|
||||
}
|
||||
|
||||
const selectedProvider = computed(() => visionStore.enabled ? activeProvider.value : '')
|
||||
|
||||
async function persistSelection() {
|
||||
await airiCardStore.updateActiveCardVision({ provider: activeProvider.value, model: activeModel.value })
|
||||
}
|
||||
@@ -106,6 +112,7 @@ function formatRelativeTime(timestamp: number | null) {
|
||||
<template>
|
||||
<div :class="['flex', 'flex-col', 'gap-6']">
|
||||
<SettingsCard>
|
||||
<FieldCheckbox v-model="visionStore.enabled" :label="t('settings.pages.modules.vision.title')" />
|
||||
<FieldCheckbox
|
||||
v-model="useForChat"
|
||||
:label="t('stage.chat.images.use-vision')"
|
||||
@@ -130,16 +137,23 @@ function formatRelativeTime(timestamp: number | null) {
|
||||
</div>
|
||||
<div :class="['max-w-full']">
|
||||
<fieldset
|
||||
v-if="moduleVisionProvidersMetadata.length > 0"
|
||||
:class="['flex', 'min-w-0', 'flex-row', 'gap-4', 'overflow-x-auto', 'scroll-smooth']"
|
||||
role="radiogroup"
|
||||
>
|
||||
<RadioCardSimple
|
||||
id="vision-none"
|
||||
:model-value="selectedProvider"
|
||||
name="provider"
|
||||
value=""
|
||||
:title="t('settings.pages.modules.availability.none')"
|
||||
@update:model-value="selectProvider"
|
||||
/>
|
||||
<RadioCardSimple
|
||||
v-for="metadata in moduleVisionProvidersMetadata"
|
||||
:id="metadata.id"
|
||||
:key="metadata.id"
|
||||
:data-settings-search="moduleProviderSearchKey('vision', metadata.id)"
|
||||
:model-value="activeProvider"
|
||||
:model-value="selectedProvider"
|
||||
name="provider"
|
||||
:value="metadata.id"
|
||||
:title="metadata.localizedName || 'Unknown'"
|
||||
@@ -215,7 +229,7 @@ function formatRelativeTime(timestamp: number | null) {
|
||||
/>
|
||||
</RouterLink>
|
||||
</fieldset>
|
||||
<div v-else>
|
||||
<div v-if="moduleVisionProvidersMetadata.length === 0">
|
||||
<RouterLink
|
||||
to="/settings/providers#vision"
|
||||
:class="[
|
||||
@@ -251,7 +265,7 @@ function formatRelativeTime(timestamp: number | null) {
|
||||
</div>
|
||||
</SettingsCard>
|
||||
|
||||
<SettingsCard v-if="activeProvider && supportsModelListing">
|
||||
<SettingsCard v-if="selectedProvider && supportsModelListing">
|
||||
<div :class="['flex', 'flex-col', 'gap-4']">
|
||||
<div>
|
||||
<h2 :class="['text-lg', 'md:text-2xl']">
|
||||
@@ -309,7 +323,7 @@ function formatRelativeTime(timestamp: number | null) {
|
||||
</div>
|
||||
</SettingsCard>
|
||||
|
||||
<SettingsCard v-else-if="activeProvider && !supportsModelListing">
|
||||
<SettingsCard v-else-if="selectedProvider && !supportsModelListing">
|
||||
<div :class="['flex', 'flex-col', 'gap-4']">
|
||||
<div>
|
||||
<h2 :class="['text-lg', 'text-neutral-500', 'md:text-2xl', 'dark:text-neutral-400']">
|
||||
|
||||
@@ -2,6 +2,18 @@
|
||||
|
||||
Shared core for stage
|
||||
|
||||
## Optional modules
|
||||
|
||||
Provider lists expose `None` for optional features. Switches control module activation.
|
||||
Consciousness remains required. Minecraft runs as an external service and keeps its own lifecycle.
|
||||
|
||||
Hearing, Speech, and Vision persist their enable states separately from provider and model selections.
|
||||
Disabling these modules preserves credentials and selections. Re-enabling restores their use.
|
||||
Their synchronized stores replicate the enable states across desktop windows.
|
||||
Hearing cancels active transcription and rejects results from its previous enabled period.
|
||||
The voice host also stops listening and dictation. Text chat can continue without transcribing stored recordings.
|
||||
Disabling Speech stops its producers and playback. Text generation and tool work continue until the chat turn ends.
|
||||
|
||||
## Experimental features
|
||||
|
||||
Register flags in `libs/feature-flags.ts`. Read their state through `useFeatureFlagsStore().isEnabled(key)`.
|
||||
|
||||
@@ -19,7 +19,7 @@ const microphoneError = computed(() => snapshot.value.microphone?.error)
|
||||
const transcript = computed(() => snapshot.value.input?.text)
|
||||
const isTranscribing = computed(() => snapshot.value.input?.phase === 'finalizing')
|
||||
const { volumeLevel } = useAudioAnalyzer()
|
||||
const { configured } = storeToRefs(useHearingStore())
|
||||
const { configured, enabled: hearingEnabled } = storeToRefs(useHearingStore())
|
||||
const issue = computed(() => {
|
||||
// Device failure prevents all input, so it takes precedence over provider feedback.
|
||||
if (microphoneError.value)
|
||||
@@ -53,7 +53,7 @@ const label = computed(() => {
|
||||
|
||||
<template>
|
||||
<StatusCapsule
|
||||
v-if="enabled || issue || isTranscribing"
|
||||
v-if="hearingEnabled && (enabled || issue || isTranscribing)"
|
||||
:align="align"
|
||||
:data-status="state"
|
||||
:tone="issue ? 'error' : 'neutral'"
|
||||
|
||||
@@ -6,6 +6,8 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { createApp } from 'vue'
|
||||
|
||||
import { useAiriCardStore } from '../stores/modules/airi-card'
|
||||
import { useHearingStore } from '../stores/modules/hearing'
|
||||
import { useSpeechStore } from '../stores/modules/speech'
|
||||
import { useProviderConfigStore } from '../stores/providers/config'
|
||||
import { useProviderStore } from '../stores/providers/provider'
|
||||
import { useSettingsNeedsAttention } from './use-settings-needs-attention'
|
||||
@@ -76,6 +78,29 @@ describe('useSettingsNeedsAttention', () => {
|
||||
|
||||
afterEach(() => vi.unstubAllGlobals())
|
||||
|
||||
// https://github.com/moeru-ai/airi/issues/3153
|
||||
// ROOT CAUSE:
|
||||
// Saved providers counted as active. Disabled modules now leave the provider usage list without deleting shared settings.
|
||||
it('issue #3153: removes disabled module warnings while retaining another module that uses the same provider', async () => {
|
||||
const cards = useAiriCardStore()
|
||||
await cards.initialize()
|
||||
const provider = 'removed-provider'
|
||||
const characterId = await cards.addCard(cardWithSpeech(provider), 'import')
|
||||
const hearing = useHearingStore()
|
||||
hearing.activeTranscriptionProvider = provider
|
||||
await waitForProviderRegistry()
|
||||
const { moduleSelections } = useSettingsNeedsAttention()
|
||||
expect(moduleSelections.value.some(selection => selection.key === 'hearing')).toBe(true)
|
||||
|
||||
hearing.enabled = false
|
||||
expect(moduleSelections.value.some(selection => selection.key === 'hearing')).toBe(false)
|
||||
expect(moduleSelections.value.some(selection => selection.key === `${characterId}:speech`)).toBe(true)
|
||||
expect(hearing.activeTranscriptionProvider).toBe(provider)
|
||||
|
||||
useSpeechStore().enabled = false
|
||||
expect(moduleSelections.value.some(selection => selection.key === `${characterId}:speech`)).toBe(false)
|
||||
})
|
||||
|
||||
// ROOT CAUSE:
|
||||
//
|
||||
// The added flag of a provider stays after an update removes the provider.
|
||||
|
||||
@@ -324,13 +324,13 @@ export function useSettingsNeedsAttention() {
|
||||
const modulesByProvider = computed(() => {
|
||||
const selections: Array<readonly [provider: string, module: Module]> = [
|
||||
[consciousnessStore.activeProvider, 'consciousness'],
|
||||
[speechStore.activeSpeechProvider, 'speech'],
|
||||
[hearingStore.activeTranscriptionProvider, 'hearing'],
|
||||
[visionStore.activeProvider, 'vision'],
|
||||
[speechStore.enabled ? speechStore.activeSpeechProvider : '', 'speech'],
|
||||
[hearingStore.enabled ? hearingStore.activeTranscriptionProvider : '', 'hearing'],
|
||||
[visionStore.enabled ? visionStore.activeProvider : '', 'vision'],
|
||||
...[...cardStore.characterModules.values()].flatMap(modules => [
|
||||
[modules.consciousness.provider, 'consciousness'],
|
||||
[modules.speech.provider, 'speech'],
|
||||
[modules.vision.provider, 'vision'],
|
||||
[speechStore.enabled ? modules.speech.provider : '', 'speech'],
|
||||
[visionStore.enabled ? modules.vision.provider : '', 'vision'],
|
||||
] as const),
|
||||
]
|
||||
const result = new Map<string, Set<Module>>()
|
||||
@@ -417,9 +417,9 @@ export function useSettingsNeedsAttention() {
|
||||
...[...cardStore.characterModules].flatMap(([characterId, modules]) => CHARACTER_MODULES.map(module => ({
|
||||
characterId: characterId as string | undefined,
|
||||
module: module as Module,
|
||||
provider: modules[module].provider,
|
||||
provider: (module === 'vision' && !visionStore.enabled) || (module === 'speech' && !speechStore.enabled) ? '' : modules[module].provider,
|
||||
}))),
|
||||
{ characterId: undefined, module: 'hearing' as Module, provider: hearingStore.activeTranscriptionProvider },
|
||||
{ characterId: undefined, module: 'hearing' as Module, provider: hearingStore.enabled ? hearingStore.activeTranscriptionProvider : '' },
|
||||
]
|
||||
return selections
|
||||
.filter(({ provider }) => provider && provider !== 'speech-noop')
|
||||
@@ -504,6 +504,8 @@ export function useSettingsNeedsAttention() {
|
||||
* A provider that the speech module cannot use shows as its own item instead.
|
||||
*/
|
||||
const speechProblems = computed(() => characterSpeech.value.flatMap((speech): Problem[] => {
|
||||
if (!speechStore.enabled)
|
||||
return []
|
||||
const missing: MissingSpeechField[] = speech.simulated ? ['voice'] : speech.providerUsable ? speech.missing : []
|
||||
const fieldsKey = speechFieldsKeyOf(missing)
|
||||
if (!fieldsKey)
|
||||
@@ -539,6 +541,8 @@ export function useSettingsNeedsAttention() {
|
||||
* Push to Talk and dictation explain it when the person presses them, so they do not show here.
|
||||
*/
|
||||
const hearingProblems = computed((): Problem[] => {
|
||||
if (!hearingStore.enabled)
|
||||
return []
|
||||
const provider = hearingStore.activeTranscriptionProvider
|
||||
const { microphoneOn, mode, configured, simulated } = voiceInput.value
|
||||
if (!simulated) {
|
||||
|
||||
@@ -107,6 +107,24 @@ describe('useVisionInference', () => {
|
||||
|
||||
const toolImage = { imageDataUrl: 'data:image/png;base64,Zm9v', workloadId: 'tool:image' as const }
|
||||
|
||||
it('cancels an active read on disable and blocks later provider calls', async () => {
|
||||
const releases = holdEachRead()
|
||||
const { runVisionInference, lastText } = useVisionInference()
|
||||
const read = runVisionInference(toolImage)
|
||||
const cancelled = expect(read).rejects.toMatchObject({ name: 'AbortError' })
|
||||
await vi.advanceTimersByTimeAsync(1)
|
||||
expect(stream).toHaveBeenCalledOnce()
|
||||
|
||||
useVisionStore().enabled = false
|
||||
expect(stream.mock.calls[0][3]?.abortSignal?.aborted).toBe(true)
|
||||
releases[0]()
|
||||
await cancelled
|
||||
await expect(runVisionInference(toolImage)).rejects.toMatchObject({ name: 'AbortError' })
|
||||
expect(stream).toHaveBeenCalledOnce()
|
||||
expect(useVisionActivityStore().inferenceCount).toBe(0)
|
||||
expect(lastText.value).toBe('')
|
||||
})
|
||||
|
||||
it('reads one image at a time for a provider that answers one read at a time', async () => {
|
||||
// ROOT CAUSE:
|
||||
//
|
||||
|
||||
@@ -6,7 +6,7 @@ import type { VisionWorkloadId } from './use-vision-workloads'
|
||||
import { errorMessageFrom } from '@moeru/std'
|
||||
import { DEFAULT_CONCURRENT_VISION_READS } from '@proj-airi/provider-inference'
|
||||
import { storeToRefs } from 'pinia'
|
||||
import { ref } from 'vue'
|
||||
import { ref, watch } from 'vue'
|
||||
|
||||
import { useLLM } from '../../stores/ai/chat-llm/llm'
|
||||
import { reportActivity, useVisionActivityStore, useVisionStore } from '../../stores/modules/vision'
|
||||
@@ -57,6 +57,7 @@ export function useVisionInference() {
|
||||
async function describeImage(providerId: string, modelId: string, input: VisionInferenceInput) {
|
||||
const thinking = ollamaThinkingEnabled.value
|
||||
const provider = await providersStore.getChatProviderInstance(providerId)
|
||||
input.abortSignal?.throwIfAborted()
|
||||
const workload = getVisionWorkload(input.workloadId)
|
||||
const prompt = input.promptOverride ?? workload.prompt
|
||||
const { url } = parseDataUrl(input.imageDataUrl)
|
||||
@@ -112,6 +113,24 @@ export function useVisionInference() {
|
||||
}
|
||||
|
||||
async function runVisionInference(input: VisionInferenceInput) {
|
||||
if (!visionStore.enabled)
|
||||
throw new DOMException('Vision is disabled', 'AbortError')
|
||||
|
||||
const cancellation = new AbortController()
|
||||
const stop = watch(() => visionStore.enabled, (enabled) => {
|
||||
if (!enabled)
|
||||
cancellation.abort(new DOMException('Vision is disabled', 'AbortError'))
|
||||
}, { flush: 'sync' })
|
||||
const signal = AbortSignal.any([cancellation.signal, ...(input.abortSignal ? [input.abortSignal] : [])])
|
||||
try {
|
||||
return await readImage({ ...input, abortSignal: signal })
|
||||
}
|
||||
finally {
|
||||
stop()
|
||||
}
|
||||
}
|
||||
|
||||
async function readImage(input: VisionInferenceInput) {
|
||||
const providerId = input.selection?.provider ?? activeProvider.value
|
||||
const modelId = input.selection?.model ?? activeModel.value
|
||||
if (!providerId || !modelId)
|
||||
@@ -133,7 +152,9 @@ export function useVisionInference() {
|
||||
}
|
||||
|
||||
try {
|
||||
lastText.value = await describeImage(providerId, modelId, input)
|
||||
const text = await describeImage(providerId, modelId, input)
|
||||
input.abortSignal?.throwIfAborted()
|
||||
lastText.value = text
|
||||
}
|
||||
catch (error) {
|
||||
if (!input.abortSignal?.aborted)
|
||||
|
||||
@@ -18,7 +18,7 @@ export function useVoiceListening() {
|
||||
const devices = useSettingsAudioDevice()
|
||||
const hearing = useHearingStore()
|
||||
|
||||
watch(() => devices.enabled && listensContinuously(hearing.inputMode), (listen) => {
|
||||
watch(() => hearing.enabled && devices.enabled && listensContinuously(hearing.inputMode), (listen) => {
|
||||
if (listen)
|
||||
voice.startListening()
|
||||
else
|
||||
@@ -48,7 +48,7 @@ export function useVoicePushToTalk(options: VoicePushToTalkOptions = {}) {
|
||||
const sessions = useChatSessionStore()
|
||||
|
||||
return useVoiceHold({
|
||||
enabled: () => devices.enabled && hearing.inputMode === 'push-to-talk',
|
||||
enabled: () => hearing.enabled && devices.enabled && hearing.inputMode === 'push-to-talk',
|
||||
sessionId: () => sessions.activeSessionId,
|
||||
begin: (sessionId) => {
|
||||
if (!hearing.configured) {
|
||||
|
||||
@@ -781,10 +781,15 @@ export const useChatStore = defineStore('chat', () => {
|
||||
}
|
||||
|
||||
if (!text) {
|
||||
if (!hearing.enabled) {
|
||||
// Stored recordings stay intact. A text-only model can continue the conversation while recognition is disabled.
|
||||
turn.content[index] = { type: 'text', text: t('stage.chat.audio.hearing-disabled') }
|
||||
continue
|
||||
}
|
||||
// The recording belongs to this turn. A voice input of another turn can run at the same time.
|
||||
transcriber ??= hearing.createTranscriber(undefined, { traceParent: turnSpan })
|
||||
if (!hearing.configured)
|
||||
throw new Error('Configure a transcription provider to send audio to this model')
|
||||
transcriber ??= hearing.createTranscriber(undefined, { traceParent: turnSpan })
|
||||
|
||||
const blob = new Blob([new Uint8Array(decodeBase64(part.data))], { type: part.format === 'mp3' ? 'audio/mpeg' : 'audio/wav' })
|
||||
let completed = false
|
||||
|
||||
@@ -648,6 +648,9 @@ export const useAiriCardStore = defineStore('airi-card', () => {
|
||||
async function selectActiveCardVisionProvider(provider: string) {
|
||||
await pendingAuthenticationSetup
|
||||
const vision = useVisionStore()
|
||||
vision.enabled = true
|
||||
if (vision.activeProvider === provider)
|
||||
return true
|
||||
vision.activeProvider = provider
|
||||
vision.resetModelSelection()
|
||||
await vision.loadModelsForProvider(provider)
|
||||
|
||||
@@ -29,6 +29,23 @@ vi.mock('vue-i18n', () => ({
|
||||
}))
|
||||
|
||||
describe('official provider module defaults', () => {
|
||||
it('keeps explicit None selections disabled when authentication applies defaults', async () => {
|
||||
const hearing = useHearingStore()
|
||||
const vision = useVisionStore()
|
||||
const speech = useSpeechStore()
|
||||
hearing.enabled = false
|
||||
vision.enabled = false
|
||||
speech.enabled = false
|
||||
speech.activeSpeechProvider = 'speech-noop'
|
||||
|
||||
await configureAsDefaultsIfEmpty()
|
||||
|
||||
expect(hearing.configured).toBe(false)
|
||||
expect(vision.configured).toBe(false)
|
||||
expect(speech.configured).toBe(false)
|
||||
expect(speech.activeSpeechProvider).toBe('speech-noop')
|
||||
})
|
||||
|
||||
beforeEach(() => {
|
||||
setActivePinia(createPinia())
|
||||
vi.stubGlobal('fetch', vi.fn(async () => new Response(JSON.stringify({
|
||||
|
||||
@@ -35,7 +35,7 @@ export async function configureAsDefaultsIfEmpty(): Promise<boolean> {
|
||||
const usesOfficialConsciousness = consciousnessStore.activeProvider === officialModuleDefaults.consciousness.provider
|
||||
const needsHearingDefault = !hearingStore.activeTranscriptionProvider
|
||||
const usesOfficialHearing = hearingStore.activeTranscriptionProvider === officialModuleDefaults.hearing.provider
|
||||
const needsSpeechDefault = !speechStore.activeSpeechProvider || speechStore.activeSpeechProvider === 'speech-noop'
|
||||
const needsSpeechDefault = speechStore.enabled && (!speechStore.activeSpeechProvider || speechStore.activeSpeechProvider === 'speech-noop')
|
||||
const usesOfficialSpeech = speechStore.activeSpeechProvider === officialModuleDefaults.speech.provider
|
||||
const needsVisionDefault = !visionStore.activeProvider
|
||||
const usesOfficialVision = visionStore.activeProvider === officialModuleDefaults.vision.provider
|
||||
|
||||
@@ -8,6 +8,7 @@ import { createI18n } from 'vue-i18n'
|
||||
import HearingStatus from '../../components/scenarios/status/hearing-status.vue'
|
||||
|
||||
import { getSpeechBusContext, voiceSnapshotChanged } from '../../services/speech/bus'
|
||||
import { useHearingStore } from './hearing'
|
||||
|
||||
const cleanups: (() => void)[] = []
|
||||
afterEach(() => {
|
||||
@@ -24,4 +25,6 @@ it('shows final transcription work and reports a provider failure through the ho
|
||||
await expect.element(view.getByRole('status')).toHaveTextContent('Transcribing')
|
||||
context.emit(voiceSnapshotChanged, { connected: true, drafts: [], error: 'Provider timed out', input: { requestId: 'recording', sessionId: 'alice', phase: 'settled', text: 'recognized words', segments: [{ id: 'speech-0', text: 'recognized words', final: true }] } })
|
||||
await expect.element(view.getByRole('status')).toHaveTextContent('Could not transcribe audio')
|
||||
useHearingStore(pinia).enabled = false
|
||||
await expect.element(view.getByRole('status')).not.toBeInTheDocument()
|
||||
})
|
||||
|
||||
@@ -16,7 +16,7 @@ import { useLocalStorageManualReset } from '@proj-airi/stage-shared/composables'
|
||||
import { refManualReset } from '@vueuse/core'
|
||||
import { generateTranscription } from '@xsai/generate-transcription'
|
||||
import { defineStore, storeToRefs } from 'pinia'
|
||||
import { computed, ref, toRaw, watch } from 'vue'
|
||||
import { computed, onScopeDispose, ref, toRaw, watch } from 'vue'
|
||||
|
||||
import { useAnalytics } from '../../composables/use-analytics'
|
||||
import { createHearingTranscriber } from '../../libs/audio/hearing-transcriber'
|
||||
@@ -324,6 +324,17 @@ export const useHearingStore = defineStore('hearing-store', () => {
|
||||
// Pinia synchronization owns live cross-window state. localStorage only
|
||||
// loads and saves durable values for this synchronized store.
|
||||
const persistenceOptions = { listenToStorageChanges: false }
|
||||
const enabled = useLocalStorageManualReset('settings/hearing/enabled', true, persistenceOptions)
|
||||
|
||||
// Each enabled period owns its requests. Disabling also invalidates transcribers captured by pending recordings.
|
||||
let recognition = new AbortController()
|
||||
watch(enabled, (value) => {
|
||||
if (value)
|
||||
recognition = new AbortController()
|
||||
else
|
||||
recognition.abort(new DOMException('Hearing is disabled', 'AbortError'))
|
||||
}, { immediate: true, flush: 'sync' })
|
||||
onScopeDispose(() => recognition.abort())
|
||||
|
||||
// State
|
||||
const activeTranscriptionProvider = useLocalStorageManualReset('settings/hearing/active-provider', '', persistenceOptions)
|
||||
@@ -387,7 +398,7 @@ export const useHearingStore = defineStore('hearing-store', () => {
|
||||
}
|
||||
|
||||
const configured = computed(() => {
|
||||
if (!activeTranscriptionProvider.value)
|
||||
if (!enabled.value || !activeTranscriptionProvider.value)
|
||||
return false
|
||||
|
||||
// Web Speech API doesn't strictly need a model selected (it has a default)
|
||||
@@ -407,6 +418,7 @@ export const useHearingStore = defineStore('hearing-store', () => {
|
||||
})
|
||||
|
||||
function resetState() {
|
||||
enabled.reset()
|
||||
activeTranscriptionProvider.reset()
|
||||
activeTranscriptionModel.reset()
|
||||
activeCustomModelName.reset()
|
||||
@@ -430,6 +442,8 @@ export const useHearingStore = defineStore('hearing-store', () => {
|
||||
format?: 'json' | 'verbose_json',
|
||||
options?: HearingTranscriptionInvokeOptions,
|
||||
): Promise<HearingTranscriptionResult> {
|
||||
const signal = AbortSignal.any([recognition.signal, ...(options?.signal ? [options.signal] : [])])
|
||||
signal.throwIfAborted()
|
||||
const normalizedInput = (input instanceof File ? { file: input } : input ?? {}) as {
|
||||
file?: File
|
||||
fileName?: string
|
||||
@@ -462,7 +476,7 @@ export const useHearingStore = defineStore('hearing-store', () => {
|
||||
|
||||
try {
|
||||
if (features.supportsStreamOutput && streamExecutor) {
|
||||
const request = { ...provider.transcription(model, options?.providerOptions), abortSignal: options?.signal }
|
||||
const request = { ...provider.transcription(model, options?.providerOptions), abortSignal: signal }
|
||||
|
||||
// Recorder files contain encoded media. Sherpaw accepts only the raw PCM16 stream from the live pipeline.
|
||||
if (providerId === SHERPAW_TRANSCRIPTION_PROVIDER_ID && normalizedInput.file && !normalizedInput.inputAudioStream) {
|
||||
@@ -522,12 +536,14 @@ export const useHearingStore = defineStore('hearing-store', () => {
|
||||
const useVerboseJson = !format && threshold > CONFIDENCE_THRESHOLD_DISABLED
|
||||
const response = await generateTranscription({
|
||||
...provider.transcription(model, options?.providerOptions),
|
||||
abortSignal: options?.signal,
|
||||
abortSignal: signal,
|
||||
file: normalizedInput.file,
|
||||
fileName: resolveTranscriptionFileName(normalizedInput.file, normalizedInput.fileName),
|
||||
responseFormat: useVerboseJson ? 'verbose_json' : format,
|
||||
})
|
||||
|
||||
signal.throwIfAborted()
|
||||
|
||||
if (useVerboseJson) {
|
||||
if (response.segments) {
|
||||
verboseJsonNotSupported.value = false
|
||||
@@ -561,6 +577,8 @@ export const useHearingStore = defineStore('hearing-store', () => {
|
||||
|
||||
/** Freezes Hearing settings for one request. The controller supplies media and owns cancellation. */
|
||||
function createTranscriber(providerId = activeTranscriptionProvider.value, trace: HearingTranscriberTraceOptions = {}): StreamingTranscriber {
|
||||
const recognitionSignal = recognition.signal
|
||||
recognitionSignal.throwIfAborted()
|
||||
const config = structuredClone(toRaw(providerStore.getProviderConfig(providerId) ?? {}))
|
||||
const model = resolveActiveTranscriptionModel(activeTranscriptionModel.value, config)
|
||||
const threshold = confidenceThreshold.value
|
||||
@@ -573,7 +591,7 @@ export const useHearingStore = defineStore('hearing-store', () => {
|
||||
upload = 'media-stream'
|
||||
else if (providerId === SHERPAW_TRANSCRIPTION_PROVIDER_ID || features.supportsStreamInput)
|
||||
upload = 'pcm'
|
||||
return createHearingTranscriber(async (audio, signal) => {
|
||||
const transcriber = createHearingTranscriber(async (audio, signal) => {
|
||||
const setupError = resolveActiveTranscriptionProviderError(providerId)
|
||||
if (setupError)
|
||||
throw new Error(setupError)
|
||||
@@ -625,9 +643,16 @@ export const useHearingStore = defineStore('hearing-store', () => {
|
||||
throw error
|
||||
}
|
||||
}, trace)
|
||||
return {
|
||||
transcribe: request => transcriber.transcribe({
|
||||
...request,
|
||||
signal: AbortSignal.any([request.signal, recognitionSignal]),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
enabled,
|
||||
activeTranscriptionProvider,
|
||||
activeTranscriptionModel,
|
||||
availableProvidersMetadata,
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
import type { PcmBlock } from '@proj-airi/pipelines-audio'
|
||||
import type { LeadershipMode, SyncedPiniaRuntime } from 'pinia-plugin-synced'
|
||||
|
||||
import { generateTranscription } from '@xsai/generate-transcription'
|
||||
import { createPinia, disposePinia } from 'pinia'
|
||||
import { createSyncedPiniaPlugin } from 'pinia-plugin-synced'
|
||||
import { afterEach, expect, it, vi } from 'vitest'
|
||||
import { createApp, nextTick } from 'vue'
|
||||
import { createI18n } from 'vue-i18n'
|
||||
|
||||
import { OFFICIAL_SPEECH_PROVIDER_ID, OFFICIAL_SPEECH_STREAMING_PROVIDER_ID } from '../../libs/providers/providers/official'
|
||||
import { useProviderConfigStore } from '../providers/config'
|
||||
import { useHearingStore } from './hearing'
|
||||
import { useSpeechStore } from './speech'
|
||||
import { useVisionStore } from './vision/store'
|
||||
|
||||
vi.mock('@xsai/generate-transcription', () => ({ generateTranscription: vi.fn() }))
|
||||
|
||||
const contexts: { pinia: ReturnType<typeof createPinia>, runtime?: SyncedPiniaRuntime, unmount: () => void }[] = []
|
||||
|
||||
function createContext(namespace?: string, leadership?: LeadershipMode) {
|
||||
const pinia = createPinia()
|
||||
const runtime = namespace ? createSyncedPiniaPlugin({ namespace, leadership, callTimeout: 1000 }) : undefined
|
||||
if (runtime)
|
||||
pinia.use(runtime.plugin)
|
||||
let hearing!: ReturnType<typeof useHearingStore>
|
||||
let vision!: ReturnType<typeof useVisionStore>
|
||||
let speech!: ReturnType<typeof useSpeechStore>
|
||||
const app = createApp({
|
||||
setup() {
|
||||
hearing = useHearingStore(pinia)
|
||||
vision = useVisionStore(pinia)
|
||||
speech = useSpeechStore(pinia)
|
||||
return () => null
|
||||
},
|
||||
}).use(createI18n({ legacy: false, locale: 'en', messages: { en: {} }, missingWarn: false, fallbackWarn: false })).use(pinia)
|
||||
app.mount(document.createElement('div'))
|
||||
contexts.push({ pinia, runtime, unmount: () => app.unmount() })
|
||||
return { pinia, runtime, hearing, vision, speech }
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
for (const { pinia, runtime, unmount } of contexts.splice(0)) {
|
||||
unmount()
|
||||
runtime?.dispose()
|
||||
disposePinia(pinia)
|
||||
}
|
||||
localStorage.clear()
|
||||
vi.mocked(generateTranscription).mockReset()
|
||||
})
|
||||
|
||||
// https://github.com/moeru-ai/airi/issues/3153
|
||||
// ROOT CAUSE:
|
||||
// Provider selection also implied activation. Persisted enable states now retain selections while disabling the modules.
|
||||
it('issue #3153: retains disabled modules and their settings after reopening', async () => {
|
||||
const first = createContext()
|
||||
first.hearing.activeTranscriptionProvider = 'browser-web-speech-api'
|
||||
first.hearing.activeTranscriptionModel = 'web-speech-api'
|
||||
first.vision.activeProvider = 'vision-ollama'
|
||||
first.vision.activeModel = 'saved-vision-model'
|
||||
first.hearing.enabled = false
|
||||
first.vision.enabled = false
|
||||
first.speech.enabled = false
|
||||
await nextTick()
|
||||
|
||||
const reopened = createContext()
|
||||
expect(reopened.hearing.enabled).toBe(false)
|
||||
expect(reopened.hearing.configured).toBe(false)
|
||||
expect(reopened.hearing.activeTranscriptionProvider).toBe('browser-web-speech-api')
|
||||
expect(reopened.hearing.activeTranscriptionModel).toBe('web-speech-api')
|
||||
expect(reopened.vision.enabled).toBe(false)
|
||||
expect(reopened.vision.configured).toBe(false)
|
||||
expect(reopened.vision.activeModel).toBe('saved-vision-model')
|
||||
expect(reopened.speech.enabled).toBe(false)
|
||||
expect(reopened.speech.configured).toBe(false)
|
||||
reopened.hearing.enabled = true
|
||||
reopened.vision.enabled = true
|
||||
expect(reopened.hearing.configured).toBe(true)
|
||||
expect(reopened.vision.configured).toBe(true)
|
||||
})
|
||||
|
||||
it('replicates disabled states without a follower proposal', async () => {
|
||||
const namespace = `optional-modules:${crypto.randomUUID()}`
|
||||
const leader = createContext(namespace, 'leader-only')
|
||||
const follower = createContext(namespace, 'follower-only')
|
||||
await vi.waitFor(() => expect(follower.runtime?.getLeaderId()).toBe(leader.runtime?.participantId))
|
||||
let leaderMutations = 0
|
||||
let followerMutations = 0
|
||||
for (const store of [leader.hearing, leader.vision, leader.speech])
|
||||
store.$subscribe(() => leaderMutations++, { flush: 'sync' })
|
||||
for (const store of [follower.hearing, follower.vision, follower.speech])
|
||||
store.$subscribe(() => followerMutations++, { flush: 'sync' })
|
||||
|
||||
leader.hearing.enabled = false
|
||||
leader.vision.enabled = false
|
||||
leader.speech.enabled = false
|
||||
await vi.waitFor(() => expect(follower.hearing.enabled).toBe(false))
|
||||
await vi.waitFor(() => expect(follower.vision.enabled).toBe(false))
|
||||
await vi.waitFor(() => expect(follower.speech.enabled).toBe(false))
|
||||
await new Promise(resolve => setTimeout(resolve, 50))
|
||||
expect(leaderMutations).toBe(3)
|
||||
expect(followerMutations).toBe(3)
|
||||
})
|
||||
|
||||
// https://github.com/moeru-ai/airi/issues/3153
|
||||
// ROOT CAUSE:
|
||||
// The official source card represents two providers. Re-enabling must retain the saved streaming provider and model.
|
||||
it('issue #3153: restores the streaming selection through the official source card', async () => {
|
||||
const { speech } = createContext()
|
||||
speech.activeSpeechProvider = OFFICIAL_SPEECH_STREAMING_PROVIDER_ID
|
||||
speech.activeSpeechModel = 'saved-streaming-model'
|
||||
speech.activeSpeechVoiceId = 'saved-voice'
|
||||
await speech.selectSource('speech-noop')
|
||||
expect(speech.enabled).toBe(false)
|
||||
const selection = await speech.selectSource(OFFICIAL_SPEECH_PROVIDER_ID)
|
||||
expect(speech.enabled).toBe(true)
|
||||
expect(selection).toEqual({ provider: OFFICIAL_SPEECH_STREAMING_PROVIDER_ID, model: 'saved-streaming-model', voice_id: 'saved-voice' })
|
||||
})
|
||||
|
||||
// https://github.com/moeru-ai/airi/issues/3153
|
||||
// ROOT CAUSE:
|
||||
// Recordings captured a provider without a module lifetime. Disabling now aborts that lifetime and rejects late results.
|
||||
it('issue #3153: cancels an active transcription and rejects a captured transcriber after re-enabling', async () => {
|
||||
const { hearing, pinia } = createContext()
|
||||
const provider = 'openai-compatible-audio-transcription'
|
||||
await useProviderConfigStore(pinia).ensureProvider(provider, provider, { apiKey: 'saved-key', baseUrl: 'https://asr.invalid/v1/' })
|
||||
hearing.activeTranscriptionProvider = provider
|
||||
hearing.activeTranscriptionModel = 'whisper-1'
|
||||
const pending = Promise.withResolvers<Awaited<ReturnType<typeof generateTranscription>>>()
|
||||
vi.mocked(generateTranscription).mockReturnValue(pending.promise)
|
||||
const transcriber = hearing.createTranscriber()
|
||||
const audio = new ReadableStream<PcmBlock>({
|
||||
start(controller) {
|
||||
controller.enqueue({ range: { sourceId: 'recording', startFrame: 0, endFrame: 160 }, sampleRate: 16000, channels: [new Float32Array(160)] })
|
||||
controller.close()
|
||||
},
|
||||
})
|
||||
const reader = transcriber.transcribe({ audio, signal: new AbortController().signal }).getReader()
|
||||
const read = reader.read().catch(error => error)
|
||||
await vi.waitFor(() => expect(generateTranscription).toHaveBeenCalledTimes(1))
|
||||
const request = vi.mocked(generateTranscription).mock.calls[0][0]
|
||||
hearing.enabled = false
|
||||
expect(request.abortSignal?.aborted).toBe(true)
|
||||
expect(await read).toMatchObject({ name: 'AbortError' })
|
||||
expect(() => hearing.createTranscriber()).toThrow()
|
||||
pending.resolve({ text: 'late words', duration: 0, language: 'en', segments: [], words: [] })
|
||||
await expect(reader.read()).rejects.toMatchObject({ name: 'AbortError' })
|
||||
expect(useProviderConfigStore(pinia).getProviderConfig(provider)?.apiKey).toBe('saved-key')
|
||||
|
||||
hearing.enabled = true
|
||||
const stale = transcriber.transcribe({ audio: new ReadableStream<PcmBlock>(), signal: new AbortController().signal }).getReader()
|
||||
await expect(stale.read()).rejects.toMatchObject({ name: 'AbortError' })
|
||||
expect(generateTranscription).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
@@ -3,9 +3,11 @@ import type { Component } from 'vue'
|
||||
import en from '@proj-airi/i18n/locales/en'
|
||||
|
||||
import { MotionPlugin } from '@vueuse/motion'
|
||||
import { generateSpeech } from '@xsai/generate-speech'
|
||||
import { createPinia, disposePinia } from 'pinia'
|
||||
import { createSyncedPiniaPlugin } from 'pinia-plugin-synced'
|
||||
import { afterEach, expect, it, vi } from 'vitest'
|
||||
import { page } from 'vitest/browser'
|
||||
import { createApp, h } from 'vue'
|
||||
import { createI18n } from 'vue-i18n'
|
||||
import { createMemoryHistory, createRouter } from 'vue-router'
|
||||
@@ -21,6 +23,7 @@ import { useSpeechStore } from './speech'
|
||||
// Analytics delivery is external IO. Exercise the real page and stores while
|
||||
// recording its outgoing payload instead of sending product events.
|
||||
vi.mock('../../libs/product-signals/client', { spy: true })
|
||||
vi.mock('@xsai/generate-speech', { spy: true })
|
||||
|
||||
const cleanups: Array<() => void> = []
|
||||
|
||||
@@ -61,6 +64,81 @@ afterEach(() => {
|
||||
localStorage.clear()
|
||||
})
|
||||
|
||||
// https://github.com/moeru-ai/airi/issues/3153
|
||||
// ROOT CAUSE:
|
||||
// None replaced the selected speech configuration. The source action now retains it and commits activation in the leader.
|
||||
it.each(['provider', 'switch'] as const)('issue #3153: disables speech with the %s control from a follower and restores the saved model and voice', async (control) => {
|
||||
localStorage.clear()
|
||||
vi.stubGlobal('fetch', vi.fn<typeof fetch>(async () => Response.json({ voices: [], data: [] })))
|
||||
const namespace = `speech-disabled:${crypto.randomUUID()}`
|
||||
const leader = mountRenderer(namespace)
|
||||
await vi.waitFor(() => expect(leader.runtime.isLeader()).toBe(true))
|
||||
const provider = 'openai-compatible-audio-speech'
|
||||
await useProviderConfigStore(leader.pinia).ensureProvider(provider, provider, { apiKey: 'key', baseUrl: 'https://voices.invalid/v1/' })
|
||||
await useProviderStore(leader.pinia).forceProviderConfigured(provider)
|
||||
await leader.speech.selectProviderModel(provider, 'tts-1', 'alloy')
|
||||
const follower = mountRenderer(namespace, SpeechSettings)
|
||||
await vi.waitFor(() => expect(follower.speech.activeSpeechVoiceId).toBe('alloy'))
|
||||
const view = page.elementLocator(follower.container)
|
||||
const toggle = view.getByRole('switch', { name: 'Speech', exact: true })
|
||||
const none = view.getByRole('radio', { name: 'None', exact: true })
|
||||
await expect.element(toggle).toBeChecked()
|
||||
if (control === 'switch')
|
||||
await toggle.click()
|
||||
else
|
||||
await page.elementLocator(none.element().closest('label')!).click()
|
||||
await vi.waitFor(() => expect(leader.speech.enabled).toBe(false))
|
||||
expect(leader.speech.configured).toBe(false)
|
||||
expect(leader.speech.activeSpeechProvider).toBe(provider)
|
||||
expect(leader.speech.activeSpeechModel).toBe('tts-1')
|
||||
expect(leader.speech.activeSpeechVoiceId).toBe('alloy')
|
||||
// The leader commits before the follower renders its snapshot. Wait for the
|
||||
// displayed state before clicking a radio that was previously selected.
|
||||
await expect.element(toggle).not.toBeChecked()
|
||||
await expect.element(none).toBeChecked()
|
||||
if (control === 'switch') {
|
||||
await toggle.click()
|
||||
}
|
||||
else {
|
||||
const savedProvider = follower.container.querySelector<HTMLInputElement>(`input[value="${provider}"]`)!
|
||||
await expect.element(page.elementLocator(savedProvider)).not.toBeChecked()
|
||||
await page.elementLocator(savedProvider.closest('label')!).click()
|
||||
}
|
||||
await vi.waitFor(() => expect(leader.speech.enabled).toBe(true))
|
||||
await expect.element(toggle).toBeChecked()
|
||||
expect(follower.speech.activeSpeechProvider).toBe(provider)
|
||||
expect(leader.speech.activeSpeechModel).toBe('tts-1')
|
||||
expect(leader.speech.activeSpeechVoiceId).toBe('alloy')
|
||||
})
|
||||
|
||||
// https://github.com/moeru-ai/airi/issues/3153
|
||||
// ROOT CAUSE:
|
||||
// The preview bypassed module cancellation. Disabling now aborts its request and rejects late audio.
|
||||
it('issue #3153: cancels a speech preview when another window disables speech', async () => {
|
||||
localStorage.clear()
|
||||
vi.stubGlobal('fetch', vi.fn<typeof fetch>(async () => Response.json({ voices: [], data: [] })))
|
||||
const pending = Promise.withResolvers<ArrayBuffer>()
|
||||
vi.mocked(generateSpeech).mockReturnValue(pending.promise)
|
||||
const namespace = `speech-preview-disabled:${crypto.randomUUID()}`
|
||||
const leader = mountRenderer(namespace)
|
||||
await vi.waitFor(() => expect(leader.runtime.isLeader()).toBe(true))
|
||||
const provider = 'openai-compatible-audio-speech'
|
||||
await useProviderConfigStore(leader.pinia).ensureProvider(provider, provider, { apiKey: 'key', baseUrl: 'https://voices.invalid/v1/' })
|
||||
await useProviderStore(leader.pinia).forceProviderConfigured(provider)
|
||||
await leader.speech.selectProviderModel(provider, 'tts-1', 'alloy')
|
||||
const follower = mountRenderer(namespace, SpeechSettings)
|
||||
const preview = () => Array.from(follower.container.querySelectorAll('button')).find(button => button.textContent?.includes('Test Voice'))!
|
||||
await vi.waitFor(() => expect(preview()?.disabled).toBe(false))
|
||||
preview().click()
|
||||
await vi.waitFor(() => expect(generateSpeech).toHaveBeenCalledTimes(1))
|
||||
const signal = vi.mocked(generateSpeech).mock.calls[0][0].abortSignal
|
||||
await leader.speech.selectSource('speech-noop')
|
||||
await vi.waitFor(() => expect(signal?.aborted).toBe(true))
|
||||
pending.resolve(new ArrayBuffer(0))
|
||||
await vi.waitFor(() => expect(preview()?.disabled).toBe(true))
|
||||
expect(follower.container.querySelector('audio')).toBeNull()
|
||||
})
|
||||
|
||||
// https://github.com/moeru-ai/airi/actions/runs/34348745853/job/102456521103
|
||||
// ROOT CAUSE: Page initialization and provider watchers awaited model RPCs
|
||||
// without handling transport disposal. Passing assertions hid a teardown rejection.
|
||||
@@ -136,7 +214,7 @@ it('reports the committed provider and model after a settings-page click', async
|
||||
tts_model_id: 'v1',
|
||||
source: 'settings',
|
||||
}))
|
||||
expect(follower.speech.activeSpeechProvider).toBe('microsoft-speech')
|
||||
await vi.waitFor(() => expect(follower.speech.activeSpeechProvider).toBe('microsoft-speech'))
|
||||
})
|
||||
|
||||
// https://github.com/moeru-ai/airi/pull/2490#discussion_r3967708960
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import type { SpeechOutput, TurnRef } from '@proj-airi/core-agent'
|
||||
import type { SpeechProviderWithExtraOptions } from '@xsai-ext/providers/utils'
|
||||
import type { SpeechProvider, SpeechProviderWithExtraOptions } from '@xsai-ext/providers/utils'
|
||||
import type {} from 'pinia-plugin-synced'
|
||||
|
||||
import type { AiriExtension } from '../../types/airiCard'
|
||||
@@ -148,6 +148,16 @@ export const useSpeechStore = defineStore('speech', () => {
|
||||
|
||||
// State
|
||||
const activeSpeechProvider = useLocalStorageManualReset<string>('settings/speech/active-provider', 'speech-noop', persistenceOptions)
|
||||
const enabled = useLocalStorageManualReset('settings/speech/enabled', true, persistenceOptions)
|
||||
// Disabling invalidates synthesis captured by an open response, including requests that have not started yet.
|
||||
let synthesis = new AbortController()
|
||||
watch(enabled, (value) => {
|
||||
if (value)
|
||||
synthesis = new AbortController()
|
||||
else
|
||||
synthesis.abort(new DOMException('Speech is disabled', 'AbortError'))
|
||||
}, { immediate: true, flush: 'sync' })
|
||||
onScopeDispose(() => synthesis.abort())
|
||||
const activeSpeechModel = useLocalStorageManualReset<string>('settings/speech/active-model', '', persistenceOptions)
|
||||
const activeSpeechVoiceId = useLocalStorageManualReset<string>('settings/speech/voice', '', persistenceOptions)
|
||||
const activeSpeechVoice = refManualReset<VoiceInfo | undefined>(undefined)
|
||||
@@ -469,6 +479,17 @@ export const useSpeechStore = defineStore('speech', () => {
|
||||
clearVoiceSelection()
|
||||
}
|
||||
|
||||
/** Selects a source in the leader. None disables speech, and selecting the saved source restores its model and voice. */
|
||||
async function selectSource(provider: string) {
|
||||
enabled.value = provider !== 'speech-noop'
|
||||
if (!enabled.value)
|
||||
return
|
||||
const restoresStreamingSource = provider === OFFICIAL_SPEECH_PROVIDER_ID && activeSpeechProvider.value === OFFICIAL_SPEECH_STREAMING_PROVIDER_ID
|
||||
if (activeSpeechProvider.value !== provider && !restoresStreamingSource)
|
||||
await selectProviderModel(provider, '')
|
||||
return { provider: activeSpeechProvider.value, model: activeSpeechModel.value, voice_id: activeSpeechVoiceId.value }
|
||||
}
|
||||
|
||||
/** Commits an explicit selection in the leader before watchers request its catalog. An omitted voice preserves an unchanged selection. */
|
||||
async function selectProviderModel(provider: string, model: string, voiceId?: string) {
|
||||
if (disposed)
|
||||
@@ -581,17 +602,19 @@ export const useSpeechStore = defineStore('speech', () => {
|
||||
* @returns ArrayBuffer containing the audio data
|
||||
*/
|
||||
async function speech(
|
||||
provider: SpeechProviderWithExtraOptions<string, any>,
|
||||
provider: SpeechProvider | SpeechProviderWithExtraOptions<string, Record<string, unknown>>,
|
||||
model: string,
|
||||
input: string,
|
||||
voice: string,
|
||||
providerConfig: Record<string, any> = {},
|
||||
providerConfig: Record<string, unknown> = {},
|
||||
analytics: SpeechAnalytics = {
|
||||
trigger: 'manual',
|
||||
source: 'manual_preview',
|
||||
voice_type: resolveVoiceType(voice),
|
||||
},
|
||||
): Promise<ArrayBuffer> {
|
||||
const signal = synthesis.signal
|
||||
signal.throwIfAborted()
|
||||
const requestProviderConfig = activeSpeechProvider.value === OFFICIAL_SPEECH_PROVIDER_ID
|
||||
|| activeSpeechProvider.value === OFFICIAL_SPEECH_STREAMING_PROVIDER_ID
|
||||
? withAiriTtsAnalytics(providerConfig, analytics)
|
||||
@@ -600,13 +623,18 @@ export const useSpeechStore = defineStore('speech', () => {
|
||||
...provider.speech(model, requestProviderConfig),
|
||||
input,
|
||||
voice,
|
||||
abortSignal: signal,
|
||||
})
|
||||
|
||||
signal.throwIfAborted()
|
||||
return response
|
||||
}
|
||||
|
||||
/** Captures one response's provider, voice, model, and settings before any asynchronous work. */
|
||||
function createOutput(turn: TurnRef, selection: AiriExtension['modules']['speech'], output: Pick<SpeechOutput, 'playback' | 'onPlaybackStart' | 'onPlaybackEnd' | 'onSpecial'>, audioContext: AudioContext): SpeechOutput {
|
||||
if (!enabled.value)
|
||||
return { ...output, synthesize: async () => null }
|
||||
const synthesisSignal = synthesis.signal
|
||||
const providerId = selection.provider
|
||||
const config = structuredClone(toRaw(providerStore.getProviderConfig(providerId) ?? {}))
|
||||
const selectedVoice = availableVoices.value[providerId]?.find(voice => voice.id === selection.voice_id)
|
||||
@@ -633,12 +661,13 @@ export const useSpeechStore = defineStore('speech', () => {
|
||||
ttsVoiceType: voiceType,
|
||||
bufferEntireSession: resource.startsWith('seed-tts-2.0') || resource.startsWith('seed-icl-2.0'),
|
||||
extraBody: { api_resource_id: resource, audio: { sample_rate: 24000, bit_rate: 64000 } },
|
||||
}, text, signal) }
|
||||
}, text, AbortSignal.any([signal, synthesisSignal])) }
|
||||
}
|
||||
if (!model || !voiceId)
|
||||
throw new Error('Speech requires a model and voice')
|
||||
const provider = providersStore.getProviderInstance<SpeechProviderWithExtraOptions<string, Record<string, unknown>>>(providerId)
|
||||
return { ...output, language, synthesize: async (request, signal) => {
|
||||
signal = AbortSignal.any([signal, synthesisSignal])
|
||||
signal.throwIfAborted()
|
||||
const client = await provider
|
||||
signal.throwIfAborted()
|
||||
@@ -647,6 +676,7 @@ export const useSpeechStore = defineStore('speech', () => {
|
||||
? { ...config, extraBody: { ...(config.extraBody as Record<string, unknown> | undefined), airi_analytics: { trigger: 'auto', source: 'chat_auto_tts', voice_type: voiceType, turn_id: turn.turnId } } }
|
||||
: config
|
||||
const audio = await generateSpeech({ ...client.speech(model, settings), input, voice: voiceId, abortSignal: signal })
|
||||
signal.throwIfAborted()
|
||||
return new Blob([audio])
|
||||
} }
|
||||
}
|
||||
@@ -727,6 +757,8 @@ export const useSpeechStore = defineStore('speech', () => {
|
||||
}
|
||||
|
||||
const configured = computed(() => {
|
||||
if (!enabled.value)
|
||||
return false
|
||||
if (activeSpeechProvider.value === 'speech-noop')
|
||||
return false
|
||||
|
||||
@@ -754,6 +786,7 @@ export const useSpeechStore = defineStore('speech', () => {
|
||||
|
||||
/** Resets shared settings in the leader and rejects catalog results started before this reset. */
|
||||
async function resetSettings() {
|
||||
enabled.reset()
|
||||
// Invalidate request ownership before the reset publishes new settings.
|
||||
cancelCatalogRequests()
|
||||
activeSpeechProvider.reset()
|
||||
@@ -774,6 +807,7 @@ export const useSpeechStore = defineStore('speech', () => {
|
||||
return {
|
||||
// State
|
||||
configured,
|
||||
enabled,
|
||||
activeSpeechProvider,
|
||||
activeSpeechModel,
|
||||
activeSpeechVoice,
|
||||
@@ -803,6 +837,7 @@ export const useSpeechStore = defineStore('speech', () => {
|
||||
loadVoiceCatalog,
|
||||
invalidateVoiceCatalogs,
|
||||
selectProviderModel,
|
||||
selectSource,
|
||||
ensureActiveSpeechVoice,
|
||||
getVoicesForProvider,
|
||||
ensureStreamingDefaultModel,
|
||||
@@ -815,7 +850,7 @@ export const useSpeechStore = defineStore('speech', () => {
|
||||
}
|
||||
}, {
|
||||
synced: {
|
||||
actions: ['loadVoiceCatalog', 'invalidateVoiceCatalogs', 'selectProviderModel', 'ensureActiveSpeechVoice', 'resetSettings'],
|
||||
actions: ['loadVoiceCatalog', 'invalidateVoiceCatalogs', 'selectProviderModel', 'selectSource', 'ensureActiveSpeechVoice', 'resetSettings'],
|
||||
state: true,
|
||||
},
|
||||
})
|
||||
|
||||
@@ -59,7 +59,7 @@ export const useVisionOrchestratorStore = defineStore('vision-orchestrator', ()
|
||||
const lastWorkloadId = ref<VisionWorkloadId>('screen:interpret')
|
||||
|
||||
async function processCapture(payload: VisionCapturePayload) {
|
||||
if (!activeProvider.value || !activeModel.value) {
|
||||
if (!visionStore.configured) {
|
||||
const configurationError = new Error('Vision model is not configured')
|
||||
recordError(configurationError)
|
||||
throw configurationError
|
||||
|
||||
@@ -13,6 +13,7 @@ export const useVisionStore = defineStore('vision', () => {
|
||||
// Pinia synchronization owns live cross-window state. localStorage only
|
||||
// loads and saves durable values for this synchronized store.
|
||||
const persistenceOptions = { listenToStorageChanges: false }
|
||||
const enabled = useLocalStorageManualReset('settings/vision/enabled', true, persistenceOptions)
|
||||
|
||||
const activeProvider = useLocalStorageManualReset('settings/vision/active-provider', '', persistenceOptions)
|
||||
const activeModel = useLocalStorageManualReset('settings/vision/active-model', '', persistenceOptions)
|
||||
@@ -49,7 +50,7 @@ export const useVisionStore = defineStore('vision', () => {
|
||||
})
|
||||
|
||||
const configured = computed(() => {
|
||||
return !!activeProvider.value && !!activeModel.value
|
||||
return enabled.value && !!activeProvider.value && !!activeModel.value
|
||||
})
|
||||
|
||||
function resetModelSelection() {
|
||||
@@ -73,6 +74,7 @@ export const useVisionStore = defineStore('vision', () => {
|
||||
}
|
||||
|
||||
function resetState() {
|
||||
enabled.reset()
|
||||
useForChat.reset()
|
||||
useForToolImages.reset()
|
||||
activeProvider.reset()
|
||||
@@ -80,6 +82,7 @@ export const useVisionStore = defineStore('vision', () => {
|
||||
}
|
||||
|
||||
return {
|
||||
enabled,
|
||||
useForChat,
|
||||
useForToolImages,
|
||||
activeProvider,
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
import type { SpeechOutput } from '@proj-airi/core-agent'
|
||||
|
||||
import { defineInvoke } from '@moeru/eventa'
|
||||
import { Playback } from '@proj-airi/pipelines-audio'
|
||||
import { createPinia, disposePinia } from 'pinia'
|
||||
import { afterEach, expect, it, vi } from 'vitest'
|
||||
import { createApp } from 'vue'
|
||||
import { createI18n } from 'vue-i18n'
|
||||
|
||||
import { getSpeechBusContext, voiceSpeechCommand } from '../services/speech/bus'
|
||||
import { useLlmStreamingControlStore } from './ai/chat-llm/streaming-control'
|
||||
import { useChatStore } from './chat'
|
||||
import { useSpeechStore } from './modules/speech'
|
||||
import { useVoiceStore } from './voice'
|
||||
|
||||
const cleanups: (() => void)[] = []
|
||||
|
||||
afterEach(() => {
|
||||
cleanups.splice(0).reverse().forEach(cleanup => cleanup())
|
||||
vi.restoreAllMocks()
|
||||
localStorage.clear()
|
||||
})
|
||||
|
||||
// https://github.com/moeru-ai/airi/issues/3153
|
||||
// ROOT CAUSE:
|
||||
// Cancelling a response also cancels chat generation and tool handlers. Disabling speech now cancels only its producers and playback.
|
||||
it.each(['local', 'remote'] as const)('issue #3153: disabling %s speech preserves the chat turn and its tool handlers', async (source) => {
|
||||
const pinia = createPinia()
|
||||
let voice!: ReturnType<typeof useVoiceStore>
|
||||
const app = createApp({ setup() {
|
||||
voice = useVoiceStore()
|
||||
return () => null
|
||||
} }).use(pinia).use(createI18n({ legacy: false, locale: 'en', messages: { en: {} }, missingWarn: false, fallbackWarn: false }))
|
||||
const container = document.createElement('div')
|
||||
app.mount(container)
|
||||
cleanups.push(() => {
|
||||
app.unmount()
|
||||
disposePinia(pinia)
|
||||
})
|
||||
|
||||
const settings = useSpeechStore(pinia)
|
||||
settings.enabled = true
|
||||
const cancelChat = vi.spyOn(useChatStore(pinia), 'cancelTurn')
|
||||
const control = useLlmStreamingControlStore(pinia)
|
||||
const turn = { sessionId: 'alice', turnId: `speech-disabled-${source}` }
|
||||
const generation = control.beginTurn({ turnId: turn.turnId })
|
||||
const tool = vi.fn()
|
||||
generation.on({ name: 'test.continue', prompt: 'Continue the task.' }, tool)
|
||||
|
||||
const stopped = vi.fn(async () => ({ throughMs: 10 }))
|
||||
const played = vi.fn(() => ({ done: new Promise<{ throughMs: number }>(() => {}), stop: stopped }))
|
||||
let synthesisSignal: AbortSignal | undefined
|
||||
const output: SpeechOutput = {
|
||||
playback: new Playback({ nowMs: () => 0, play: played }),
|
||||
stream: async (_text, signal) => {
|
||||
synthesisSignal = signal
|
||||
return new ReadableStream({ start(controller) {
|
||||
controller.enqueue({ audio: new Blob(['speech']), text: 'Hello.' })
|
||||
} })
|
||||
},
|
||||
}
|
||||
cleanups.push(voice.connectOutput(() => output))
|
||||
const command = defineInvoke(getSpeechBusContext(), voiceSpeechCommand)
|
||||
const producer = { originId: 'chat-window', producerId: 'answer', turn }
|
||||
if (source === 'local') {
|
||||
voice.startResponse(turn)
|
||||
await voice.getSpeech(turn)!.write('Hello.')
|
||||
}
|
||||
else {
|
||||
await command({ ...producer, type: 'open', purpose: 'answer' })
|
||||
await command({ ...producer, type: 'text', value: 'Hello.' })
|
||||
}
|
||||
await expect.poll(() => played.mock.calls.length).toBe(1)
|
||||
|
||||
settings.enabled = false
|
||||
|
||||
expect(cancelChat).not.toHaveBeenCalled()
|
||||
expect(voice.activeTurns).toEqual([turn])
|
||||
await expect.poll(() => synthesisSignal?.aborted).toBe(true)
|
||||
await expect.poll(() => stopped.mock.calls.length).toBe(1)
|
||||
await control.dispatchWith('<|CALL ["test.continue"]|>', { turnId: turn.turnId })
|
||||
expect(tool).toHaveBeenCalledOnce()
|
||||
if (source === 'local')
|
||||
expect(await voice.getSpeech(turn)!.write('Later text.')).toEqual({ status: 'closed' })
|
||||
else
|
||||
expect(await command({ ...producer, type: 'text', value: 'Later text.' })).toEqual({ status: 'closed' })
|
||||
expect(await voice.finishResponse(turn)).toBe('finished')
|
||||
expect(voice.activeTurns).toEqual([])
|
||||
expect(cancelChat).not.toHaveBeenCalled()
|
||||
await expect(generation.done).resolves.toEqual({ type: 'completed' })
|
||||
})
|
||||
@@ -29,6 +29,7 @@ import { useAudioContext, useSpeakingStore } from './audio'
|
||||
import { useChatStore } from './chat'
|
||||
import { useChatSessionStore } from './chat/session-store'
|
||||
import { startsInputOnSpeech, useHearingStore } from './modules/hearing'
|
||||
import { useSpeechStore } from './modules/speech'
|
||||
import { useSettingsAudioDevice } from './settings/audio-device'
|
||||
import { useSpeakerTrackerStore } from './speaker-tracker'
|
||||
import { useVoiceCaptureLeaseStore } from './voice-capture-lease'
|
||||
@@ -52,6 +53,7 @@ const VOICE_END_SILENCE = { pauseMs: 1200, graceMs: 3000 } as const
|
||||
export const useVoiceStore = defineStore('voice', () => {
|
||||
const devices = useSettingsAudioDevice()
|
||||
const hearing = useHearingStore()
|
||||
const speechSettings = useSpeechStore()
|
||||
const chat = useChatStore()
|
||||
const sessions = useChatSessionStore()
|
||||
const voiceMessages = useVoiceMessagesStore()
|
||||
@@ -165,6 +167,16 @@ export const useVoiceStore = defineStore('voice', () => {
|
||||
watch([state, transcript, frontDraftId, error, microphoneStream, microphoneEnabled, microphoneError, () => wakeWordDetection.preparation, () => wakeWordDetection.error], publishSnapshot)
|
||||
watch(drafts, publishSnapshot, { deep: true })
|
||||
|
||||
// The host cancels dictation as well as continuous input. Remote snapshots trigger only local runtime cleanup.
|
||||
watch(() => hearing.enabled, (enabled) => {
|
||||
error.value = undefined
|
||||
if (enabled)
|
||||
return
|
||||
void stopListening()
|
||||
presentedInput = undefined
|
||||
publishSnapshot()
|
||||
}, { flush: 'sync' })
|
||||
|
||||
function beginInput(options: BeginSpeechInput) {
|
||||
error.value = undefined
|
||||
return controller.beginInput(options)
|
||||
@@ -172,7 +184,7 @@ export const useVoiceStore = defineStore('voice', () => {
|
||||
|
||||
/** Begins an input that a control ends. Returns nothing while another window holds a capture lease. */
|
||||
function beginManual(sessionId: string) {
|
||||
if (captureLease.held)
|
||||
if (!hearing.enabled || captureLease.held)
|
||||
return undefined
|
||||
return beginInput({ sessionId, interruptTurns: activeTurns.value.filter(turn => turn.sessionId === sessionId), start: { kind: 'after-silence' }, cause: 'control' })
|
||||
}
|
||||
@@ -316,6 +328,17 @@ export const useVoiceStore = defineStore('voice', () => {
|
||||
const producers = new Map<string, SpeechStream>()
|
||||
const opened = new Set<string>()
|
||||
const stops = [
|
||||
// Disabling speech stops local and remote producers. The response stays open for text generation and tools.
|
||||
watch(() => speechSettings.enabled, (enabled) => {
|
||||
if (enabled)
|
||||
return
|
||||
for (const { response, speech } of responses.values()) {
|
||||
void response.playback.stop({ fadeMs: 0 }).catch(report)
|
||||
speech?.cancel('Speech is disabled')
|
||||
}
|
||||
for (const speech of producers.values())
|
||||
speech.cancel('Speech is disabled')
|
||||
}, { flush: 'sync' }),
|
||||
voiceMessages.connect(),
|
||||
captureLease.connect(),
|
||||
// Automatic inputs stop at the `enabled` check of the voice activity plugin. Manual inputs do not, so end them here.
|
||||
|
||||
Reference in New Issue
Block a user