mirror of
https://github.com/moeru-ai/airi.git
synced 2026-10-30 19:34:10 +08:00
feat(stage-ui,stage-tamagotchi,stage-layouts): add a microphone switch that closes the microphone for every feature (#3210)
This commit is contained in:
@@ -24,7 +24,7 @@ const devices = useSettingsAudioDevice()
|
||||
const { accelerator, registrationError } = usePushToTalkShortcutSettings()
|
||||
const listShortcuts = useElectronEventaInvoke(electronShortcutList)
|
||||
const recording = shallowRef(false)
|
||||
const active = computed(() => devices.enabled && hearing.inputMode === 'push-to-talk')
|
||||
const active = computed(() => devices.listening && hearing.inputMode === 'push-to-talk')
|
||||
|
||||
const shortcutLabel = computed(() => recording.value
|
||||
? tt('actions.recording')
|
||||
|
||||
+20
-2
@@ -80,7 +80,8 @@ afterEach(() => {
|
||||
})
|
||||
|
||||
describe('controls Island microphone', () => {
|
||||
it('turns the microphone on after permission and off again without opening Hearing controls', async () => {
|
||||
// The Island button must close the microphone for every feature, not only stop listening.
|
||||
it('starts listening after permission, then turns the microphone off without opening Hearing controls', async () => {
|
||||
const { screen, device, askPermission } = mountControlsIsland()
|
||||
askPermission.mockResolvedValue(true)
|
||||
const microphone = screen.getByTestId('controls-island-microphone')
|
||||
@@ -91,16 +92,33 @@ describe('controls Island microphone', () => {
|
||||
await expect.element(microphone).toHaveAttribute('aria-pressed', 'true')
|
||||
await expect.element(microphone).toHaveAccessibleName('Turn off microphone')
|
||||
expect(device.enabled).toBe(true)
|
||||
expect(device.off).toBe(false)
|
||||
expect(askPermission).toHaveBeenCalledOnce()
|
||||
expect(screen.getByTestId('hearing-config-dialog').query()).toBeNull()
|
||||
|
||||
await microphone.click()
|
||||
|
||||
await expect.element(microphone).toHaveAttribute('aria-pressed', 'false')
|
||||
expect(device.enabled).toBe(false)
|
||||
expect(device.off).toBe(true)
|
||||
expect(device.enabled).toBe(true)
|
||||
expect(askPermission).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('turns the microphone on again and listens without a second permission request', async () => {
|
||||
localStorage.setItem('settings/audio/input/enabled', 'true')
|
||||
localStorage.setItem('settings/audio/input/off', 'true')
|
||||
const { screen, device, askPermission } = mountControlsIsland()
|
||||
const microphone = screen.getByTestId('controls-island-microphone')
|
||||
await expect.element(microphone).toHaveAttribute('aria-pressed', 'false')
|
||||
|
||||
await microphone.click()
|
||||
|
||||
await expect.element(microphone).toHaveAttribute('aria-pressed', 'true')
|
||||
expect(device.off).toBe(false)
|
||||
expect(device.listening).toBe(true)
|
||||
expect(askPermission).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('ignores presses while the permission prompt is pending', async () => {
|
||||
const { screen, device, askPermission } = mountControlsIsland()
|
||||
const permission = Promise.withResolvers<boolean>()
|
||||
|
||||
+16
-8
@@ -73,7 +73,7 @@ const { dock, isLeft, isTop, motionPhase } = placement
|
||||
const settingsAudioDeviceStore = useSettingsAudioDevice()
|
||||
const settingsStore = useSettings()
|
||||
const context = useElectronEventaContext()
|
||||
const { enabled } = storeToRefs(settingsAudioDeviceStore)
|
||||
const { enabled, listening, off: microphoneOff } = storeToRefs(settingsAudioDeviceStore)
|
||||
const { alwaysOnTop, controlsIslandIconSize } = storeToRefs(settingsStore)
|
||||
const openSettings = useElectronEventaInvoke(electronOpenSettings)
|
||||
const isLinux = useElectronEventaInvoke(electron.app.isLinux)
|
||||
@@ -141,22 +141,30 @@ function setOverlay(key: string, active: boolean) {
|
||||
|
||||
/** True while the microphone button waits for the permission prompt. The button is disabled, so a second press cannot start another request. */
|
||||
const microphonePending = ref(false)
|
||||
const microphoneLabel = computed(() => enabled.value
|
||||
const microphoneLabel = computed(() => listening.value
|
||||
? t('tamagotchi.stage.controls-island.disable-microphone')
|
||||
: t('tamagotchi.stage.controls-island.enable-microphone'))
|
||||
|
||||
/**
|
||||
* Turns the microphone preference on or off. The Hearing input mode still decides when the microphone captures.
|
||||
* Only this press asks for permission. A refusal opens Hearing controls, which show the reason from the device store.
|
||||
* Turns listening on, or turns the microphone off. The button is on while AIRI listens.
|
||||
*
|
||||
* Off is the microphone switch, so no feature can record until the microphone is on again. The listening preference
|
||||
* stays, so the next press listens again without a permission request.
|
||||
* On turns the microphone on. Only a first listening press asks for permission. A refusal opens Hearing controls,
|
||||
* which show the reason from the device store. The Hearing input mode still decides when the microphone captures.
|
||||
*/
|
||||
async function toggleMicrophone() {
|
||||
if (microphonePending.value)
|
||||
return
|
||||
if (enabled.value) {
|
||||
enabled.value = false
|
||||
if (listening.value) {
|
||||
microphoneOff.value = true
|
||||
return
|
||||
}
|
||||
|
||||
microphoneOff.value = false
|
||||
if (enabled.value)
|
||||
return
|
||||
|
||||
microphonePending.value = true
|
||||
try {
|
||||
if (await settingsAudioDeviceStore.askPermission()) {
|
||||
@@ -642,13 +650,13 @@ function resetMainWindowPosition() {
|
||||
data-testid="controls-island-microphone"
|
||||
:button-style="adjustStyleClasses.button"
|
||||
:aria-label="microphoneLabel"
|
||||
:aria-pressed="enabled"
|
||||
:aria-pressed="listening"
|
||||
:aria-busy="microphonePending"
|
||||
:disabled="microphonePending"
|
||||
@click="toggleMicrophone"
|
||||
>
|
||||
<Transition name="fade" mode="out-in">
|
||||
<IndicatorMicVolume v-if="enabled" :class="adjustStyleClasses.icon" />
|
||||
<IndicatorMicVolume v-if="listening" :class="adjustStyleClasses.icon" />
|
||||
<div v-else :class="['i-ph:microphone-slash text-neutral-800 dark:text-neutral-300', adjustStyleClasses.icon]" />
|
||||
</Transition>
|
||||
</ControlButton>
|
||||
|
||||
+3
-3
@@ -7,7 +7,7 @@ import { computed, onMounted, onUnmounted, watch } from 'vue'
|
||||
|
||||
const props = withDefaults(defineProps<{ colorClass?: string }>(), { colorClass: 'text-primary-500 dark:text-primary-200' })
|
||||
const settingsAudio = useSettingsAudioDevice()
|
||||
const { stream, enabled } = storeToRefs(settingsAudio)
|
||||
const { stream, listening } = storeToRefs(settingsAudio)
|
||||
|
||||
const { audioContext } = storeToRefs(useAudioContext())
|
||||
const { startAnalyzer, stopAnalyzer, volumeLevel } = useAudioAnalyzer()
|
||||
@@ -30,7 +30,7 @@ function teardown() {
|
||||
|
||||
async function setup() {
|
||||
teardown()
|
||||
if (!enabled.value || !stream.value)
|
||||
if (!listening.value || !stream.value)
|
||||
return
|
||||
|
||||
const ctx = audioContext.value
|
||||
@@ -45,7 +45,7 @@ async function setup() {
|
||||
}
|
||||
|
||||
onMounted(() => {
|
||||
watch([enabled, stream], () => setup(), { immediate: true })
|
||||
watch([listening, stream], () => setup(), { immediate: true })
|
||||
})
|
||||
|
||||
onUnmounted(() => teardown())
|
||||
|
||||
@@ -940,9 +940,13 @@ pages:
|
||||
description: Select the suitable speech recognition provider
|
||||
microphone:
|
||||
title: Microphone
|
||||
label: Microphone
|
||||
description: Turn on the microphone for Hearing. The input mode controls when AIRI listens.
|
||||
error: The microphone is not available
|
||||
microphone-switch:
|
||||
label: Use microphone
|
||||
description: When this is off, no feature in AIRI can use the microphone.
|
||||
listen:
|
||||
label: Listen
|
||||
description: AIRI listens in the input mode above. This needs the microphone.
|
||||
input-device:
|
||||
label: Input device
|
||||
description: Select the microphone that AIRI uses.
|
||||
|
||||
@@ -60,6 +60,10 @@ chat:
|
||||
push-to-talk-unavailable: This device has no Push to Talk key, so AIRI does not listen. Change the input mode in Hearing settings.
|
||||
wake-word: Listen for wake words
|
||||
hearing-settings: More Hearing settings…
|
||||
use-microphone: Use microphone
|
||||
microphone-off: The microphone is off
|
||||
microphone-off-description: Turn on the microphone to record.
|
||||
turn-on-microphone: Turn on
|
||||
options-hint: Hover or right-click for options
|
||||
recording: Recording
|
||||
stop: Stop recording
|
||||
|
||||
@@ -891,9 +891,13 @@ pages:
|
||||
description: 选择合适的语音转文本的服务来源
|
||||
microphone:
|
||||
title: 麦克风
|
||||
label: 麦克风
|
||||
description: 为听觉打开麦克风。输入模式决定 AIRI 何时聆听。
|
||||
error: 麦克风不可用
|
||||
microphone-switch:
|
||||
label: 启用麦克风
|
||||
description: 关闭后,AIRI 的所有功能都不能使用麦克风。
|
||||
listen:
|
||||
label: 聆听
|
||||
description: AIRI 按上面的输入模式聆听。需要启用麦克风。
|
||||
input-device:
|
||||
label: 输入设备
|
||||
description: 选择 AIRI 使用的麦克风。
|
||||
|
||||
@@ -60,6 +60,10 @@ chat:
|
||||
push-to-talk-unavailable: 当前设备没有「按住说话」按键,所以 AIRI 不会聆听。请在听觉设置中更改输入模式。
|
||||
wake-word: 监听唤醒词
|
||||
hearing-settings: 更多听觉设置…
|
||||
use-microphone: 启用麦克风
|
||||
microphone-off: 麦克风已关闭
|
||||
microphone-off-description: 打开麦克风后才能录音。
|
||||
turn-on-microphone: 打开
|
||||
options-hint: 悬停或右键查看选项
|
||||
recording: 正在录音
|
||||
stop: 停止录音
|
||||
|
||||
@@ -166,12 +166,7 @@ const viewControlsEnabled = computed(() => {
|
||||
return false
|
||||
})
|
||||
const settingsAudioDevice = useSettingsAudioDevice()
|
||||
const { enabled, stream } = storeToRefs(settingsAudioDevice)
|
||||
|
||||
watch(hearingOpen, async (open) => {
|
||||
if (open)
|
||||
await settingsAudioDevice.askPermission()
|
||||
})
|
||||
const { listening, stream } = storeToRefs(settingsAudioDevice)
|
||||
|
||||
function restoreComposerFocus(event: Event) {
|
||||
event.preventDefault()
|
||||
@@ -282,7 +277,7 @@ function teardownAnalyzer() {
|
||||
|
||||
async function setupAnalyzer() {
|
||||
teardownAnalyzer()
|
||||
if (!enabled.value || !stream.value)
|
||||
if (!listening.value || !stream.value)
|
||||
return
|
||||
if (audioContext.state === 'suspended')
|
||||
await audioContext.resume()
|
||||
@@ -293,7 +288,7 @@ async function setupAnalyzer() {
|
||||
analyzerSource.connect(analyser)
|
||||
}
|
||||
|
||||
watch([enabled, stream], () => {
|
||||
watch([listening, stream], () => {
|
||||
setupAnalyzer()
|
||||
}, { immediate: true })
|
||||
|
||||
@@ -449,7 +444,7 @@ onUnmounted(() => {
|
||||
>
|
||||
<span
|
||||
aria-hidden="true"
|
||||
:class="[enabled ? 'i-solar:soundwave-bold' : 'i-solar:soundwave-linear', 'size-5']"
|
||||
:class="[listening ? 'i-solar:soundwave-bold' : 'i-solar:soundwave-linear', 'size-5']"
|
||||
/>
|
||||
</BasicButton>
|
||||
<input ref="imageInput" type="file" accept="image/png,image/jpeg,image/webp,image/gif" multiple :class="['hidden']" @change="selectFiles">
|
||||
|
||||
+78
-3
@@ -7,11 +7,12 @@ import { afterEach, describe, expect, it } from 'vitest'
|
||||
import { render } from 'vitest-browser-vue'
|
||||
import { defineComponent, h, ref, shallowRef } from 'vue'
|
||||
import { createI18n } from 'vue-i18n'
|
||||
import { Toaster } from 'vue-sonner'
|
||||
|
||||
import VoiceInputButton from './voice-input-button.vue'
|
||||
|
||||
import { pushToTalkTriggerKey } from '../../../../composables/push-to-talk-trigger'
|
||||
import { getSpeechBusContext, voiceInputCommand, voiceMessageCommand, voiceMessagesChanged, voiceRequestSnapshot, voiceSnapshotChanged } from '../../../../services/speech/bus'
|
||||
import { getSpeechBusContext, voiceInputCommand, voiceLevelMonitor, voiceMessageCommand, voiceMessagesChanged, voiceRequestSnapshot, voiceSnapshotChanged } from '../../../../services/speech/bus'
|
||||
|
||||
const cleanups: (() => void)[] = []
|
||||
|
||||
@@ -55,8 +56,9 @@ async function mountButton(options: { afterFinish?: Partial<VoiceMessageSnapshot
|
||||
const attachment = document.body.appendChild(document.createElement('div'))
|
||||
cleanups.push(() => status.remove(), () => attachment.remove())
|
||||
const button = shallowRef<{ sendPending: (text: string) => Promise<'none' | 'sent' | 'failed'> }>()
|
||||
// The Toaster shows the toasts that the button raises, as the app root does.
|
||||
const component = defineComponent({ setup() {
|
||||
return () => h(VoiceInputButton, {
|
||||
return () => [h(Toaster), h(VoiceInputButton, {
|
||||
'modelValue': draft.value,
|
||||
'onUpdate:modelValue': (value: string) => draft.value = value,
|
||||
'ref': button,
|
||||
@@ -64,7 +66,7 @@ async function mountButton(options: { afterFinish?: Partial<VoiceMessageSnapshot
|
||||
'attachmentElement': attachment,
|
||||
'sessionId': 'alice',
|
||||
'onSubmit': () => submitted.push(true),
|
||||
})
|
||||
})]
|
||||
} })
|
||||
const screen = await render(component, {
|
||||
global: {
|
||||
@@ -88,6 +90,79 @@ describe('voiceInputButton', () => {
|
||||
await expect.element(screen.getByTestId('voice-menu-push-to-talk-stranded')).toHaveTextContent('stage.chat.voice-composer.push-to-talk-unavailable')
|
||||
})
|
||||
|
||||
// A hover opens the menu. Users saw the system microphone indicator turn on while microphone input was off,
|
||||
// because the menu asked the host for a level, and the host opened the microphone for it.
|
||||
it('asks the host for no level while microphone input is off', async () => {
|
||||
const monitorRequests: unknown[] = []
|
||||
cleanups.push(getSpeechBusContext().on(voiceLevelMonitor, ({ body }) => monitorRequests.push(body)))
|
||||
const { screen } = await mountButton()
|
||||
|
||||
await screen.getByTestId('voice-input-button').click({ button: 'right' })
|
||||
await expect.element(screen.getByTestId('voice-menu-device')).toBeVisible()
|
||||
await new Promise(resolve => setTimeout(resolve, 100))
|
||||
|
||||
expect(monitorRequests).toEqual([])
|
||||
expect(screen.getByTestId('voice-menu-level').query()).toBeNull()
|
||||
})
|
||||
|
||||
it('shows the level while microphone input holds the microphone open', async () => {
|
||||
localStorage.setItem('settings/audio/input/enabled', 'true')
|
||||
const monitorRequests: unknown[] = []
|
||||
cleanups.push(getSpeechBusContext().on(voiceLevelMonitor, ({ body }) => monitorRequests.push(body)))
|
||||
const { screen } = await mountButton()
|
||||
|
||||
await screen.getByTestId('voice-input-button').click({ button: 'right' })
|
||||
|
||||
await expect.element(screen.getByTestId('voice-menu-level')).toBeInTheDocument()
|
||||
await expect.poll(() => monitorRequests.length).toBeGreaterThan(0)
|
||||
})
|
||||
|
||||
it('starts no recording while the microphone is off', async () => {
|
||||
localStorage.setItem('settings/audio/input/off', 'true')
|
||||
const { screen, commands } = await mountButton()
|
||||
const button = screen.getByTestId('voice-input-button')
|
||||
await expect.element(button).toHaveAccessibleName('stage.chat.voice-composer.microphone-off')
|
||||
|
||||
await button.click()
|
||||
|
||||
await expect.element(screen.getByText('stage.chat.voice-composer.microphone-off-description')).toBeVisible()
|
||||
expect(commands).toEqual([])
|
||||
await expect.element(button).toHaveAttribute('aria-pressed', 'false')
|
||||
})
|
||||
|
||||
// The microphone switch and a capture lease finish a recording on the host. The control stayed on its spinner.
|
||||
it('frees the control and keeps the recording when the host finishes it', async () => {
|
||||
const { screen, commands } = await mountButton()
|
||||
const button = screen.getByTestId('voice-input-button')
|
||||
await button.click()
|
||||
await expect.element(screen.getByTestId('voice-status-bar')).toBeInTheDocument()
|
||||
const record = commands[0] as Extract<VoiceMessageCommand, { type: 'record' }>
|
||||
|
||||
getSpeechBusContext().emit(voiceMessagesChanged, [{ id: record.id, sessionId: 'alice', phase: 'ready', audio: new Blob(['wav'], { type: 'audio/wav' }) }])
|
||||
|
||||
await expect.element(button).toHaveAttribute('aria-pressed', 'false')
|
||||
await expect.element(screen.getByTestId('voice-status-bar')).not.toBeInTheDocument()
|
||||
expect(commands).toHaveLength(1)
|
||||
})
|
||||
|
||||
it('turns the microphone off and on from the menu', async () => {
|
||||
const { screen } = await mountButton()
|
||||
await screen.getByTestId('voice-input-button').click({ button: 'right' })
|
||||
const microphone = screen.getByTestId('voice-menu-microphone')
|
||||
await expect.element(microphone).toHaveAttribute('aria-checked', 'true')
|
||||
|
||||
await microphone.click()
|
||||
|
||||
await expect.element(microphone).toHaveAttribute('aria-checked', 'false')
|
||||
expect(localStorage.getItem('settings/audio/input/off')).toBe('true')
|
||||
await expect.element(screen.getByTestId('voice-menu-listen')).toHaveAttribute('data-disabled')
|
||||
|
||||
await microphone.click()
|
||||
|
||||
await expect.element(microphone).toHaveAttribute('aria-checked', 'true')
|
||||
expect(localStorage.getItem('settings/audio/input/off')).toBe('false')
|
||||
})
|
||||
|
||||
it('shows no Push to Talk warning in the menu when the host has a trigger', async () => {
|
||||
localStorage.setItem('settings/hearing/input-mode', 'push-to-talk')
|
||||
const { screen } = await mountButton({ pushToTalkTrigger: true })
|
||||
|
||||
@@ -76,7 +76,7 @@ const listenLabelKeys = {
|
||||
'wake-word': 'stage.chat.voice-composer.wake-word',
|
||||
} as const satisfies Record<HearingInputMode, string>
|
||||
const devices = useSettingsAudioDevice()
|
||||
const { audioInputOptions, permission, selectedAudioInput, enabled: listening } = storeToRefs(devices)
|
||||
const { audioInputOptions, heldOpen, off: microphoneOff, permission, selectedAudioInput, enabled: listening } = storeToRefs(devices)
|
||||
const controls = useVoiceControlsStore()
|
||||
|
||||
/** Composer text before dictation started. The live transcript is shown after it until the input ends. */
|
||||
@@ -129,6 +129,14 @@ watch(transcript, (text) => {
|
||||
async function toggle() {
|
||||
menu.close()
|
||||
if (!active.value) {
|
||||
// The microphone switch is off, so nothing records. The toast tells why and turns the microphone on only on request.
|
||||
if (microphoneOff.value) {
|
||||
toast(t('stage.chat.voice-composer.microphone-off'), {
|
||||
description: t('stage.chat.voice-composer.microphone-off-description'),
|
||||
action: { label: t('stage.chat.voice-composer.turn-on-microphone'), onClick: () => microphoneOff.value = false },
|
||||
})
|
||||
return
|
||||
}
|
||||
if (mode.value === 'transcription' && !voice.transcriptionConfigured.value) {
|
||||
toast(t('stage.chat.voice-composer.configure-title'), {
|
||||
description: t('stage.chat.voice-composer.configure-description'),
|
||||
@@ -165,32 +173,22 @@ useEventListener(window, 'keydown', (event: KeyboardEvent) => {
|
||||
void cancel()
|
||||
})
|
||||
|
||||
// NOTICE:
|
||||
// A browser resumes the microphone AudioContext only after a user activation, and a hover is none.
|
||||
// So the meter waits for sticky activation. Electron plays without a gesture.
|
||||
// Source: `microphoneSource` in packages/audio/src/browser/sources.ts awaits `context.resume()`.
|
||||
// Remove when that source stops waiting for resume.
|
||||
const monitor = useIntervalFn(() => {
|
||||
if (navigator.userActivation?.hasBeenActive ?? true)
|
||||
controls.monitorLevel(LEVEL_MONITOR_RENEW_MS * 1.5)
|
||||
}, LEVEL_MONITOR_RENEW_MS, { immediate: false, immediateCallback: true })
|
||||
watch(menuOpen, async (open) => {
|
||||
monitor.pause()
|
||||
if (!open)
|
||||
return
|
||||
// The meter must not cause a permission prompt on hover. Without granted permission it stays empty.
|
||||
const permission = await navigator.permissions?.query({ name: 'microphone' as PermissionName }).catch(() => undefined)
|
||||
if (menuOpen.value && (!permission || permission.state === 'granted'))
|
||||
monitor.resume()
|
||||
})
|
||||
/**
|
||||
* The meter shows only while the microphone preference already holds the microphone open.
|
||||
* A hover opens the menu, and a hover must never open the microphone, cause a permission prompt, or turn on the system
|
||||
* microphone indicator. While the microphone is closed, the menu shows the device without a meter.
|
||||
*/
|
||||
const meterVisible = computed(() => menuOpen.value && heldOpen.value)
|
||||
const monitor = useIntervalFn(() => controls.monitorLevel(LEVEL_MONITOR_RENEW_MS * 1.5), LEVEL_MONITOR_RENEW_MS, { immediate: false, immediateCallback: true })
|
||||
watch(meterVisible, visible => visible ? monitor.resume() : monitor.pause(), { immediate: true })
|
||||
|
||||
/** Level of the selected microphone while the menu is open. The host's level from an earlier capture is not shown. */
|
||||
/** Level of the selected microphone while the meter shows. The host's level from an earlier capture is not shown. */
|
||||
const meterLevel = shallowRef(0)
|
||||
watch(() => controls.level, (value) => {
|
||||
if (menuOpen.value)
|
||||
if (meterVisible.value)
|
||||
meterLevel.value = value
|
||||
})
|
||||
watch(menuOpen, () => meterLevel.value = 0)
|
||||
watch(meterVisible, () => meterLevel.value = 0)
|
||||
|
||||
const currentDevice = computed(() => {
|
||||
const option = audioInputOptions.value.find(item => item.value === selectedAudioInput.value)
|
||||
@@ -235,10 +233,14 @@ const iconClass = computed(() => {
|
||||
return 'i-svg-spinners:ring-resize'
|
||||
if (active.value)
|
||||
return 'i-solar:stop-bold'
|
||||
if (microphoneOff.value)
|
||||
return 'i-mingcute:mic-off-fill'
|
||||
return activeMode.value === 'audio' ? 'i-solar:microphone-3-bold' : 'i-solar:text-field-focus-linear'
|
||||
})
|
||||
|
||||
const buttonLabel = computed(() => {
|
||||
if (!active.value && microphoneOff.value)
|
||||
return t('stage.chat.voice-composer.microphone-off')
|
||||
if (!active.value)
|
||||
return t(`stage.chat.voice-composer.${mode.value}`)
|
||||
return t(autoSendEnabled.value ? 'stage.chat.voice-composer.stop-send' : 'stage.chat.voice-composer.stop')
|
||||
@@ -248,6 +250,7 @@ const itemClasses = [
|
||||
'w-full flex cursor-pointer select-none items-center gap-2 rounded-lg px-3 py-2 text-left',
|
||||
'text-sm leading-none outline-none text-neutral-700 dark:text-neutral-200',
|
||||
'data-[highlighted]:bg-primary-100/80 dark:data-[highlighted]:bg-primary-900/40',
|
||||
'data-[disabled]:cursor-not-allowed data-[disabled]:opacity-50',
|
||||
'transition-colors duration-150 ease-in-out',
|
||||
]
|
||||
const radioItemClasses = [
|
||||
@@ -330,12 +333,25 @@ const separatorClasses = ['mx-2 my-1 h-px bg-neutral-200/80 dark:bg-neutral-700/
|
||||
<DropdownMenuLabel :class="labelClasses">
|
||||
{{ t('stage.chat.voice-composer.input-device') }}
|
||||
</DropdownMenuLabel>
|
||||
<!-- The current microphone and its live level. Its submenu lists the other devices. -->
|
||||
<!-- The microphone switch. Off closes the microphone for every feature, and nothing records until it is on again. -->
|
||||
<DropdownMenuCheckboxItem
|
||||
:model-value="!microphoneOff"
|
||||
data-testid="voice-menu-microphone"
|
||||
:class="itemClasses"
|
||||
@update:model-value="value => microphoneOff = !value"
|
||||
@select.prevent
|
||||
>
|
||||
<span :class="['flex-1']">{{ t('stage.chat.voice-composer.use-microphone') }}</span>
|
||||
<span aria-hidden="true" :class="switchTrackClasses(!microphoneOff)">
|
||||
<span :class="switchThumbClasses(!microphoneOff)" />
|
||||
</span>
|
||||
</DropdownMenuCheckboxItem>
|
||||
<!-- The current microphone, with its live level while the microphone is open. Its submenu lists the other devices. -->
|
||||
<DropdownMenuSub>
|
||||
<DropdownMenuSubTrigger data-testid="voice-menu-device" :class="[itemClasses, 'py-1.5']">
|
||||
<span :class="['min-w-0 flex flex-1 flex-col gap-1.5']">
|
||||
<span :class="['truncate']">{{ currentDevice }}</span>
|
||||
<span aria-hidden="true" :class="['h-1 w-full overflow-hidden rounded-full bg-neutral-900/10 dark:bg-white/10']">
|
||||
<span v-if="meterVisible" aria-hidden="true" :class="['h-1 w-full overflow-hidden rounded-full bg-neutral-900/10 dark:bg-white/10']">
|
||||
<span
|
||||
data-testid="voice-menu-level"
|
||||
:class="['block h-full rounded-full bg-primary-500 transition-[width] duration-75 motion-reduce:transition-none']"
|
||||
@@ -390,8 +406,8 @@ const separatorClasses = ['mx-2 my-1 h-px bg-neutral-200/80 dark:bg-neutral-700/
|
||||
<span :class="switchThumbClasses(autoSendEnabled)" />
|
||||
</span>
|
||||
</DropdownMenuCheckboxItem>
|
||||
<DropdownMenuCheckboxItem :model-value="listening" data-testid="voice-menu-listen" :class="itemClasses" @update:model-value="setListening" @select.prevent>
|
||||
<!-- The switch turns on the Hearing input mode. The mode itself is chosen in Hearing settings. -->
|
||||
<DropdownMenuCheckboxItem :model-value="listening" :disabled="microphoneOff" data-testid="voice-menu-listen" :class="itemClasses" @update:model-value="setListening" @select.prevent>
|
||||
<!-- The switch turns on the Hearing input mode. The mode itself is chosen in Hearing settings. It waits while the microphone is off. -->
|
||||
<span :class="['flex-1']">{{ t(listenLabelKeys[inputMode]) }}</span>
|
||||
<span aria-hidden="true" :class="switchTrackClasses(listening)">
|
||||
<span :class="switchThumbClasses(listening)" />
|
||||
|
||||
@@ -120,7 +120,9 @@ export function useVoiceComposer(options: UseVoiceComposerOptions) {
|
||||
active.value = undefined
|
||||
}
|
||||
// A recording finished without sending becomes a pending message. The control is free for the next recording.
|
||||
else if (snapshot.phase === 'ready' && recording.finishing && !recording.send) {
|
||||
// The host can also finish a recording that the user did not stop, when the microphone turns off or another
|
||||
// window takes it. That recording waits for the user too.
|
||||
else if (snapshot.phase === 'ready' && (!recording.finishing || !recording.send)) {
|
||||
active.value = undefined
|
||||
}
|
||||
// The host owns a message that it sends. The send can wait in the session queue behind a reply, so the control
|
||||
|
||||
+23
@@ -95,6 +95,29 @@ describe('hearing config audio device ownership', () => {
|
||||
localStorage.clear()
|
||||
})
|
||||
|
||||
// The island and the mobile Hearing drawer show these controls. Users must be able to close the microphone there.
|
||||
it('turns the microphone off for every feature, and keeps listening waiting until it is on again', async () => {
|
||||
const { app, host } = mountHearingConfig()
|
||||
const offSwitch = () => host.querySelector<HTMLButtonElement>('[data-testid="hearing-microphone-off-switch"] [role="switch"]')!
|
||||
await nextTick()
|
||||
expect(offSwitch().getAttribute('aria-checked')).toBe('true')
|
||||
|
||||
offSwitch().click()
|
||||
|
||||
await expect.poll(() => offSwitch().getAttribute('aria-checked')).toBe('false')
|
||||
expect(localStorage.getItem('settings/audio/input/off')).toBe('true')
|
||||
expect(microphoneSwitch(host).disabled).toBe(true)
|
||||
|
||||
offSwitch().click()
|
||||
|
||||
await expect.poll(() => offSwitch().getAttribute('aria-checked')).toBe('true')
|
||||
expect(microphoneSwitch(host).disabled).toBe(false)
|
||||
expect(audioDeviceMocks.storeAskPermission).not.toHaveBeenCalled()
|
||||
|
||||
app.unmount()
|
||||
host.remove()
|
||||
})
|
||||
|
||||
it('does not request microphone permission when the controls render', async () => {
|
||||
const { app, host } = mountHearingConfig()
|
||||
await nextTick()
|
||||
|
||||
@@ -22,7 +22,7 @@ const { t } = useI18n()
|
||||
const { inputMode } = storeToRefs(useHearingStore())
|
||||
const { available: pushToTalkAvailable, stranded: pushToTalkStranded } = usePushToTalkTrigger()
|
||||
const deviceStore = useSettingsAudioDevice()
|
||||
const { audioInputOptions, enabled, error, permission, selectedAudioInput } = storeToRefs(deviceStore)
|
||||
const { audioInputOptions, enabled, error, off: microphoneOff, permission, selectedAudioInput } = storeToRefs(deviceStore)
|
||||
// A phone shows this config in a bottom drawer. A select there opens a second floating layer, so choices are rows.
|
||||
const { isDesktop } = useBreakpoints()
|
||||
|
||||
@@ -55,7 +55,7 @@ const enabling = shallowRef(false)
|
||||
* Only this explicit switch action asks for microphone permission. Rendering the controls never opens the microphone.
|
||||
* A denied request leaves the switch off. The device store keeps the reason in `error`, which the callout shows.
|
||||
*/
|
||||
async function setMicrophoneEnabled(value: boolean) {
|
||||
async function setListening(value: boolean) {
|
||||
if (enabling.value)
|
||||
return
|
||||
if (!value) {
|
||||
@@ -78,6 +78,15 @@ async function setMicrophoneEnabled(value: boolean) {
|
||||
|
||||
<template>
|
||||
<div :class="['flex flex-col gap-4']">
|
||||
<!-- The microphone switch. Off closes the microphone for every feature, so listening below waits until it is on. -->
|
||||
<FieldCheckbox
|
||||
data-testid="hearing-microphone-off-switch"
|
||||
:model-value="!microphoneOff"
|
||||
:label="t('settings.pages.modules.hearing.sections.section.microphone-switch.label')"
|
||||
:description="t('settings.pages.modules.hearing.sections.section.microphone-switch.description')"
|
||||
@update:model-value="value => microphoneOff = !value"
|
||||
/>
|
||||
|
||||
<FieldSelect
|
||||
v-if="isDesktop"
|
||||
v-model="inputMode"
|
||||
@@ -107,10 +116,10 @@ async function setMicrophoneEnabled(value: boolean) {
|
||||
<FieldCheckbox
|
||||
data-testid="hearing-microphone-switch"
|
||||
:model-value="enabled"
|
||||
:disabled="enabling"
|
||||
:label="t('settings.pages.modules.hearing.sections.section.microphone.label')"
|
||||
:description="t('settings.pages.modules.hearing.sections.section.microphone.description')"
|
||||
@update:model-value="setMicrophoneEnabled"
|
||||
:disabled="enabling || microphoneOff"
|
||||
:label="t('settings.pages.modules.hearing.sections.section.listen.label')"
|
||||
:description="t('settings.pages.modules.hearing.sections.section.listen.description')"
|
||||
@update:model-value="setListening"
|
||||
/>
|
||||
|
||||
<MicrophonePicker
|
||||
|
||||
@@ -7,12 +7,13 @@ import { errorMessageFrom } from '@moeru/std'
|
||||
import { encodeWav } from '@proj-airi/audio/encoding'
|
||||
import { capture as captureAudio, observe } from '@proj-airi/pipelines-audio'
|
||||
import { Button, FieldRange } from '@proj-airi/ui'
|
||||
import { computed, onScopeDispose, ref, watch } from 'vue'
|
||||
import { computed, onScopeDispose, ref, toRef, watch } from 'vue'
|
||||
import { useI18n } from 'vue-i18n'
|
||||
|
||||
import MicrophonePicker from '../settings/microphone-picker.vue'
|
||||
|
||||
import { useAudioDevice } from '../../../composables/audio/audio-device'
|
||||
import { useSettingsAudioDevice } from '../../../stores/settings/audio-device'
|
||||
import { LevelMeter, TestDummyMarker, ThresholdMeter } from '../../gadgets'
|
||||
|
||||
const props = defineProps<{
|
||||
@@ -21,7 +22,8 @@ const props = defineProps<{
|
||||
}>()
|
||||
|
||||
const { t } = useI18n()
|
||||
const devices = useAudioDevice()
|
||||
// The playground picks its own device, but the microphone switch of AIRI still applies to it.
|
||||
const devices = useAudioDevice(ref(''), { off: toRef(useSettingsAudioDevice(), 'off') })
|
||||
const { audioInputOptions, permission, selectedAudioInput } = devices
|
||||
const speakingThreshold = ref(25)
|
||||
const isMonitoring = ref(false)
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
import { createPinia, disposePinia } from 'pinia'
|
||||
import { afterEach, expect, it, vi } from 'vitest'
|
||||
import { createApp, defineComponent, h, ref } from 'vue'
|
||||
import { createI18n } from 'vue-i18n'
|
||||
|
||||
import { useAudioDevice } from './audio-device'
|
||||
|
||||
const cleanups: (() => void)[] = []
|
||||
|
||||
afterEach(() => {
|
||||
cleanups.splice(0).forEach(cleanup => cleanup())
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
|
||||
function mountDevice() {
|
||||
const off = ref(false)
|
||||
const selected = ref('')
|
||||
let device!: ReturnType<typeof useAudioDevice>
|
||||
const app = createApp(defineComponent({ setup() {
|
||||
device = useAudioDevice(selected, { off })
|
||||
return () => h('div')
|
||||
} }))
|
||||
// Analytics reads a store and translations.
|
||||
const pinia = createPinia()
|
||||
app.use(pinia).use(createI18n({ legacy: false, locale: 'en', messages: { en: {} }, missingWarn: false, fallbackWarn: false })).mount(document.createElement('div'))
|
||||
cleanups.push(() => {
|
||||
app.unmount()
|
||||
disposePinia(pinia)
|
||||
})
|
||||
return { device, off, selected }
|
||||
}
|
||||
|
||||
// Users must be able to turn the microphone off for every feature, and see the system indicator turn off.
|
||||
it('closes every capture and opens no new one while the microphone is off', async () => {
|
||||
const { device, off, selected } = mountDevice()
|
||||
// The device list decides which device the system default opens. A later list change replaces the input.
|
||||
await device.askPermission()
|
||||
await expect.poll(() => selected.value).not.toBe('')
|
||||
const subscription = new AbortController()
|
||||
const reader = device.input.value.subscribe({ signal: subscription.signal }).getReader()
|
||||
await reader.read()
|
||||
const track = device.stream.value!.getAudioTracks()[0]
|
||||
|
||||
off.value = true
|
||||
|
||||
await expect.poll(() => track.readyState).toBe('ended')
|
||||
expect(await reader.read()).toEqual({ done: true, value: undefined })
|
||||
const openMicrophone = vi.spyOn(navigator.mediaDevices, 'getUserMedia')
|
||||
await expect(device.input.value.subscribe().getReader().read()).rejects.toThrow('The microphone is turned off in AIRI')
|
||||
await expect(device.unprocessedInput.value.subscribe().getReader().read()).rejects.toThrow('The microphone is turned off in AIRI')
|
||||
expect(openMicrophone).not.toHaveBeenCalled()
|
||||
|
||||
off.value = false
|
||||
const reopened = new AbortController()
|
||||
cleanups.push(() => reopened.abort())
|
||||
await device.input.value.subscribe({ signal: reopened.signal }).getReader().read()
|
||||
|
||||
expect(openMicrophone).toHaveBeenCalledOnce()
|
||||
})
|
||||
@@ -40,6 +40,17 @@ function audioDeviceErrorCode(error: unknown): 'permission_denied' | 'device_una
|
||||
*/
|
||||
export type MicrophonePermission = 'granted' | 'denied' | 'prompt' | 'unknown'
|
||||
|
||||
/**
|
||||
* The error of a capture that starts while the user has turned the microphone off in AIRI.
|
||||
* Controls check the switch before they start, so this error reaches only a consumer that did not check it.
|
||||
*/
|
||||
export class MicrophoneOffError extends Error {
|
||||
constructor() {
|
||||
super('The microphone is turned off in AIRI')
|
||||
this.name = 'MicrophoneOffError'
|
||||
}
|
||||
}
|
||||
|
||||
/** Prefers the system default microphone, then the first detected input. */
|
||||
function resolvePreferredAudioInput(audioInputs: readonly MediaDeviceInfo[]) {
|
||||
return audioInputs.find(device => device.deviceId === 'default')?.deviceId || audioInputs[0]?.deviceId || ''
|
||||
@@ -58,8 +69,13 @@ function resolvePreferredAudioInput(audioInputs: readonly MediaDeviceInfo[]) {
|
||||
*
|
||||
* `selectedAudioInput` can be a persisted preference. An empty or unavailable selection falls back to the
|
||||
* preferred device once the browser lists device IDs, so controls show the device that capture uses.
|
||||
*
|
||||
* While `off` is true, the microphone stays closed for every consumer. Turning it off ends every open subscription
|
||||
* of both inputs, so the system microphone indicator turns off at once. A new subscription then fails with
|
||||
* {@link MicrophoneOffError} and opens no device.
|
||||
*/
|
||||
export function useAudioDevice(selectedAudioInput: Ref<string> = ref('')) {
|
||||
export function useAudioDevice(selectedAudioInput: Ref<string> = ref(''), options: { off?: Readonly<Ref<boolean>> } = {}) {
|
||||
const off = options.off ?? ref(false)
|
||||
const { trackMicrophonePermissionDenied } = useAnalytics()
|
||||
const { devices, audioInputs } = useDevicesList({ requestPermissions: false })
|
||||
const permissionGranted = ref(false)
|
||||
@@ -109,21 +125,37 @@ export function useAudioDevice(selectedAudioInput: Ref<string> = ref('')) {
|
||||
const input = shallowRef(markRaw(new AudioInput(source.value, { historyMs: MICROPHONE_HISTORY_MS })))
|
||||
const unprocessedInput = shallowRef(createUnprocessedInput())
|
||||
|
||||
/** Refuses to open the device while the microphone is off. The check runs for each new connection. */
|
||||
function guard(source: MicrophoneSource): MicrophoneSource {
|
||||
return {
|
||||
live: true,
|
||||
get stream() {
|
||||
return source.stream
|
||||
},
|
||||
get echoCancellation() {
|
||||
return source.echoCancellation
|
||||
},
|
||||
open: signal => off.value
|
||||
? new ReadableStream({ start: controller => controller.error(new MicrophoneOffError()) })
|
||||
: source.open(signal),
|
||||
}
|
||||
}
|
||||
|
||||
function createSource(constraints: MediaStreamConstraints): MicrophoneSource {
|
||||
const created = markRaw(microphoneSource(constraints, {
|
||||
const created = markRaw(guard(microphoneSource(constraints, {
|
||||
contextOptions: MICROPHONE_CONTEXT,
|
||||
// A replaced device can close after the new one opened. Only the selected source updates the view.
|
||||
onStream: (opened) => {
|
||||
if (source.value === created)
|
||||
stream.value = opened
|
||||
},
|
||||
}))
|
||||
})))
|
||||
return created
|
||||
}
|
||||
|
||||
/** Keyword spotting needs no history, because a wake starts its input after the keyword. */
|
||||
function createUnprocessedInput() {
|
||||
return markRaw(new AudioInput(markRaw(microphoneSource(unprocessedConstraints.value, { contextOptions: MICROPHONE_CONTEXT }))))
|
||||
return markRaw(new AudioInput(markRaw(guard(microphoneSource(unprocessedConstraints.value, { contextOptions: MICROPHONE_CONTEXT })))))
|
||||
}
|
||||
|
||||
// Browsers hide device IDs until permission, so the fallback waits for a list with IDs.
|
||||
@@ -142,11 +174,33 @@ export function useAudioDevice(selectedAudioInput: Ref<string> = ref('')) {
|
||||
unprocessedInput.value = createUnprocessedInput()
|
||||
})
|
||||
|
||||
// Closing ends the subscriptions without an error, as if the device stopped. Owners of an active recording
|
||||
// stop it first in their own watchers of the same switch, so a closed input never looks like finished speech.
|
||||
watch(off, (value) => {
|
||||
if (!value)
|
||||
return
|
||||
void input.value.close()
|
||||
void unprocessedInput.value.close()
|
||||
})
|
||||
|
||||
/**
|
||||
* Subscribes until the first block arrives, then leaves. It reports a denied or missing device.
|
||||
* Device labels need enumeration after permission, so this also refreshes the device list.
|
||||
* Gets microphone access, and refreshes the device list, because device labels need enumeration after permission.
|
||||
*
|
||||
* With access already granted, it only refreshes the list. Every capture turns on the system microphone indicator,
|
||||
* so a capture only to check access shows the indicator although nothing records. A missing device then fails at
|
||||
* the first real capture.
|
||||
* Without granted access, it subscribes until the first block arrives, then leaves. That asks the user for access,
|
||||
* and it reports a denied or missing device.
|
||||
*/
|
||||
async function askPermission() {
|
||||
// The state comes from the Permissions API watch above, not from a new query. The probe must open in the same
|
||||
// task as the click that called it, so that the click still counts as a user gesture for its AudioContext.
|
||||
if (permission.value === 'granted') {
|
||||
permissionGranted.value = true
|
||||
devices.value = await navigator.mediaDevices.enumerateDevices()
|
||||
return
|
||||
}
|
||||
|
||||
const probe = new AbortController()
|
||||
const reader = input.value.subscribe({ signal: probe.signal }).getReader()
|
||||
try {
|
||||
|
||||
@@ -18,7 +18,7 @@ export function useVoiceListening() {
|
||||
const devices = useSettingsAudioDevice()
|
||||
const hearing = useHearingStore()
|
||||
|
||||
watch(() => hearing.enabled && devices.enabled && listensContinuously(hearing.inputMode), (listen) => {
|
||||
watch(() => hearing.enabled && devices.listening && listensContinuously(hearing.inputMode), (listen) => {
|
||||
if (listen)
|
||||
voice.startListening()
|
||||
else
|
||||
@@ -48,7 +48,7 @@ export function useVoicePushToTalk(options: VoicePushToTalkOptions = {}) {
|
||||
const sessions = useChatSessionStore()
|
||||
|
||||
return useVoiceHold({
|
||||
enabled: () => hearing.enabled && devices.enabled && hearing.inputMode === 'push-to-talk',
|
||||
enabled: () => hearing.enabled && devices.listening && hearing.inputMode === 'push-to-talk',
|
||||
sessionId: () => sessions.activeSessionId,
|
||||
begin: (sessionId) => {
|
||||
if (!hearing.configured) {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { createPinia, disposePinia } from 'pinia'
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import { createApp, defineComponent, h } from 'vue'
|
||||
import { createI18n } from 'vue-i18n'
|
||||
|
||||
@@ -30,6 +30,7 @@ function mountDevices() {
|
||||
|
||||
afterEach(() => {
|
||||
cleanups.splice(0).forEach(cleanup => cleanup())
|
||||
vi.restoreAllMocks()
|
||||
localStorage.clear()
|
||||
})
|
||||
|
||||
@@ -129,6 +130,21 @@ describe('microphone settings through the browser adapter', () => {
|
||||
await expect.poll(() => track.readyState).toBe('ended')
|
||||
})
|
||||
|
||||
// Turning microphone input on asked for access first, then opened the microphone again for the hold.
|
||||
// The system microphone indicator then flashed on, off, and on.
|
||||
it('opens no microphone to ask for access that is already granted', async () => {
|
||||
const devices = mountDevices()
|
||||
expect(await devices.askPermission()).toBe(true)
|
||||
await expect.poll(() => devices.permission).toBe('granted')
|
||||
await expect.poll(() => devices.stream).toBeUndefined()
|
||||
const openMicrophone = vi.spyOn(navigator.mediaDevices, 'getUserMedia')
|
||||
|
||||
expect(await devices.askPermission()).toBe(true)
|
||||
|
||||
expect(openMicrophone).not.toHaveBeenCalled()
|
||||
expect(devices.audioInputOptions.length).toBeGreaterThan(0)
|
||||
})
|
||||
|
||||
it('keeps the selected device and exposes unavailable-device failures', async () => {
|
||||
localStorage.setItem('settings/audio/input', 'missing-test-microphone')
|
||||
const devices = mountDevices()
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { errorMessageFrom } from '@moeru/std'
|
||||
import { useLocalStorageManualReset } from '@proj-airi/stage-shared/composables'
|
||||
import { defineStore } from 'pinia'
|
||||
import { onScopeDispose, ref, watch } from 'vue'
|
||||
import { computed, onScopeDispose, ref, watch } from 'vue'
|
||||
|
||||
import { useAudioDevice } from '../../composables/audio'
|
||||
import { listensContinuously, useHearingStore } from '../modules/hearing'
|
||||
@@ -13,28 +13,43 @@ function isAbort(cause: unknown) {
|
||||
/**
|
||||
* Stores microphone preferences and keeps the selected microphone open while it is enabled.
|
||||
*
|
||||
* Two persisted preferences control the microphone:
|
||||
* - `off` is the microphone switch. While `off` is true, no feature in AIRI can open the microphone. Setting it to
|
||||
* true ends every open capture at once. Controls then show the microphone as off and do not start a recording.
|
||||
* - `enabled` is the listening preference. It has effect only while `off` is false. Then the Hearing input mode
|
||||
* controls when AIRI listens. `listening` combines the two.
|
||||
*
|
||||
* `input` is the shared input for the selected device. Consumers subscribe with their own signal.
|
||||
* `unprocessedInput` is the same device without browser processing. Only keyword spotting subscribes to it, so the
|
||||
* preference does not hold it open.
|
||||
* The enabled preference holds one extra subscription, so later consumers start without waiting for permission.
|
||||
* While AIRI listens, the store holds one extra subscription, so later consumers start without waiting for permission.
|
||||
* A Hearing input mode that does not listen continuously, such as Push to Talk, skips that subscription.
|
||||
* Then the microphone opens only while a speech input or a voice message uses it.
|
||||
*/
|
||||
export const useSettingsAudioDevice = defineStore('settings-audio-devices', () => {
|
||||
const selectedAudioInput = useLocalStorageManualReset<string>('settings/audio/input', '')
|
||||
const device = useAudioDevice(selectedAudioInput)
|
||||
const enabled = useLocalStorageManualReset<boolean>('settings/audio/input/enabled', false)
|
||||
const off = useLocalStorageManualReset<boolean>('settings/audio/input/off', false)
|
||||
const device = useAudioDevice(selectedAudioInput, { off })
|
||||
const hearing = useHearingStore()
|
||||
const error = ref<string>()
|
||||
let permissionStatus: PermissionStatus | undefined
|
||||
let disposed = false
|
||||
let initialized = false
|
||||
let holding: AbortController | undefined
|
||||
/** True while AIRI listens in the Hearing input mode: the microphone is on and the listening preference is on. */
|
||||
const listening = computed(() => enabled.value && !off.value)
|
||||
/**
|
||||
* True while the preference keeps the microphone open in the window that initialized this store.
|
||||
* Each window computes it from the same persisted settings. A level meter observes the input only while this is
|
||||
* true, so a meter alone never opens the microphone.
|
||||
*/
|
||||
const heldOpen = computed(() => listening.value && listensContinuously(hearing.inputMode))
|
||||
|
||||
/** Holds the current input open. A failure other than release turns the preference off. */
|
||||
function hold() {
|
||||
holding?.abort('Microphone hold replaced')
|
||||
if (!enabled.value || !listensContinuously(hearing.inputMode) || !initialized)
|
||||
if (!heldOpen.value || !initialized)
|
||||
return
|
||||
|
||||
const current = new AbortController()
|
||||
@@ -48,7 +63,7 @@ export const useSettingsAudioDevice = defineStore('settings-audio-devices', () =
|
||||
})
|
||||
}
|
||||
|
||||
watch([enabled, () => hearing.inputMode, device.input], hold, { flush: 'sync' })
|
||||
watch([heldOpen, device.input], hold, { flush: 'sync' })
|
||||
|
||||
function initialize() {
|
||||
if (initialized || disposed)
|
||||
@@ -89,6 +104,7 @@ export const useSettingsAudioDevice = defineStore('settings-audio-devices', () =
|
||||
error.value = undefined
|
||||
selectedAudioInput.reset()
|
||||
enabled.reset()
|
||||
off.reset()
|
||||
}
|
||||
|
||||
onScopeDispose(() => {
|
||||
@@ -101,6 +117,9 @@ export const useSettingsAudioDevice = defineStore('settings-audio-devices', () =
|
||||
return {
|
||||
error,
|
||||
enabled,
|
||||
off,
|
||||
listening,
|
||||
heldOpen,
|
||||
selectedAudioInput,
|
||||
audioInputs: device.audioInputs,
|
||||
audioInputOptions: device.audioInputOptions,
|
||||
|
||||
@@ -8,6 +8,7 @@ import { createI18n } from 'vue-i18n'
|
||||
|
||||
import { holdCaptureLease } from '../libs/voice/capture-lease'
|
||||
import { getSpeechBusContext, voiceMessageCommand, voiceMessagesChanged } from '../services/speech/bus'
|
||||
import { useSettingsAudioDevice } from './settings/audio-device'
|
||||
import { useVoiceCaptureLeaseStore } from './voice-capture-lease'
|
||||
import { useVoiceMessagesStore } from './voice-messages'
|
||||
|
||||
@@ -44,7 +45,7 @@ function connectHost() {
|
||||
disposePinia(pinia)
|
||||
element.remove()
|
||||
})
|
||||
return { store, leases: useVoiceCaptureLeaseStore(pinia), latest: () => snapshots.at(-1) ?? [] }
|
||||
return { store, leases: useVoiceCaptureLeaseStore(pinia), devices: useSettingsAudioDevice(pinia), latest: () => snapshots.at(-1) ?? [] }
|
||||
}
|
||||
|
||||
describe('voice messages host', () => {
|
||||
@@ -107,4 +108,39 @@ describe('voice messages host', () => {
|
||||
|
||||
expect(await command({ type: 'record', id: 'message', sessionId: 'alice' })).toEqual({ status: 'accepted' })
|
||||
})
|
||||
|
||||
// The microphone switch in a composer menu or in Hearing controls must stop a recording at once.
|
||||
// The user keeps the audio that was recorded before, as with a capture lease.
|
||||
it('keeps a recording as a draft and closes the microphone when the microphone turns off', async () => {
|
||||
const { store, devices, latest } = connectHost()
|
||||
// The device list decides which device the system default opens. A later list change replaces the input.
|
||||
expect(await devices.askPermission()).toBe(true)
|
||||
await expect.poll(() => devices.selectedAudioInput).not.toBe('')
|
||||
const command = defineInvoke(getSpeechBusContext(), voiceMessageCommand)
|
||||
await command({ type: 'record', id: 'message', sessionId: 'alice' })
|
||||
await expect.poll(() => latest()[0]?.phase).toBe('capturing')
|
||||
await expect.poll(() => devices.stream?.getAudioTracks()[0].readyState).toBe('live')
|
||||
const track = devices.stream!.getAudioTracks()[0]
|
||||
|
||||
devices.off = true
|
||||
|
||||
await expect.poll(() => latest()[0]?.phase).toBe('ready')
|
||||
expect(store.isRecording).toBe(false)
|
||||
await expect.poll(() => track.readyState).toBe('ended')
|
||||
// Browser test files run in parallel and share storage. Turn the persisted switch back at once.
|
||||
devices.off = false
|
||||
})
|
||||
|
||||
it('does not record while the microphone is off', async () => {
|
||||
const { devices } = connectHost()
|
||||
const command = defineInvoke(getSpeechBusContext(), voiceMessageCommand)
|
||||
devices.off = true
|
||||
const refused = await command({ type: 'record', id: 'message', sessionId: 'alice' })
|
||||
// Browser test files run in parallel and share storage. Turn the persisted switch back at once.
|
||||
devices.off = false
|
||||
|
||||
expect(refused).toEqual({ status: 'closed' })
|
||||
|
||||
expect(await command({ type: 'record', id: 'message', sessionId: 'alice' })).toEqual({ status: 'accepted' })
|
||||
})
|
||||
})
|
||||
|
||||
@@ -63,9 +63,9 @@ export const useVoiceMessagesStore = defineStore('voice-messages', () => {
|
||||
return completed ? text.trim() : (text.trim() || undefined)
|
||||
}
|
||||
|
||||
/** Returns false for a used ID, and while another window holds a capture lease. */
|
||||
/** Returns false for a used ID, while another window holds a capture lease, and while the microphone is off. */
|
||||
function record(command: Extract<VoiceMessageCommand, { type: 'record' }>) {
|
||||
if (opened.has(command.id) || captureLease.held)
|
||||
if (opened.has(command.id) || captureLease.held || devices.off)
|
||||
return false
|
||||
opened.add(command.id)
|
||||
const message = new VoiceMessage(command.id, command.sessionId, devices.input, async (draft) => {
|
||||
@@ -129,15 +129,17 @@ export const useVoiceMessagesStore = defineStore('voice-messages', () => {
|
||||
}
|
||||
return { status: 'accepted' }
|
||||
}),
|
||||
// A capture lease needs the microphone alone. Keep what the user recorded as a draft that they can send or discard.
|
||||
watch(() => captureLease.held, (held) => {
|
||||
if (!held)
|
||||
// A capture lease needs the microphone alone, and the microphone switch closes it for everyone.
|
||||
// Keep what the user recorded as a draft that they can send or discard.
|
||||
// The watcher is synchronous, so the recording stops before the device store closes the input.
|
||||
watch(() => captureLease.held || devices.off, (stop) => {
|
||||
if (!stop)
|
||||
return
|
||||
for (const message of messages.values()) {
|
||||
if (['pending', 'capturing'].includes(message.snapshot.phase))
|
||||
void message.finish()
|
||||
}
|
||||
}),
|
||||
}, { flush: 'sync' }),
|
||||
]
|
||||
publish()
|
||||
return () => {
|
||||
|
||||
@@ -7,10 +7,11 @@ import { afterEach, expect, it, vi } from 'vitest'
|
||||
import { createApp } from 'vue'
|
||||
import { createI18n } from 'vue-i18n'
|
||||
|
||||
import { getSpeechBusContext, voiceSpeechCommand } from '../services/speech/bus'
|
||||
import { getSpeechBusContext, voiceInputLevel, voiceLevelMonitor, voiceSpeechCommand } from '../services/speech/bus'
|
||||
import { useLlmStreamingControlStore } from './ai/chat-llm/streaming-control'
|
||||
import { useChatStore } from './chat'
|
||||
import { useSpeechStore } from './modules/speech'
|
||||
import { useSettingsAudioDevice } from './settings/audio-device'
|
||||
import { useVoiceStore } from './voice'
|
||||
|
||||
const cleanups: (() => void)[] = []
|
||||
@@ -89,3 +90,54 @@ it.each(['local', 'remote'] as const)('issue #3153: disabling %s speech preserve
|
||||
expect(cancelChat).not.toHaveBeenCalled()
|
||||
await expect(generation.done).resolves.toEqual({ type: 'completed' })
|
||||
})
|
||||
|
||||
// A level meter in a menu asks for the level when a hover opens the menu. Users saw the system microphone indicator
|
||||
// turn on although microphone input was off, because the host opened the microphone for the meter.
|
||||
it('publishes a requested level only while microphone input holds the microphone open', async () => {
|
||||
const pinia = createPinia()
|
||||
let voice!: ReturnType<typeof useVoiceStore>
|
||||
let devices!: ReturnType<typeof useSettingsAudioDevice>
|
||||
const app = createApp({ setup() {
|
||||
voice = useVoiceStore()
|
||||
devices = useSettingsAudioDevice()
|
||||
return () => null
|
||||
} }).use(pinia).use(createI18n({ legacy: false, locale: 'en', messages: { en: {} }, missingWarn: false, fallbackWarn: false }))
|
||||
app.mount(document.createElement('div'))
|
||||
cleanups.push(() => {
|
||||
app.unmount()
|
||||
disposePinia(pinia)
|
||||
})
|
||||
const output: SpeechOutput = {
|
||||
playback: new Playback({ nowMs: () => 0, play: () => ({ done: new Promise<{ throughMs: number }>(() => {}), stop: async () => ({ throughMs: 0 }) }) }),
|
||||
stream: async () => new ReadableStream(),
|
||||
}
|
||||
cleanups.push(voice.connectOutput(() => output))
|
||||
devices.initialize()
|
||||
// The device list decides which device the system default opens. A later list change replaces the input.
|
||||
expect(await devices.askPermission()).toBe(true)
|
||||
await expect.poll(() => devices.selectedAudioInput).not.toBe('')
|
||||
await expect.poll(() => devices.stream).toBeUndefined()
|
||||
const context = getSpeechBusContext()
|
||||
const levels: number[] = []
|
||||
cleanups.push(context.on(voiceInputLevel, ({ body }) => body && levels.push(body.level)))
|
||||
const openMicrophone = vi.spyOn(navigator.mediaDevices, 'getUserMedia')
|
||||
|
||||
context.emit(voiceLevelMonitor, { until: Date.now() + 5000 })
|
||||
await new Promise(resolve => setTimeout(resolve, 200))
|
||||
|
||||
expect(openMicrophone).not.toHaveBeenCalled()
|
||||
expect(levels).toEqual([])
|
||||
|
||||
devices.enabled = true
|
||||
context.emit(voiceLevelMonitor, { until: Date.now() + 10_000 })
|
||||
|
||||
// The first level waits for the device and the capture worklet, which is slow while other test files run.
|
||||
await expect.poll(() => levels.length, { timeout: 10_000 }).toBeGreaterThan(0)
|
||||
expect(openMicrophone).toHaveBeenCalledOnce()
|
||||
const track = devices.stream!.getAudioTracks()[0]
|
||||
|
||||
// The menu still asks for the level. Turning microphone input off must close the microphone anyway.
|
||||
devices.enabled = false
|
||||
|
||||
await expect.poll(() => track.readyState).toBe('ended')
|
||||
})
|
||||
|
||||
@@ -145,7 +145,7 @@ export const useVoiceStore = defineStore('voice', () => {
|
||||
|
||||
const snapshot: VoiceHostSnapshot = {
|
||||
connected: true,
|
||||
microphone: { enabled: devices.enabled, ready: !!devices.stream, error: devices.error },
|
||||
microphone: { enabled: devices.listening, ready: !!devices.stream, error: devices.error },
|
||||
wakeWords: { preparation: wakeWordDetection.preparation, ...(wakeWordDetection.error ? { error: wakeWordDetection.error } : {}) },
|
||||
drafts: drafts.value.map(draft => ({ ...draft })),
|
||||
frontDraftId: frontDraftId.value,
|
||||
@@ -184,7 +184,7 @@ export const useVoiceStore = defineStore('voice', () => {
|
||||
|
||||
/** Begins an input that a control ends. Returns nothing while another window holds a capture lease. */
|
||||
function beginManual(sessionId: string) {
|
||||
if (!hearing.enabled || captureLease.held)
|
||||
if (!hearing.enabled || captureLease.held || devices.off)
|
||||
return undefined
|
||||
return beginInput({ sessionId, interruptTurns: activeTurns.value.filter(turn => turn.sessionId === sessionId), start: { kind: 'after-silence' }, cause: 'control' })
|
||||
}
|
||||
@@ -267,7 +267,11 @@ export const useVoiceStore = defineStore('voice', () => {
|
||||
*/
|
||||
function publishInputLevel() {
|
||||
let observation: AbortController | undefined
|
||||
/** A control shows a level meter. It turns off when the newest request expires. */
|
||||
/**
|
||||
* A control shows a level meter. It turns off when the newest request expires.
|
||||
* The meter reads the level only while the microphone preference holds the input open. A user can hover the meter
|
||||
* without intent to speak, so the meter alone must not open the microphone. While the input is closed, no level arrives.
|
||||
*/
|
||||
const monitoring = shallowRef(false)
|
||||
let monitorTimer: ReturnType<typeof setTimeout> | undefined
|
||||
const stopMonitor = getSpeechBusContext().on(voiceLevelMonitor, ({ body }) => {
|
||||
@@ -280,7 +284,7 @@ export const useVoiceStore = defineStore('voice', () => {
|
||||
monitorTimer = setTimeout(() => monitoring.value = false, remaining)
|
||||
})
|
||||
const stop = watch(
|
||||
[() => voiceMessages.isRecording || state.value?.phase === 'capturing' || monitoring.value, () => devices.input],
|
||||
[() => voiceMessages.isRecording || state.value?.phase === 'capturing' || (monitoring.value && devices.heldOpen), () => devices.input],
|
||||
([capturing, input]) => {
|
||||
observation?.abort('Voice capture changed')
|
||||
observation = undefined
|
||||
@@ -346,6 +350,13 @@ export const useVoiceStore = defineStore('voice', () => {
|
||||
if (held)
|
||||
controller.activeInput?.cancel('Another window holds the microphone')
|
||||
}),
|
||||
// The microphone switch ends every input, also dictation in a composer. The input is cancelled, not submitted,
|
||||
// because the user turned the microphone off and did not finish speaking.
|
||||
// The watcher is synchronous, so the input ends before the device store closes the microphone.
|
||||
watch(() => devices.off, (off) => {
|
||||
if (off)
|
||||
controller.activeInput?.cancel('The microphone is turned off')
|
||||
}, { flush: 'sync' }),
|
||||
publishInputLevel(),
|
||||
context.on(voiceGenerationEnded, ({ body }) => {
|
||||
if (!body)
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
// NOTICE:
|
||||
// Browser test files run in parallel as separate documents of one origin, so they share `localStorage`.
|
||||
// A browser fires a trusted `storage` event in every other document when one document writes. VueUse storage refs
|
||||
// listen to it, so a setting that one test file writes, such as the microphone switch, changed stores in another
|
||||
// file while its test ran. Tests then failed only when a certain other file ran at the same time.
|
||||
// This listener drops the trusted events. A test that dispatches its own `StorageEvent` still reaches its listeners,
|
||||
// because a dispatched event is not trusted.
|
||||
// Remove this when Vitest gives each browser test file its own storage.
|
||||
window.addEventListener('storage', (event) => {
|
||||
if (event.isTrusted)
|
||||
event.stopImmediatePropagation()
|
||||
}, { capture: true })
|
||||
@@ -71,6 +71,7 @@ export default defineConfig({
|
||||
name: 'browser',
|
||||
include: ['src/**/*.browser.test.ts'],
|
||||
exclude: ['**/node_modules/**'],
|
||||
setupFiles: ['./vitest.browser.setup.ts'],
|
||||
browser: {
|
||||
enabled: true,
|
||||
headless: true,
|
||||
|
||||
Reference in New Issue
Block a user