mirror of
https://github.com/moeru-ai/airi.git
synced 2026-10-29 18:48:55 +08:00
feat(stage-ui,stage-tamagotchi): trace spoken turns from the wake word to playback (#2939)
This commit is contained in:
@@ -20,7 +20,21 @@ The first path is the packaged app on macOS. The second is `pnpm dev` on macOS.
|
||||
|
||||
## Record shape
|
||||
|
||||
Each line is one span with `traceId`, `spanId`, `parentSpanId`, `name`, `startTimeNano`, `endTimeNano`, `status.code` (2 means error), `attributes`, and `events`. Time values are decimal strings in nanoseconds. A trace is one interaction turn. Subsystems are in `attributes["ai.moeru.airi.io.subsystem"]`.
|
||||
Each line is one span with `traceId`, `spanId`, `parentSpanId`, `name`, `startTimeNano`, `endTimeNano`, `status.code` (2 means error), `attributes`, and `events`. Time values are decimal strings in nanoseconds. Subsystems are in `attributes["ai.moeru.airi.io.subsystem"]`.
|
||||
|
||||
A trace is one turn. A typed turn starts at `Interaction turn`. A spoken turn starts earlier, and one trace covers the whole path:
|
||||
|
||||
```text
|
||||
Wake word (kws) only when a wake word started the input
|
||||
└── Voice input (voice-input) trigger: control, speech, or wake-word. outcome: committed, drafted, cancelled, or failed
|
||||
├── Voice activity detection one span for each speech segment
|
||||
├── Speech recognition (asr)
|
||||
├── End of speech judgement one span for each pause that the chat model judged
|
||||
├── Transcript rephrase
|
||||
└── Interaction turn LLM inference, streaming control, speech turn, TTS synthesis, audio playback
|
||||
```
|
||||
|
||||
A span ends before its parent can end, so a file can hold children whose parent is missing. The parent is still open, or the app stopped first. A `Voice input` with the outcome `cancelled` or `drafted` has no `Interaction turn`. Attribute `ai.moeru.airi.io.cancelled` marks work that the caller aborted, such as a judgement after the speaker continued.
|
||||
|
||||
## Common queries
|
||||
|
||||
@@ -36,6 +50,15 @@ jq -c 'select(.status.code == 2) | {name, traceId, message: .status.message}' "$
|
||||
# All spans of one trace in start order
|
||||
jq -c 'select(.traceId == "TRACE_ID")' "$F" | jq -s 'sort_by(.startTimeNano|tonumber)'
|
||||
|
||||
# Spoken turns: trigger, outcome, and how long the input took
|
||||
jq -c 'select(.name == "Voice input") | {traceId, trigger: .attributes["ai.moeru.airi.io.voice_input.trigger"], outcome: .attributes["ai.moeru.airi.io.voice_input.outcome"], reason: .attributes["ai.moeru.airi.io.voice_input.reason"], ms: (((.endTimeNano|tonumber)-(.startTimeNano|tonumber))/1e6)}' "$F"
|
||||
|
||||
# Wake words that started an input, with the matched tokens
|
||||
jq -c 'select(.name == "Wake word") | {traceId, text: .attributes["ai.moeru.airi.io.kws.text"], tokens: .attributes["ai.moeru.airi.io.kws.tokens"]}' "$F"
|
||||
|
||||
# End of speech judgements, and judgements that a newer utterance cancelled
|
||||
jq -c 'select(.name == "End of speech judgement") | {traceId, judgement: .attributes["ai.moeru.airi.io.end_judge.judgement"], cancelled: .attributes["ai.moeru.airi.io.cancelled"]}' "$F"
|
||||
|
||||
# Time to first token for each LLM span
|
||||
jq -c 'select(.events | any(.name | endswith("llm.first_token"))) | {traceId, events}' "$F"
|
||||
```
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import type { InvocableEventContext } from '@moeru/eventa'
|
||||
import type { BrowserWindow } from 'electron'
|
||||
|
||||
import type { IOTraceRecordingService } from '.'
|
||||
|
||||
@@ -14,11 +15,21 @@ import {
|
||||
export function registerIOTraceRecording(
|
||||
context: InvocableEventContext<any, any>,
|
||||
service: IOTraceRecordingService,
|
||||
window: BrowserWindow,
|
||||
): () => void {
|
||||
const disposers = [
|
||||
defineInvokeHandler(context, ioTraceRecordingGet, () => service.getState()),
|
||||
defineInvokeHandler(context, ioTraceRecordingSetEnabled, ({ enabled }) => service.setEnabled(enabled)),
|
||||
defineInvokeHandler(context, ioTraceRecordingRecordSpan, span => service.recordSpan(span)),
|
||||
// NOTICE:
|
||||
// Every window context listens on the shared `ipcMain`, so each registered window also receives the spans of
|
||||
// the other windows. Without this check, an open settings window writes each span of the main window twice.
|
||||
// Get and set stay unchecked: they are idempotent, and a handler that returns early would answer with undefined.
|
||||
// Source: the eventa TODO about window-namespaced contexts in main/windows/*/rpc/index.electron.ts.
|
||||
// Remove this check when eventa dispatches invokes only to the context of the sender window.
|
||||
defineInvokeHandler(context, ioTraceRecordingRecordSpan, (span, options) => {
|
||||
if (options?.raw.ipcMainEvent.sender.id === window.webContents.id)
|
||||
return service.recordSpan(span)
|
||||
}),
|
||||
service.onStateChange((state) => {
|
||||
void context.emit(ioTraceRecordingChanged, state)
|
||||
}),
|
||||
|
||||
@@ -72,7 +72,7 @@ export async function setupMainWindowElectronInvokes(params: {
|
||||
createGodotStageService({ context, manager: params.godotStageManager, window: params.window })
|
||||
createOnboardingService({ context, onboardingWindowManager: params.onboardingWindowManager, mainWindow: params.window })
|
||||
createAuthService({ context, window: params.window })
|
||||
const stopIOTraceRecording = registerIOTraceRecording(context, params.ioTraceRecording)
|
||||
const stopIOTraceRecording = registerIOTraceRecording(context, params.ioTraceRecording, params.window)
|
||||
params.window.once('closed', stopIOTraceRecording)
|
||||
|
||||
defineInvokeHandler(context, electronCenterMainWindow, () => centerWindowOnDisplay(params.window))
|
||||
|
||||
@@ -59,7 +59,7 @@ export async function setupSettingsWindowInvokes(params: {
|
||||
createMcpServersService({ context, manager: params.mcpManager })
|
||||
createGodotStageService({ context, manager: params.godotStageManager, window: params.settingsWindow })
|
||||
createAuthService({ context, window: params.settingsWindow })
|
||||
const stopIOTraceRecording = registerIOTraceRecording(context, params.ioTraceRecording)
|
||||
const stopIOTraceRecording = registerIOTraceRecording(context, params.ioTraceRecording, params.settingsWindow)
|
||||
params.settingsWindow.once('closed', stopIOTraceRecording)
|
||||
|
||||
// Register the global shortcut service for the settings window.
|
||||
|
||||
@@ -7,6 +7,7 @@ import { computed, ref, watch } from 'vue'
|
||||
|
||||
import {
|
||||
GAP_WARN_THRESHOLD_MS,
|
||||
INPUT_SUBSYSTEMS,
|
||||
LABEL_COL_WIDTH,
|
||||
MINIMAP_HEIGHT,
|
||||
ROW_HEIGHT,
|
||||
@@ -147,9 +148,10 @@ const layout = computed(() => {
|
||||
}
|
||||
isFirstTurn = false
|
||||
|
||||
const inputSpans = turnSpans.filter(s => INPUT_SUBSYSTEMS.has(s.subsystem)).sort((a, b) => a.startTs - b.startTs)
|
||||
const llmSpans = turnSpans.filter(s => s.subsystem === IOSubsystems.LLM).sort((a, b) => a.startTs - b.startTs)
|
||||
const auxiliarySpans = turnSpans
|
||||
.filter(s => s.subsystem !== IOSubsystems.LLM && !ttsSubsystems.has(s.subsystem))
|
||||
.filter(s => s.subsystem !== IOSubsystems.LLM && !ttsSubsystems.has(s.subsystem) && !INPUT_SUBSYSTEMS.has(s.subsystem))
|
||||
.sort((a, b) => a.startTs - b.startTs)
|
||||
const ttsSpanList = turnSpans.filter(s => ttsSubsystems.has(s.subsystem))
|
||||
|
||||
@@ -169,6 +171,11 @@ const layout = computed(() => {
|
||||
const sortedSegments = [...segmentGroups.values()]
|
||||
.sort((a, b) => a[0].startTs - b[0].startTs)
|
||||
|
||||
for (const span of inputSpans) {
|
||||
rows.push({ type: 'span', span, turn, subsystem: span.subsystem, y })
|
||||
y += ROW_HEIGHT
|
||||
}
|
||||
|
||||
for (const span of llmSpans) {
|
||||
rows.push({ type: 'span', span, turn, subsystem: IOSubsystems.LLM, y })
|
||||
y += ROW_HEIGHT
|
||||
|
||||
@@ -11,7 +11,12 @@ export interface SubsystemConfig {
|
||||
}
|
||||
|
||||
export const SUBSYSTEM_CONFIGS: SubsystemConfig[] = [
|
||||
{ subsystem: IOSubsystems.KWS, label: 'Wake Word', color: '#ec4899', bgColor: '#ec489918', icon: 'i-lucide:bell-ring' },
|
||||
{ subsystem: IOSubsystems.VoiceInput, label: 'Voice Input', color: '#6366f1', bgColor: '#6366f118', icon: 'i-lucide:audio-waveform' },
|
||||
{ subsystem: IOSubsystems.VAD, label: 'VAD', color: '#f59e0b', bgColor: '#f59e0b18', icon: 'i-lucide:activity' },
|
||||
{ subsystem: IOSubsystems.ASR, label: 'ASR', color: '#3b82f6', bgColor: '#3b82f618', icon: 'i-lucide:mic' },
|
||||
{ subsystem: IOSubsystems.EndJudge, label: 'End Judge', color: '#f97316', bgColor: '#f9731618', icon: 'i-lucide:scale' },
|
||||
{ subsystem: IOSubsystems.Rephrase, label: 'Rephrase', color: '#14b8a6', bgColor: '#14b8a618', icon: 'i-lucide:pen-line' },
|
||||
{ subsystem: IOSubsystems.LLM, label: 'LLM', color: '#a855f7', bgColor: '#a855f718', icon: 'i-lucide:brain' },
|
||||
{ subsystem: IOSubsystems.StreamingControl, label: 'Streaming Control', color: '#06b6d4', bgColor: '#06b6d418', icon: 'i-lucide:radio-tower' },
|
||||
{ subsystem: IOSubsystems.TTS, label: 'TTS', color: '#22c55e', bgColor: '#22c55e18', icon: 'i-lucide:audio-lines' },
|
||||
@@ -20,6 +25,16 @@ export const SUBSYSTEM_CONFIGS: SubsystemConfig[] = [
|
||||
|
||||
export const SUBSYSTEM_CONFIG_MAP = new Map(SUBSYSTEM_CONFIGS.map(c => [c.subsystem, c]))
|
||||
|
||||
/** Subsystems that run before the LLM in a spoken turn. The chart draws them above the LLM rows, in start order. */
|
||||
export const INPUT_SUBSYSTEMS: ReadonlySet<IOSubsystem> = new Set([
|
||||
IOSubsystems.KWS,
|
||||
IOSubsystems.VoiceInput,
|
||||
IOSubsystems.VAD,
|
||||
IOSubsystems.ASR,
|
||||
IOSubsystems.EndJudge,
|
||||
IOSubsystems.Rephrase,
|
||||
])
|
||||
|
||||
/** Height of one span row in pixels */
|
||||
export const ROW_HEIGHT = 28
|
||||
/** Height of subsystem group header */
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
export const IOSubsystems = {
|
||||
VoiceInput: 'voice-input',
|
||||
KWS: 'kws',
|
||||
VAD: 'vad',
|
||||
ASR: 'asr',
|
||||
EndJudge: 'end-judge',
|
||||
Rephrase: 'rephrase',
|
||||
LLM: 'llm',
|
||||
StreamingControl: 'streaming-control',
|
||||
TTS: 'tts',
|
||||
@@ -8,7 +12,19 @@ export const IOSubsystems = {
|
||||
} as const
|
||||
export type IOSubsystem = (typeof IOSubsystems)[keyof typeof IOSubsystems]
|
||||
|
||||
/**
|
||||
* Span names of the voice and chat pipeline.
|
||||
*
|
||||
* A spoken turn is one trace: `Wake word` (when a wake started the input) or `Voice input` is the root.
|
||||
* `Voice input` holds VAD segments, speech recognition, end judgements, and the rephrase.
|
||||
* The `Interaction turn` that the input submits is its child, and it holds the LLM, TTS, and playback spans.
|
||||
* A typed turn starts its own trace at `Interaction turn`.
|
||||
*/
|
||||
export const IOSpanNames = {
|
||||
WakeWord: 'Wake word',
|
||||
VoiceInput: 'Voice input',
|
||||
EndOfSpeechJudgement: 'End of speech judgement',
|
||||
TranscriptRephrase: 'Transcript rephrase',
|
||||
InteractionTurn: 'Interaction turn',
|
||||
SpeechTurn: 'Speech turn',
|
||||
VoiceActivityDetection: 'Voice activity detection',
|
||||
@@ -29,6 +45,19 @@ export const IOAttributes = {
|
||||
Subsystem: `${customPrefix}.subsystem`,
|
||||
TooltipKeys: `${customPrefix}.tooltip.keys`,
|
||||
LLM_TTFT: `${customPrefix}.llm.time_to_first_token`,
|
||||
VoiceInputId: `${customPrefix}.voice_input.id`,
|
||||
/** `control` (a voice button or Push to Talk), `speech` (VAD), or `wake-word`. */
|
||||
VoiceInputTrigger: `${customPrefix}.voice_input.trigger`,
|
||||
/** `committed`, `drafted`, `cancelled`, or `failed`. */
|
||||
VoiceInputOutcome: `${customPrefix}.voice_input.outcome`,
|
||||
VoiceInputReason: `${customPrefix}.voice_input.reason`,
|
||||
WakeCharacterId: `${customPrefix}.kws.character_id`,
|
||||
WakeText: `${customPrefix}.kws.text`,
|
||||
WakeTokens: `${customPrefix}.kws.tokens`,
|
||||
EndJudgement: `${customPrefix}.end_judge.judgement`,
|
||||
EndJudgeTranscriptLength: `${customPrefix}.end_judge.transcript_length`,
|
||||
RephraseSegmentCount: `${customPrefix}.rephrase.segment_count`,
|
||||
RephraseChangedCount: `${customPrefix}.rephrase.changed_count`,
|
||||
VADAudioDurationMs: `${customPrefix}.vad.audio_duration_ms`,
|
||||
VADAborted: `${customPrefix}.vad.aborted`,
|
||||
ASRText: `${customPrefix}.asr.text`,
|
||||
@@ -58,6 +87,8 @@ export const IOAttributes = {
|
||||
TTSInterruptReason: `${customPrefix}.tts.interrupt_reason`,
|
||||
TTSCanceled: `${customPrefix}.tts.canceled`,
|
||||
TurnId: `${customPrefix}.turn_id`,
|
||||
/** The caller aborted the work. The span has no error status then. */
|
||||
Cancelled: `${customPrefix}.cancelled`,
|
||||
} as const
|
||||
|
||||
export const IOEvents = {
|
||||
|
||||
@@ -204,6 +204,21 @@ Tool and content-array compatibility failures belong to the model of the failing
|
||||
Session resets and autonomous artistry hooks also use the session character.
|
||||
Each concurrent turn owns its IO trace. `activeTurnSpan` points to the most recently started turn that is still running.
|
||||
|
||||
A spoken turn is one IO trace, from the wake word to playback:
|
||||
|
||||
- `Wake word` (KWS) is the root when a wake started the input. It covers the match and the session selection.
|
||||
- `Voice input` is the root otherwise. It records the trigger (`control`, `speech`, or `wake-word`) and the outcome.
|
||||
`activeVoiceInputSpan` points to it while the input runs.
|
||||
- Its children are `Voice activity detection` (one span for each speech segment), `Speech recognition`,
|
||||
`End of speech judgement`, and `Transcript rephrase`.
|
||||
- The `Interaction turn` of the submission is also its child. The input ID is the turn ID, and `takeVoiceTurnParent`
|
||||
connects them. The turn holds the LLM, streaming control, TTS, and playback spans.
|
||||
- A composer dictation returns its text to the composer, which sends it later under a new ID. The next turn of the
|
||||
same session within 60 seconds continues the dictation trace through `takeComposerVoiceParent`. This link is by
|
||||
session, so a typed turn that replaces discarded dictated text in that time also joins the dictation trace.
|
||||
|
||||
A typed turn starts its trace at `Interaction turn`.
|
||||
|
||||
## Button analytics
|
||||
|
||||
Register the shared plugin once in each Vue application:
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
import type { Span, SpanContext, SpanStatusCode } from '@opentelemetry/api'
|
||||
import type { Span, SpanContext } from '@opentelemetry/api'
|
||||
import type { ReadableSpan, SpanExporter } from '@opentelemetry/sdk-trace-base'
|
||||
import type { SerializedIOSpan } from '@proj-airi/stage-shared/types/io-trace'
|
||||
|
||||
import { context, trace } from '@opentelemetry/api'
|
||||
import { errorMessageFrom } from '@moeru/std'
|
||||
import { context, SpanStatusCode, trace } from '@opentelemetry/api'
|
||||
import { hrTimeToNanoseconds } from '@opentelemetry/core'
|
||||
import { BasicTracerProvider, SimpleSpanProcessor } from '@opentelemetry/sdk-trace-base'
|
||||
import { IOAttributes } from '@proj-airi/stage-shared'
|
||||
import { shallowRef } from 'vue'
|
||||
|
||||
export type { ReadableSpan } from '@opentelemetry/sdk-trace-base'
|
||||
@@ -153,3 +155,82 @@ export function startSpan(name: string, parent?: Span, attrs?: Record<string, bo
|
||||
}
|
||||
|
||||
export const activeTurnSpan = shallowRef<Span | undefined>()
|
||||
|
||||
/**
|
||||
* Runs `work` and ends `span` when it settles.
|
||||
*
|
||||
* A rejection after `signal` aborted is a cancellation that the caller chose, so the span gets the `Cancelled`
|
||||
* attribute and no error status. Any other rejection sets the error status. The rejection still reaches the caller.
|
||||
*/
|
||||
export async function runInSpan<T>(span: Span, signal: AbortSignal, work: () => Promise<T>): Promise<T> {
|
||||
try {
|
||||
return await work()
|
||||
}
|
||||
catch (error) {
|
||||
if (signal.aborted)
|
||||
span.setAttribute(IOAttributes.Cancelled, true)
|
||||
else
|
||||
span.setStatus({ code: SpanStatusCode.ERROR, message: errorMessageFrom(error) ?? 'Operation failed' })
|
||||
throw error
|
||||
}
|
||||
finally {
|
||||
span.end()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The span of the speech input that the voice host captures now.
|
||||
* Speech recognition, end judgements, and the rephrase of that input are its children.
|
||||
* The voice host sets it when an input starts and clears it when the input settles.
|
||||
*/
|
||||
export const activeVoiceInputSpan = shallowRef<Span | undefined>()
|
||||
|
||||
/**
|
||||
* Voice input spans by the chat turn ID that their submission uses.
|
||||
* A submission uses the input ID as the message ID, and the chat runtime uses the message ID as the turn ID.
|
||||
* The chat store takes the parent when it starts the `Interaction turn` span, so a spoken turn is one trace.
|
||||
*/
|
||||
const voiceTurnParents = new Map<string, Span>()
|
||||
|
||||
/** Recent submissions only. A draft that is never sent leaves its entry until newer inputs replace it. */
|
||||
const MAX_VOICE_TURN_PARENTS = 32
|
||||
|
||||
export function linkVoiceTurn(turnId: string, span: Span) {
|
||||
voiceTurnParents.delete(turnId)
|
||||
voiceTurnParents.set(turnId, span)
|
||||
while (voiceTurnParents.size > MAX_VOICE_TURN_PARENTS)
|
||||
voiceTurnParents.delete(voiceTurnParents.keys().next().value!)
|
||||
}
|
||||
|
||||
/** Returns the voice input span of a turn once. A typed turn has none. */
|
||||
export function takeVoiceTurnParent(turnId: string): Span | undefined {
|
||||
const span = voiceTurnParents.get(turnId)
|
||||
voiceTurnParents.delete(turnId)
|
||||
return span
|
||||
}
|
||||
|
||||
/**
|
||||
* Time in which the next turn of a session can continue the trace of a composer dictation.
|
||||
* A dictation returns text to the composer, and the composer sends it later as a new message with a new turn ID.
|
||||
*/
|
||||
const COMPOSER_VOICE_LINK_MS = 60_000
|
||||
|
||||
/**
|
||||
* The newest finished composer dictation of each session.
|
||||
*
|
||||
* The link is by session, not by message, so it is a heuristic: the next turn of the session within
|
||||
* {@link COMPOSER_VOICE_LINK_MS} takes it once. If the user discards the dictated text and sends typed text in that
|
||||
* time, the typed turn also joins the dictation trace. The composer does not carry the input ID to its send.
|
||||
*/
|
||||
const composerVoiceParents = new Map<string, { span: Span, finishedAt: number }>()
|
||||
|
||||
export function linkComposerVoice(sessionId: string, span: Span) {
|
||||
composerVoiceParents.set(sessionId, { span, finishedAt: Date.now() })
|
||||
}
|
||||
|
||||
/** Returns the composer dictation span of a session once, if it finished recently. */
|
||||
export function takeComposerVoiceParent(sessionId: string): Span | undefined {
|
||||
const link = composerVoiceParents.get(sessionId)
|
||||
composerVoiceParents.delete(sessionId)
|
||||
return link && Date.now() - link.finishedAt < COMPOSER_VOICE_LINK_MS ? link.span : undefined
|
||||
}
|
||||
|
||||
@@ -2,12 +2,14 @@ import type { Conversation, TurnEvidence } from '@proj-airi/core-agent'
|
||||
|
||||
import type { EndJudgement } from '../libs/voice/voice-end-judge-plugin'
|
||||
|
||||
import { IOAttributes, IOSpanNames, IOSubsystems } from '@proj-airi/stage-shared'
|
||||
import { boolean, object, parse } from 'valibot'
|
||||
|
||||
import { useLLM } from '../stores/ai/chat-llm/llm'
|
||||
import { useConsciousnessStore } from '../stores/modules/consciousness'
|
||||
import { useHearingStore } from '../stores/modules/hearing'
|
||||
import { useProviderStore } from '../stores/providers/provider'
|
||||
import { activeVoiceInputSpan, runInSpan, startSpan } from './use-io-tracer'
|
||||
|
||||
const END_JUDGE_INSTRUCTIONS = [
|
||||
'You decide if a speaker finished a spoken message, from the speech recognition text of it so far.',
|
||||
@@ -54,6 +56,20 @@ export function useVoiceEndJudge() {
|
||||
if (!providerId || !model)
|
||||
throw new Error('No chat model is configured for voice end judgement')
|
||||
|
||||
const span = startSpan(IOSpanNames.EndOfSpeechJudgement, activeVoiceInputSpan.value, {
|
||||
[IOAttributes.Subsystem]: IOSubsystems.EndJudge,
|
||||
[IOAttributes.GenAIRequestModel]: model,
|
||||
[IOAttributes.GenAIProviderName]: providerId,
|
||||
[IOAttributes.EndJudgeTranscriptLength]: transcript.length,
|
||||
})
|
||||
return runInSpan(span, signal, async () => {
|
||||
const judgement = await requestJudgement(providerId, model, transcript, context, signal)
|
||||
span.setAttribute(IOAttributes.EndJudgement, judgement)
|
||||
return judgement
|
||||
})
|
||||
}
|
||||
|
||||
async function requestJudgement(providerId: string, model: string, transcript: string, context: TurnEvidence['context'], signal: AbortSignal) {
|
||||
const provider = await providers.getChatProviderInstance(providerId)
|
||||
const conversation: Conversation = { turns: [
|
||||
{ id: 'voice-end-judge-instructions', type: 'system', authority: 'system', content: [{ type: 'text', text: END_JUDGE_INSTRUCTIONS }] },
|
||||
|
||||
@@ -0,0 +1,167 @@
|
||||
import type { SpeechInputAttemptOutcome } from '@proj-airi/core-agent'
|
||||
|
||||
import type { ReadableSpan } from './use-io-tracer'
|
||||
|
||||
import { SpanStatusCode } from '@opentelemetry/api'
|
||||
import { IOAttributes, IOSpanNames } from '@proj-airi/stage-shared'
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
import { activeVoiceInputSpan, onIOSpan, runInSpan, startSpan, takeComposerVoiceParent, takeVoiceTurnParent } from './use-io-tracer'
|
||||
import { createVoiceInputTracer } from './voice-input-trace'
|
||||
|
||||
const spans: ReadableSpan[] = []
|
||||
onIOSpan(span => spans.push(span))
|
||||
|
||||
afterEach(() => {
|
||||
spans.length = 0
|
||||
})
|
||||
|
||||
function spansNamed(name: string) {
|
||||
return spans.filter(span => span.name === name)
|
||||
}
|
||||
|
||||
/** An input whose outcome the test settles. Only the ID and the outcome reach the tracer. */
|
||||
function attempt(id: string) {
|
||||
const done = Promise.withResolvers<SpeechInputAttemptOutcome>()
|
||||
return { id, done: done.promise, settle: done.resolve }
|
||||
}
|
||||
|
||||
describe('createVoiceInputTracer', () => {
|
||||
it('traces an input from start to outcome and makes it the parent of its recognition', async () => {
|
||||
const tracer = createVoiceInputTracer()
|
||||
const input = attempt('input-1')
|
||||
|
||||
tracer.start(input, 'speech')
|
||||
const recognition = startSpan(IOSpanNames.SpeechRecognition, activeVoiceInputSpan.value)
|
||||
recognition.end()
|
||||
input.settle({ status: 'committed', messageId: 'input-1' })
|
||||
|
||||
await vi.waitFor(() => expect(spansNamed(IOSpanNames.VoiceInput)).toHaveLength(1))
|
||||
const span = spansNamed(IOSpanNames.VoiceInput)[0]!
|
||||
expect(span.attributes[IOAttributes.VoiceInputTrigger]).toBe('speech')
|
||||
expect(span.attributes[IOAttributes.VoiceInputOutcome]).toBe('committed')
|
||||
expect(spansNamed(IOSpanNames.SpeechRecognition)[0]!.parentSpanContext?.spanId).toBe(span.spanContext().spanId)
|
||||
expect(activeVoiceInputSpan.value).toBeUndefined()
|
||||
})
|
||||
|
||||
it('continues the trace of the wake that started the input', async () => {
|
||||
const tracer = createVoiceInputTracer()
|
||||
const wake = startSpan(IOSpanNames.WakeWord)
|
||||
wake.end()
|
||||
const input = attempt('input-2')
|
||||
|
||||
tracer.start(input, 'wake-word', wake)
|
||||
input.settle({ status: 'cancelled', reason: 'Speech segment was too short' })
|
||||
|
||||
await vi.waitFor(() => expect(spansNamed(IOSpanNames.VoiceInput)).toHaveLength(1))
|
||||
const span = spansNamed(IOSpanNames.VoiceInput)[0]!
|
||||
expect(span.spanContext().traceId).toBe(wake.spanContext().traceId)
|
||||
expect(span.parentSpanContext?.spanId).toBe(wake.spanContext().spanId)
|
||||
expect(span.attributes[IOAttributes.VoiceInputReason]).toBe('Speech segment was too short')
|
||||
})
|
||||
|
||||
it('links the chat turn of the submission, and of a draft that the input merged into', async () => {
|
||||
const tracer = createVoiceInputTracer()
|
||||
const input = attempt('input-3')
|
||||
|
||||
tracer.start(input, 'control')
|
||||
const span = activeVoiceInputSpan.value
|
||||
input.settle({ status: 'drafted', draftId: 'earlier-draft' })
|
||||
await vi.waitFor(() => expect(spansNamed(IOSpanNames.VoiceInput)).toHaveLength(1))
|
||||
|
||||
expect(takeVoiceTurnParent('input-3')).toBe(span)
|
||||
expect(takeVoiceTurnParent('earlier-draft')).toBe(span)
|
||||
// A turn takes its parent once.
|
||||
expect(takeVoiceTurnParent('earlier-draft')).toBeUndefined()
|
||||
})
|
||||
|
||||
// Codex review on PR #2939: the composer sends dictated text under a new message ID, so the input ID never matches.
|
||||
it('links a composer dictation to the next turn of its session, once and only for a short time', async () => {
|
||||
const tracer = createVoiceInputTracer()
|
||||
const first = attempt('dictation-1')
|
||||
tracer.start(first, 'control')
|
||||
const span = activeVoiceInputSpan.value
|
||||
tracer.markComposer('dictation-1', 'alice')
|
||||
first.settle({ status: 'drafted', draftId: 'dictation-1' })
|
||||
await vi.waitFor(() => expect(spansNamed(IOSpanNames.VoiceInput)).toHaveLength(1))
|
||||
|
||||
expect(takeComposerVoiceParent('bob')).toBeUndefined()
|
||||
expect(takeComposerVoiceParent('alice')).toBe(span)
|
||||
expect(takeComposerVoiceParent('alice')).toBeUndefined()
|
||||
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const second = attempt('dictation-2')
|
||||
tracer.start(second, 'control')
|
||||
tracer.markComposer('dictation-2', 'alice')
|
||||
second.settle({ status: 'drafted', draftId: 'dictation-2' })
|
||||
await vi.waitFor(() => expect(spansNamed(IOSpanNames.VoiceInput)).toHaveLength(2))
|
||||
vi.advanceTimersByTime(61_000)
|
||||
|
||||
expect(takeComposerVoiceParent('alice')).toBeUndefined()
|
||||
}
|
||||
finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('records one VAD span for each speech segment, and closes an open segment with the input', async () => {
|
||||
const tracer = createVoiceInputTracer()
|
||||
const input = attempt('input-4')
|
||||
tracer.start(input, 'speech')
|
||||
const window = (speech: boolean) => tracer.activity({ inputId: 'input-4', speech, durationMs: 100 })
|
||||
|
||||
// A short breath stays in one segment. A 300 ms pause closes it.
|
||||
window(true)
|
||||
window(true)
|
||||
window(false)
|
||||
window(true)
|
||||
window(false)
|
||||
window(false)
|
||||
window(false)
|
||||
window(true)
|
||||
input.settle({ status: 'committed', messageId: 'input-4' })
|
||||
|
||||
await vi.waitFor(() => expect(spansNamed(IOSpanNames.VoiceInput)).toHaveLength(1))
|
||||
const segments = spansNamed(IOSpanNames.VoiceActivityDetection)
|
||||
expect(segments.map(segment => segment.attributes[IOAttributes.VADAudioDurationMs])).toEqual([300, 100])
|
||||
expect(segments.every(segment => segment.parentSpanContext?.spanId === spansNamed(IOSpanNames.VoiceInput)[0]!.spanContext().spanId)).toBe(true)
|
||||
})
|
||||
|
||||
it('marks a failed input as an error', async () => {
|
||||
const tracer = createVoiceInputTracer()
|
||||
const input = attempt('input-5')
|
||||
|
||||
tracer.start(input, 'speech')
|
||||
input.settle({ status: 'failed', stage: 'transcription', error: new Error('provider down') })
|
||||
|
||||
await vi.waitFor(() => expect(spansNamed(IOSpanNames.VoiceInput)).toHaveLength(1))
|
||||
const span = spansNamed(IOSpanNames.VoiceInput)[0]!
|
||||
expect(span.status).toEqual({ code: SpanStatusCode.ERROR, message: 'provider down' })
|
||||
expect(span.attributes[IOAttributes.VoiceInputReason]).toBe('transcription')
|
||||
})
|
||||
})
|
||||
|
||||
describe('runInSpan', () => {
|
||||
it('ends the span with the result, an error status, or a cancellation', async () => {
|
||||
const ok = startSpan('ok')
|
||||
const failed = startSpan('failed')
|
||||
const cancelled = startSpan('cancelled')
|
||||
const abort = new AbortController()
|
||||
|
||||
await expect(runInSpan(ok, abort.signal, async () => 'done')).resolves.toBe('done')
|
||||
await expect(runInSpan(failed, abort.signal, async () => {
|
||||
throw new Error('model unavailable')
|
||||
})).rejects.toThrow('model unavailable')
|
||||
abort.abort('Speech evidence changed')
|
||||
await expect(runInSpan(cancelled, abort.signal, async () => {
|
||||
throw abort.signal.reason
|
||||
})).rejects.toBe('Speech evidence changed')
|
||||
|
||||
await vi.waitFor(() => expect(spans).toHaveLength(3))
|
||||
expect(spansNamed('ok')[0]!.status.code).toBe(SpanStatusCode.UNSET)
|
||||
expect(spansNamed('failed')[0]!.status).toEqual({ code: SpanStatusCode.ERROR, message: 'model unavailable' })
|
||||
expect(spansNamed('cancelled')[0]!.status.code).toBe(SpanStatusCode.UNSET)
|
||||
expect(spansNamed('cancelled')[0]!.attributes[IOAttributes.Cancelled]).toBe(true)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,133 @@
|
||||
import type { Span } from '@opentelemetry/api'
|
||||
import type { SpeechInputAttempt, SpeechInputAttemptOutcome } from '@proj-airi/core-agent'
|
||||
|
||||
import { SpanStatusCode } from '@opentelemetry/api'
|
||||
import { IOAttributes, IOSpanNames, IOSubsystems } from '@proj-airi/stage-shared'
|
||||
|
||||
import { activeVoiceInputSpan, linkComposerVoice, linkVoiceTurn, startSpan } from './use-io-tracer'
|
||||
|
||||
/**
|
||||
* What started a speech input.
|
||||
*
|
||||
* - `control`: a voice button, the inlay, or the Push to Talk key.
|
||||
* - `speech`: voice activity detection found speech.
|
||||
* - `wake-word`: a wake word selected a character.
|
||||
*/
|
||||
export type VoiceInputTrigger = 'control' | 'speech' | 'wake-word'
|
||||
|
||||
/**
|
||||
* Non-speech that ends a traced speech segment. Shorter pauses are breaths inside one segment.
|
||||
* The value only shapes the trace. It does not affect when an input ends.
|
||||
*/
|
||||
const SEGMENT_PAUSE_MS = 300
|
||||
|
||||
interface TracedInput {
|
||||
readonly span: Span
|
||||
segment?: { span: Span, speechMs: number, silenceMs: number, lastSpeechAt: number }
|
||||
/** The session of a composer dictation. Its text returns to the composer instead of a draft. */
|
||||
composerSessionId?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Traces the speech inputs of one voice host.
|
||||
*
|
||||
* Each input gets a `Voice input` span. It is the trace root, or the child of the `Wake word` span that started it.
|
||||
* While the input runs, `activeVoiceInputSpan` points to it, so speech recognition, end judgements, and the rephrase
|
||||
* become its children. VAD results become `Voice activity detection` child spans, one for each speech segment.
|
||||
* The input ID is also the turn ID of the chat turn that its submission starts, so that turn continues the trace.
|
||||
*
|
||||
* State: `inputs` holds the open inputs by ID. The input outcome ends the input span and its open segment.
|
||||
*/
|
||||
export function createVoiceInputTracer() {
|
||||
const inputs = new Map<string, TracedInput>()
|
||||
|
||||
/** Starts the trace of an input that the controller has just begun. */
|
||||
function start(attempt: Pick<SpeechInputAttempt, 'id' | 'done'>, trigger: VoiceInputTrigger, parent?: Span) {
|
||||
const span = startSpan(IOSpanNames.VoiceInput, parent, {
|
||||
[IOAttributes.Subsystem]: IOSubsystems.VoiceInput,
|
||||
[IOAttributes.VoiceInputId]: attempt.id,
|
||||
[IOAttributes.VoiceInputTrigger]: trigger,
|
||||
})
|
||||
inputs.set(attempt.id, { span })
|
||||
activeVoiceInputSpan.value = span
|
||||
linkVoiceTurn(attempt.id, span)
|
||||
void attempt.done.then(outcome => settle(attempt.id, outcome))
|
||||
}
|
||||
|
||||
/**
|
||||
* Marks an input as a composer dictation. Its trace then waits for the next turn of the session, because the
|
||||
* composer sends the dictated text later under a new message ID.
|
||||
*/
|
||||
function markComposer(inputId: string, sessionId: string) {
|
||||
const input = inputs.get(inputId)
|
||||
if (input)
|
||||
input.composerSessionId = sessionId
|
||||
}
|
||||
|
||||
/**
|
||||
* Records the VAD result of one window of an input.
|
||||
* The first speech window opens a segment. A pause of `SEGMENT_PAUSE_MS` closes it at the time of its last speech window.
|
||||
*/
|
||||
function activity(event: { inputId: string, speech: boolean, durationMs: number }) {
|
||||
const input = inputs.get(event.inputId)
|
||||
if (!input)
|
||||
return
|
||||
|
||||
const now = performance.now()
|
||||
if (event.speech) {
|
||||
input.segment ??= {
|
||||
span: startSpan(IOSpanNames.VoiceActivityDetection, input.span, { [IOAttributes.Subsystem]: IOSubsystems.VAD }),
|
||||
speechMs: 0,
|
||||
silenceMs: 0,
|
||||
lastSpeechAt: now,
|
||||
}
|
||||
input.segment.speechMs += event.durationMs
|
||||
input.segment.silenceMs = 0
|
||||
input.segment.lastSpeechAt = now
|
||||
return
|
||||
}
|
||||
|
||||
if (!input.segment)
|
||||
return
|
||||
input.segment.silenceMs += event.durationMs
|
||||
if (input.segment.silenceMs >= SEGMENT_PAUSE_MS)
|
||||
closeSegment(input)
|
||||
}
|
||||
|
||||
function closeSegment(input: TracedInput) {
|
||||
const segment = input.segment
|
||||
if (!segment)
|
||||
return
|
||||
input.segment = undefined
|
||||
segment.span.setAttribute(IOAttributes.VADAudioDurationMs, Math.round(segment.speechMs))
|
||||
segment.span.end(segment.lastSpeechAt)
|
||||
}
|
||||
|
||||
function settle(inputId: string, outcome: SpeechInputAttemptOutcome) {
|
||||
const input = inputs.get(inputId)
|
||||
if (!input)
|
||||
return
|
||||
inputs.delete(inputId)
|
||||
closeSegment(input)
|
||||
|
||||
input.span.setAttribute(IOAttributes.VoiceInputOutcome, outcome.status)
|
||||
if (outcome.status === 'cancelled')
|
||||
input.span.setAttribute(IOAttributes.VoiceInputReason, outcome.reason)
|
||||
if (outcome.status === 'failed') {
|
||||
input.span.setAttribute(IOAttributes.VoiceInputReason, outcome.stage)
|
||||
input.span.setStatus({ code: SpanStatusCode.ERROR, message: outcome.error.message })
|
||||
}
|
||||
// A dictation that produced text continues in the next composer turn of its session.
|
||||
if (input.composerSessionId && outcome.status === 'drafted')
|
||||
linkComposerVoice(input.composerSessionId, input.span)
|
||||
// A draft that merged into an earlier draft is sent later under that draft's ID.
|
||||
else if (outcome.status === 'drafted' && outcome.draftId !== inputId)
|
||||
linkVoiceTurn(outcome.draftId, input.span)
|
||||
|
||||
input.span.end()
|
||||
if (activeVoiceInputSpan.value === input.span)
|
||||
activeVoiceInputSpan.value = undefined
|
||||
}
|
||||
|
||||
return { start, markComposer, activity }
|
||||
}
|
||||
@@ -1,10 +1,13 @@
|
||||
import type { Conversation } from '@proj-airi/core-agent'
|
||||
|
||||
import { IOAttributes, IOSpanNames, IOSubsystems } from '@proj-airi/stage-shared'
|
||||
|
||||
import { parseRephrasedSegments } from '../libs/voice/voice-rephrase-plugin'
|
||||
import { useLLM } from '../stores/ai/chat-llm/llm'
|
||||
import { useConsciousnessStore } from '../stores/modules/consciousness'
|
||||
import { useHearingStore } from '../stores/modules/hearing'
|
||||
import { useProviderStore } from '../stores/providers/provider'
|
||||
import { activeVoiceInputSpan, runInSpan, startSpan } from './use-io-tracer'
|
||||
|
||||
const REPHRASE_INSTRUCTIONS = [
|
||||
'You clean up text that speech recognition produced from one spoken message.',
|
||||
@@ -37,6 +40,20 @@ export function useVoiceRephrase() {
|
||||
if (!providerId || !model)
|
||||
throw new Error('No chat model is configured for voice rephrasing')
|
||||
|
||||
const span = startSpan(IOSpanNames.TranscriptRephrase, activeVoiceInputSpan.value, {
|
||||
[IOAttributes.Subsystem]: IOSubsystems.Rephrase,
|
||||
[IOAttributes.GenAIRequestModel]: model,
|
||||
[IOAttributes.GenAIProviderName]: providerId,
|
||||
[IOAttributes.RephraseSegmentCount]: segments.length,
|
||||
})
|
||||
return runInSpan(span, signal, async () => {
|
||||
const rewritten = await requestRephrase(providerId, model, segments, signal)
|
||||
span.setAttribute(IOAttributes.RephraseChangedCount, rewritten.filter((text, index) => text.trim() !== segments[index]?.trim()).length)
|
||||
return rewritten
|
||||
})
|
||||
}
|
||||
|
||||
async function requestRephrase(providerId: string, model: string, segments: readonly string[], signal: AbortSignal) {
|
||||
const provider = await providers.getChatProviderInstance(providerId)
|
||||
const conversation: Conversation = { turns: [
|
||||
{ id: 'voice-rephrase-instructions', type: 'system', authority: 'system', content: [{ type: 'text', text: REPHRASE_INSTRUCTIONS }] },
|
||||
|
||||
@@ -7,7 +7,7 @@ import { errorMessageFrom } from '@moeru/std'
|
||||
import { SpanStatusCode } from '@opentelemetry/api'
|
||||
import { IOAttributes, IOSpanNames, IOSubsystems } from '@proj-airi/stage-shared'
|
||||
|
||||
import { activeTurnSpan, startSpan } from '../../composables/use-io-tracer'
|
||||
import { activeTurnSpan, activeVoiceInputSpan, startSpan } from '../../composables/use-io-tracer'
|
||||
|
||||
/**
|
||||
* Converts provider output into complete transcript revisions. Each adapter owns its segmenters and cancellation.
|
||||
@@ -26,7 +26,9 @@ export function createHearingTranscriber(
|
||||
const signal = AbortSignal.any([request.signal, cancelled.signal])
|
||||
let reader: ReadableStreamDefaultReader<AIRIStreamTranscriptionDelta> | undefined
|
||||
let closed = false
|
||||
const span = startSpan(IOSpanNames.SpeechRecognition, activeTurnSpan.value, { [IOAttributes.Subsystem]: IOSubsystems.ASR })
|
||||
// Recognition belongs to the input that it transcribes. A transcription outside a voice input, such as a voice
|
||||
// message, attaches to the most recent turn.
|
||||
const span = startSpan(IOSpanNames.SpeechRecognition, activeVoiceInputSpan.value ?? activeTurnSpan.value, { [IOAttributes.Subsystem]: IOSubsystems.ASR })
|
||||
|
||||
return new ReadableStream<TranscriptionEvent>({
|
||||
start: (output) => {
|
||||
|
||||
@@ -15,6 +15,8 @@ it('uses ordered VAD signals to capture, stream transcription, and submit to the
|
||||
const submit = vi.fn(async () => ({ status: 'committed' as const, messageId: 'accepted' }))
|
||||
let sessionId = 'alice'
|
||||
let attempt: SpeechInputAttempt | undefined
|
||||
const activity = vi.fn()
|
||||
const causes = vi.fn()
|
||||
const controller = new VoiceController({ audio, submit, transcriber: () => ({
|
||||
transcribe: (request) => {
|
||||
const media = request.audio
|
||||
@@ -31,6 +33,8 @@ it('uses ordered VAD signals to capture, stream transcription, and submit to the
|
||||
detect: async window => window.channels[0][0],
|
||||
target: () => ({ sessionId, interruptTurns: [] }),
|
||||
onInput: () => { attempt = controller.activeInput },
|
||||
onActivity: activity,
|
||||
beforeBeginInput: causes,
|
||||
minSpeechMs: 32,
|
||||
silenceMs: 64,
|
||||
}), { grants: ['input-control', 'cancel-input'] })
|
||||
@@ -42,6 +46,13 @@ it('uses ordered VAD signals to capture, stream transcription, and submit to the
|
||||
expect(await attempt!.done).toEqual({ status: 'committed', messageId: 'accepted' })
|
||||
expect(received).toHaveLength(96)
|
||||
expect(submit).toHaveBeenCalledWith(expect.objectContaining({ sessionId: 'alice', text: 'Hello' }), expect.any(AbortSignal))
|
||||
// Each noted window reaches the host trace in order.
|
||||
expect(activity.mock.calls.map(([event]) => [event.inputId, event.speech, event.durationMs])).toEqual([
|
||||
[attempt!.id, true, 32],
|
||||
[attempt!.id, false, 32],
|
||||
[attempt!.id, false, 32],
|
||||
])
|
||||
expect(causes.mock.calls).toEqual([['speech']])
|
||||
await controller.close()
|
||||
})
|
||||
|
||||
@@ -77,10 +88,12 @@ it('rejects playback echo and opens the matched character input only after playb
|
||||
let accepted = false
|
||||
const detected = vi.fn(async () => ({ sessionId: 'alice', interruptTurns: [turn] }))
|
||||
const gate = vi.fn(async () => accepted)
|
||||
const causes = vi.fn()
|
||||
controller.use(createVoiceActivityPlugin({
|
||||
detect: async window => window.channels[0][0],
|
||||
detectWakeWord: async window => window.range.startFrame < 96 ? detected() : undefined,
|
||||
acceptSpeech: gate,
|
||||
beforeBeginInput: causes,
|
||||
target: () => ({ sessionId: 'other', interruptTurns: [] }),
|
||||
minSpeechMs: 32,
|
||||
silenceMs: 32,
|
||||
@@ -93,9 +106,12 @@ it('rejects playback echo and opens the matched character input only after playb
|
||||
await expect.poll(() => gate).toHaveBeenCalledOnce()
|
||||
expect(controller.activeInput).toBeUndefined()
|
||||
expect(fade).not.toHaveBeenCalled()
|
||||
// Codex review on PR #2939: a wake that echo rejection dropped must not report a wake-word input.
|
||||
expect(causes).not.toHaveBeenCalled()
|
||||
accepted = true
|
||||
push(32, 1)
|
||||
await expect.poll(() => fade).toHaveBeenCalledOnce()
|
||||
expect(causes.mock.calls).toEqual([['wake-word']])
|
||||
const attempt = controller.activeInput!
|
||||
expect(attempt.state).toEqual({ phase: 'pending', waitingFor: 'silence' })
|
||||
push(64, 1)
|
||||
|
||||
@@ -15,6 +15,17 @@ export interface VoiceActivityOptions {
|
||||
readonly acceptSpeech?: (window: AudioWindow, signal: AbortSignal) => Promise<boolean>
|
||||
readonly enabled?: () => boolean
|
||||
readonly onInput?: () => void
|
||||
/**
|
||||
* Runs right before the plugin begins an input, with the reason. The controller announces the input
|
||||
* synchronously inside that call, so the host knows the cause of the input that it is told about next.
|
||||
* A wake that does not lead to a begin, for example after echo rejection, never reaches this callback.
|
||||
*/
|
||||
readonly beforeBeginInput?: (cause: 'wake-word' | 'speech') => void
|
||||
/**
|
||||
* Receives the VAD result of each window that the plugin noted on its input, in window order.
|
||||
* The voice host traces speech segments with it. It never changes how the input starts or ends.
|
||||
*/
|
||||
readonly onActivity?: (activity: { inputId: string, speech: boolean, durationMs: number }) => void
|
||||
/** @default 0.6. A live input uses 30 percent of this score to keep speech active. */
|
||||
readonly threshold?: number
|
||||
/** @default 300. Shorter speech is discarded when silence ends an input. */
|
||||
@@ -74,6 +85,7 @@ export function createVoiceActivityPlugin(options: VoiceActivityOptions): VoiceP
|
||||
if (!target)
|
||||
return
|
||||
|
||||
options.beforeBeginInput?.(wake ? 'wake-word' : 'speech')
|
||||
const result = ctx.controls.beginInput({ ...target, start: wake
|
||||
? { kind: 'after-silence' }
|
||||
: { kind: 'speech-onset', at: { sourceId: ctx.window.range.sourceId, frame: ctx.window.range.startFrame }, preRollMs: options.preRollMs ?? 360 } })
|
||||
@@ -92,6 +104,7 @@ export function createVoiceActivityPlugin(options: VoiceActivityOptions): VoiceP
|
||||
|
||||
if (!input.noteActivity({ range: ctx.window.range, speech }))
|
||||
return
|
||||
options.onActivity?.({ inputId: input.id, speech, durationMs: duration })
|
||||
|
||||
speechMs += speech ? duration : 0
|
||||
silenceMs = speech ? 0 : silenceMs + duration
|
||||
|
||||
@@ -96,7 +96,7 @@ describe('wakeWordDetector', () => {
|
||||
])
|
||||
expect(harness.spotter.processAudio.mock.calls[0]![0]).toHaveLength(BATCH)
|
||||
expect(harness.spotter.processAudio.mock.calls[0]![1]).toBe(16_000)
|
||||
expect(harness.resolveTarget).toHaveBeenCalledWith('bob', expect.any(AbortSignal))
|
||||
expect(harness.resolveTarget).toHaveBeenCalledWith('bob', expect.any(AbortSignal), bob)
|
||||
expect(target).toEqual({ sessionId: 'session-bob', interruptTurns: [] })
|
||||
expect(harness.detector.take(new AbortController().signal)).toBeUndefined()
|
||||
expect(harness.onPreparationChange).toHaveBeenLastCalledWith('ready')
|
||||
|
||||
@@ -28,8 +28,11 @@ export interface WakeWordDetectorOptions {
|
||||
readonly modelId: string
|
||||
/** Creates a spotter Worker. Initialization requires at least one keyword. The detector owns and disposes the result. */
|
||||
readonly createSpotter: (keywords: readonly KeywordEntry[]) => Promise<KeywordSpotter>
|
||||
/** Selects the character session after a match. The signal aborts when the observed audio window is abandoned. */
|
||||
readonly resolveTarget: (characterId: string, signal: AbortSignal) => Promise<WakeTarget>
|
||||
/**
|
||||
* Selects the character session after a match. The signal aborts when the observed audio window is abandoned.
|
||||
* `pronunciation` is the matched catalog entry, so the caller can trace which word woke the character.
|
||||
*/
|
||||
readonly resolveTarget: (characterId: string, signal: AbortSignal, pronunciation: WakePronunciation) => Promise<WakeTarget>
|
||||
readonly onPreparationChange?: (preparation: WakeWordPreparation, error?: string) => void
|
||||
/** Receives a failure to select the target session. Recognition continues with the next window. */
|
||||
readonly onError?: (error: unknown) => void
|
||||
@@ -378,7 +381,7 @@ export class WakeWordDetector {
|
||||
private async resolveWake(pronunciation: WakePronunciation, generation: number) {
|
||||
const signal = this.lifetime.signal
|
||||
try {
|
||||
const target = await this.options.resolveTarget(pronunciation.characterId, signal)
|
||||
const target = await this.options.resolveTarget(pronunciation.characterId, signal, pronunciation)
|
||||
if (generation === this.generation && !signal.aborted)
|
||||
this.wake = target
|
||||
}
|
||||
|
||||
@@ -47,6 +47,10 @@ const ioTracerMocks = vi.hoisted(() => {
|
||||
activeTurnSpan,
|
||||
spans,
|
||||
startSpanMock,
|
||||
/** Voice input spans by turn ID. A typed turn has none. */
|
||||
takeVoiceTurnParent: vi.fn((_turnId: string): unknown => undefined),
|
||||
/** The composer dictation span of a session. */
|
||||
takeComposerVoiceParent: vi.fn((_sessionId: string): unknown => undefined),
|
||||
}
|
||||
})
|
||||
|
||||
@@ -132,6 +136,8 @@ vi.mock('../libs/product-signals', () => ({
|
||||
vi.mock('../composables/use-io-tracer', () => ({
|
||||
activeTurnSpan: ioTracerMocks.activeTurnSpan,
|
||||
startSpan: ioTracerMocks.startSpanMock,
|
||||
takeVoiceTurnParent: ioTracerMocks.takeVoiceTurnParent,
|
||||
takeComposerVoiceParent: ioTracerMocks.takeComposerVoiceParent,
|
||||
}))
|
||||
|
||||
vi.mock('./chat/context-providers', () => ({
|
||||
@@ -365,6 +371,8 @@ describe('chat store contract', () => {
|
||||
ioTracerMocks.activeTurnSpan.value = undefined
|
||||
ioTracerMocks.spans.length = 0
|
||||
ioTracerMocks.startSpanMock.mockClear()
|
||||
ioTracerMocks.takeVoiceTurnParent.mockReset()
|
||||
ioTracerMocks.takeComposerVoiceParent.mockReset()
|
||||
activeSessionIdRef.value = 'session-1'
|
||||
activeProviderRef.value = 'mock-provider'
|
||||
cardSelections.clear()
|
||||
@@ -1572,6 +1580,30 @@ describe('chat store contract', () => {
|
||||
expect(store.$state).not.toHaveProperty('activeStreamingMessage')
|
||||
})
|
||||
|
||||
it('continues the trace of the voice input that submitted the turn', async () => {
|
||||
const voiceInput = { name: IOSpanNames.VoiceInput }
|
||||
ioTracerMocks.takeVoiceTurnParent.mockReturnValueOnce(voiceInput)
|
||||
|
||||
const store = useChatStore()
|
||||
await store.ingest('spoken text', { model: 'gpt-test', chatProvider: provider })
|
||||
|
||||
const [turnId] = ioTracerMocks.takeVoiceTurnParent.mock.calls[0]!
|
||||
expect(turnId).toEqual(expect.any(String))
|
||||
expect(turnId).not.toBe('')
|
||||
expect(ioTracerMocks.startSpanMock).toHaveBeenCalledWith(IOSpanNames.InteractionTurn, voiceInput)
|
||||
})
|
||||
|
||||
it('continues a composer dictation trace when the turn has no voice input of its own', async () => {
|
||||
const dictation = { name: IOSpanNames.VoiceInput }
|
||||
ioTracerMocks.takeComposerVoiceParent.mockReturnValueOnce(dictation)
|
||||
|
||||
const store = useChatStore()
|
||||
await store.ingest('dictated text', { model: 'gpt-test', chatProvider: provider })
|
||||
|
||||
expect(ioTracerMocks.takeComposerVoiceParent).toHaveBeenCalledOnce()
|
||||
expect(ioTracerMocks.startSpanMock).toHaveBeenCalledWith(IOSpanNames.InteractionTurn, dictation)
|
||||
})
|
||||
|
||||
it('does not end the owned IO turn span when external sending mirror is cleared mid-send', async () => {
|
||||
let releaseStream: (() => void) | undefined
|
||||
llmStreamMock.mockImplementationOnce(async () => {
|
||||
|
||||
@@ -20,7 +20,7 @@ import { useI18n } from 'vue-i18n'
|
||||
|
||||
import { getConversationAnalyticsSurface } from '../composables'
|
||||
import { useAiriRuntimePrompt } from '../composables/use-airi-runtime-prompt'
|
||||
import { activeTurnSpan, startSpan } from '../composables/use-io-tracer'
|
||||
import { activeTurnSpan, startSpan, takeComposerVoiceParent, takeVoiceTurnParent } from '../composables/use-io-tracer'
|
||||
import { useChatVision } from '../composables/vision/use-chat-vision'
|
||||
import { useVisionInference } from '../composables/vision/use-vision-inference'
|
||||
import { chatAssetIdFrom, inlineConversationAssets, storeChatAttachments } from '../libs/chat-assets'
|
||||
@@ -342,10 +342,16 @@ export const useChatStore = defineStore('chat', () => {
|
||||
: undefined
|
||||
let turnSpan = turnKey ? ownedTurnSpans.get(turnKey) : activeTurnSpan.value
|
||||
if (!turnSpan && turnKey) {
|
||||
turnSpan = startSpan(IOSpanNames.InteractionTurn)
|
||||
// A spoken turn continues the trace of its voice input. A turn after a composer dictation continues that
|
||||
// dictation, because the composer sends the dictated text under a new ID. Any other turn starts a new trace.
|
||||
const correlation = options?.requestCorrelation
|
||||
const voiceParent = correlation
|
||||
? takeVoiceTurnParent(correlation.turnId) ?? takeComposerVoiceParent(correlation.conversationId)
|
||||
: undefined
|
||||
turnSpan = startSpan(IOSpanNames.InteractionTurn, voiceParent)
|
||||
ownedTurnSpans.set(turnKey, turnSpan)
|
||||
}
|
||||
// Speech recognition, speech output, and streaming control attach to the most recently started turn.
|
||||
// Speech output and streaming control attach to the most recently started turn.
|
||||
if (turnSpan)
|
||||
activeTurnSpan.value = turnSpan
|
||||
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
import { IOAttributes, IOSpanNames, IOSubsystems } from '@proj-airi/stage-shared'
|
||||
import { createPinia, setActivePinia } from 'pinia'
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import { computed } from 'vue'
|
||||
|
||||
import { startSpan } from '../../composables/use-io-tracer'
|
||||
import { useIOTracerStore } from './io-tracer'
|
||||
|
||||
describe('useIOTracerStore', () => {
|
||||
afterEach(() => {
|
||||
useIOTracerStore().stopRecording()
|
||||
})
|
||||
|
||||
it('updates views of a turn when more spans of its trace end', () => {
|
||||
setActivePinia(createPinia())
|
||||
const store = useIOTracerStore()
|
||||
store.startRecording()
|
||||
// The chart receives the turn list as a prop, so it keeps one array reference and reads spans through it.
|
||||
const turns = store.turns
|
||||
const spanNames = computed(() => turns.map(turn => turn.spans.map(span => span.name)))
|
||||
|
||||
const voiceInput = startSpan(IOSpanNames.VoiceInput, undefined, { [IOAttributes.Subsystem]: IOSubsystems.VoiceInput })
|
||||
const recognition = startSpan(IOSpanNames.SpeechRecognition, voiceInput, { [IOAttributes.Subsystem]: IOSubsystems.ASR })
|
||||
recognition.end()
|
||||
expect(spanNames.value).toEqual([[IOSpanNames.SpeechRecognition]])
|
||||
|
||||
voiceInput.end()
|
||||
|
||||
expect(spanNames.value).toEqual([[IOSpanNames.SpeechRecognition, IOSpanNames.VoiceInput]])
|
||||
// The child ended first, but the turn starts where its parent started.
|
||||
expect(store.turns[0]!.startTs).toBe(store.turns[0]!.spans[1]!.startTs)
|
||||
})
|
||||
})
|
||||
@@ -83,13 +83,15 @@ export const useIOTracerStore = defineStore('devtools:io-tracer', () => {
|
||||
function getOrCreateTurn(): IOTurn {
|
||||
let turn = turnsByTraceId.get(traceId)
|
||||
if (!turn) {
|
||||
turn = {
|
||||
turns.value.push({
|
||||
id: traceId,
|
||||
startTs: startMs,
|
||||
spans: [],
|
||||
}
|
||||
})
|
||||
// The reactive copy, not the raw object. Later spans of this trace mutate it, and views that hold the turn
|
||||
// list, such as the chart through its prop, only see mutations through the proxy.
|
||||
turn = turns.value.at(-1)!
|
||||
turnsByTraceId.set(traceId, turn)
|
||||
turns.value.push(turn)
|
||||
|
||||
while (turns.value.length > MAX_TURNS) {
|
||||
const evicted = turns.value.shift()
|
||||
@@ -97,6 +99,9 @@ export const useIOTracerStore = defineStore('devtools:io-tracer', () => {
|
||||
turnsByTraceId.delete(evicted.id)
|
||||
}
|
||||
}
|
||||
// Spans arrive when they end, so a child, such as a VAD segment, can arrive before the span that starts the turn.
|
||||
if (startMs < turn.startTs)
|
||||
turn.startTs = startMs
|
||||
return turn
|
||||
}
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ import { useVoiceController } from '../composables/audio/voice-controller'
|
||||
import { traceSpeechOutput } from '../composables/speech-output-trace'
|
||||
import { useVoiceDrafts } from '../composables/voice-drafts'
|
||||
import { useVoiceEndJudge } from '../composables/voice-end-judge'
|
||||
import { createVoiceInputTracer } from '../composables/voice-input-trace'
|
||||
import { useVoiceRephrase } from '../composables/voice-rephrase'
|
||||
import { observeInputLevel } from '../libs/voice/input-level'
|
||||
import { createVoiceActivityPlugin } from '../libs/voice/voice-activity-plugin'
|
||||
@@ -69,6 +70,7 @@ export const useVoiceStore = defineStore('voice', () => {
|
||||
const composerInputs = new Map<string, string | undefined>()
|
||||
let presentedInput: { requestId: string, attempt: SpeechInputAttempt } | undefined
|
||||
|
||||
const inputTracer = createVoiceInputTracer()
|
||||
const { controller, state, snapshot: transcript, error } = useVoiceController({
|
||||
transcriber: () => hearing.createTranscriber(),
|
||||
speech: (turn) => {
|
||||
@@ -85,6 +87,9 @@ export const useVoiceStore = defineStore('voice', () => {
|
||||
|
||||
// The composer owns this text as its own draft. No voice draft or message is created.
|
||||
composerInputs.set(submission.submissionId, submission.text)
|
||||
// The composer sends dictated text later. Empty text gives the composer nothing to send.
|
||||
if (submission.text.trim())
|
||||
inputTracer.markComposer(submission.submissionId, submission.sessionId)
|
||||
return { status: 'drafted', draftId: submission.submissionId }
|
||||
},
|
||||
recordInterruption: async (event) => {
|
||||
@@ -111,8 +116,18 @@ export const useVoiceStore = defineStore('voice', () => {
|
||||
console.error('[voice]', cause)
|
||||
}
|
||||
|
||||
/**
|
||||
* The cause of the input that begins next. A control sets `control` around its begin call, and the voice activity
|
||||
* plugin reports `speech` or `wake-word` right before its begin call. `onInput` runs inside that call and reads it.
|
||||
*/
|
||||
let beginCause: 'control' | 'speech' | 'wake-word' | undefined
|
||||
|
||||
controller.onInput((attempt) => {
|
||||
presentedInput = { requestId: attempt.id, attempt }
|
||||
const cause = beginCause ?? 'speech'
|
||||
beginCause = undefined
|
||||
// Only an input that a wake began takes the wake span, so a wake that started nothing cannot join a later input.
|
||||
inputTracer.start(attempt, cause, cause === 'wake-word' ? wakeWordDetection.claimWakeSpan() : undefined)
|
||||
})
|
||||
|
||||
function publishSnapshot() {
|
||||
@@ -149,7 +164,13 @@ export const useVoiceStore = defineStore('voice', () => {
|
||||
}
|
||||
|
||||
function beginManual(sessionId: string) {
|
||||
return beginInput({ sessionId, interruptTurns: activeTurns.value.filter(turn => turn.sessionId === sessionId), start: { kind: 'after-silence' } })
|
||||
beginCause = 'control'
|
||||
try {
|
||||
return beginInput({ sessionId, interruptTurns: activeTurns.value.filter(turn => turn.sessionId === sessionId), start: { kind: 'after-silence' } })
|
||||
}
|
||||
finally {
|
||||
beginCause = undefined
|
||||
}
|
||||
}
|
||||
|
||||
function endInput() {
|
||||
@@ -442,6 +463,10 @@ export const useVoiceStore = defineStore('voice', () => {
|
||||
const wakeWords = options.detectWakeWord ? undefined : wakeWordDetection.start({ resolveTarget: resolveWakeTarget, onError: report })
|
||||
const plugin = createVoiceActivityPlugin({
|
||||
detect: (window, signal) => model.score(window, signal),
|
||||
onActivity: inputTracer.activity,
|
||||
beforeBeginInput: (cause) => {
|
||||
beginCause = cause
|
||||
},
|
||||
detectWakeWord: options.detectWakeWord ?? wakeWords?.detect,
|
||||
enabled: () => !voiceMessages.isRecording,
|
||||
silenceMs: () => hearing.endJudgeEnabled ? VOICE_END_SILENCE.pauseMs + VOICE_END_SILENCE.graceMs : VOICE_END_SILENCE.pauseMs,
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
import type { Span } from '@opentelemetry/api'
|
||||
import type { AudioWindow } from '@proj-airi/pipelines-audio'
|
||||
|
||||
import type { WakeActivity, WakeTarget, WakeWordPreparation } from '../libs/voice/wake-word-detector'
|
||||
|
||||
import { IOAttributes, IOSpanNames, IOSubsystems } from '@proj-airi/stage-shared'
|
||||
import { defineStore } from 'pinia'
|
||||
import { computed, shallowRef, watch } from 'vue'
|
||||
|
||||
import { runInSpan, startSpan } from '../composables/use-io-tracer'
|
||||
import { createKwsSpotter, kwsModel, kwsSearch, kwsToneVariants } from '../libs/voice/kws-model'
|
||||
import { WakeWordDetector } from '../libs/voice/wake-word-detector'
|
||||
import { useSettingsAudioDevice } from './settings/audio-device'
|
||||
@@ -31,6 +34,11 @@ export const useWakeWordDetectionStore = defineStore('wake-word-detection', () =
|
||||
/** Pronunciations of other models are not in the pinned vocabulary. They stay on their cards but do not run. */
|
||||
const pronunciations = computed(() => wakeWords.catalog.active.filter(pronunciation => pronunciation.modelId === kwsModel.id))
|
||||
let running: { detector: WakeWordDetector, stopWatches: readonly (() => void)[] } | undefined
|
||||
/**
|
||||
* The `Wake word` span of the newest resolved wake, until an input claims it as its trace root.
|
||||
* A wake that starts no input, for example because that session already listens, leaves a span that nothing claims.
|
||||
*/
|
||||
let unclaimedWake: { span: Span, resolvedAt: number } | undefined
|
||||
|
||||
function stop() {
|
||||
const current = running
|
||||
@@ -55,7 +63,20 @@ export const useWakeWordDetectionStore = defineStore('wake-word-detection', () =
|
||||
createSpotter: createKwsSpotter,
|
||||
search: { score: kwsSearch.score, threshold: kwsSearch.threshold },
|
||||
variants: kwsToneVariants,
|
||||
resolveTarget: options.resolveTarget,
|
||||
// The span covers the match and the session selection. The voice input that the wake starts becomes its child.
|
||||
resolveTarget: (characterId, signal, pronunciation) => {
|
||||
const span = startSpan(IOSpanNames.WakeWord, undefined, {
|
||||
[IOAttributes.Subsystem]: IOSubsystems.KWS,
|
||||
[IOAttributes.WakeCharacterId]: characterId,
|
||||
[IOAttributes.WakeText]: pronunciation.text,
|
||||
[IOAttributes.WakeTokens]: pronunciation.tokens.join(' '),
|
||||
})
|
||||
return runInSpan(span, signal, async () => {
|
||||
const target = await options.resolveTarget(characterId, signal)
|
||||
unclaimedWake = { span, resolvedAt: performance.now() }
|
||||
return target
|
||||
})
|
||||
},
|
||||
onError: options.onError,
|
||||
onPreparationChange: (value, message) => {
|
||||
// A replaced detection can still settle a model load. Only the running detector updates the UI state.
|
||||
@@ -91,5 +112,16 @@ export const useWakeWordDetectionStore = defineStore('wake-word-detection', () =
|
||||
}
|
||||
}
|
||||
|
||||
return { preparation, error, pronunciations, start, stop }
|
||||
/**
|
||||
* Returns the span of the newest resolved wake, once. Call it only for an input that a wake began.
|
||||
* A wake that started no input stays unclaimed until a newer wake replaces it.
|
||||
* The second limit drops a wake that resolved long before, for example while the plugin was paused.
|
||||
*/
|
||||
function claimWakeSpan(): Span | undefined {
|
||||
const wake = unclaimedWake
|
||||
unclaimedWake = undefined
|
||||
return wake && performance.now() - wake.resolvedAt < 1000 ? wake.span : undefined
|
||||
}
|
||||
|
||||
return { preparation, error, pronunciations, start, stop, claimWakeSpan }
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user