diff --git a/packages/stage-ui-three/src/components/Model/VRMModel.vue b/packages/stage-ui-three/src/components/Model/VRMModel.vue index 1886914c48..0a9710c4b0 100644 --- a/packages/stage-ui-three/src/components/Model/VRMModel.vue +++ b/packages/stage-ui-three/src/components/Model/VRMModel.vue @@ -202,7 +202,10 @@ type UpdatableMaterial = Material & { // Expressions const blink = useBlink() const idleEyeSaccades = useIdleEyeSaccades() -const vrmEmote = ref>() +// shallowRef: the composable object must stay non-reactive, otherwise Vue +// deep-unwraps nested refs/computed (e.g. isEmoteActive) and breaks reads +// like vrmEmote.value.isEmoteActive.value in the render loop. +const vrmEmote = shallowRef>() const vrmLipSync = useVRMLipSync(audioContext, currentAudioSource) // For sky box update @@ -477,15 +480,31 @@ function bindManagedVrmInstanceRenderLoop() { const lookAtMs = measureFrameStep(tracingEnabled, () => { activeVrm?.lookAt?.update?.(delta) }) + const isEmoteActive = vrmEmote.value?.isEmoteActive?.value ?? false + const blinkAndSaccadeMs = measureFrameStep(tracingEnabled, () => { - blink.update(activeVrm, delta) - }) - const emoteMs = measureFrameStep(tracingEnabled, () => { - vrmEmote.value?.update(delta) + // The blink controller always advances so an emote starting mid-blink + // cannot leave the eyelid stuck; during an emote it only holds the + // blink morph at 0 instead of driving the sine curve. + blink.update(activeVrm, delta, { suppress: isEmoteActive }) }) const lipSyncMs = measureFrameStep(tracingEnabled, () => { vrmLipSync.update(activeVrm, delta) }) + // Read after lipSyncMs, not before: vrmLipSync.update() is what flips + // this flag for the current frame. Reading it earlier in the frame would + // hand the emote a stale value from the previous frame, letting it + // overwrite the mouth morph lip sync just wrote on the first frame of an + // utterance. + const isLipSyncActive = vrmLipSync.isLipSyncActive?.value ?? false + const emoteMs = measureFrameStep(tracingEnabled, () => { + // Runs after lip sync: while speech is active the emote yields viseme + // mouth morphs (skipVisemes), and once lip sync falls silent the emote + // re-asserts its mouth targets over lip sync's zeroed weights. The + // state machine keeps advancing either way, so emotion transitions and + // the reset timeout stay in sync during long utterances. + vrmEmote.value?.update(delta, { skipVisemes: isLipSyncActive }) + }) const expressionMs = measureFrameStep(tracingEnabled, () => { activeVrm?.expressionManager?.update() }) diff --git a/packages/stage-ui-three/src/composables/vrm/animation.test.ts b/packages/stage-ui-three/src/composables/vrm/animation.test.ts new file mode 100644 index 0000000000..04ec95fa3b --- /dev/null +++ b/packages/stage-ui-three/src/composables/vrm/animation.test.ts @@ -0,0 +1,108 @@ +import type { VRMCore } from '@pixiv/three-vrm-core' + +import { afterEach, describe, expect, it, vi } from 'vitest' + +import { useBlink } from './animation' + +function createMockVRMCore() { + return { + expressionManager: { + setValue: vi.fn(), + }, + } as unknown as VRMCore +} + +function lastBlinkValue(vrm: VRMCore) { + const calls = vi.mocked(vrm.expressionManager!.setValue).mock.calls + return calls.length ? calls[calls.length - 1][1] : undefined +} + +describe('useBlink', () => { + afterEach(() => { + vi.restoreAllMocks() + }) + + it('holds the lid at 0 and still completes the blink cycle while suppressed mid-blink', () => { + // ROOT CAUSE: + // + // Previously the render loop simply stopped calling blink.update while an + // emote was active. Since update is the only place that advances + // blinkProgress and resets the morph to 0, an emote starting mid-blink + // froze the sine cycle and left the eyelid stuck closed until the emote + // reset. + // + // We fixed this by always advancing the controller and holding the blink + // morph at 0 while suppressed. + vi.spyOn(Math, 'random').mockReturnValue(0) // nextBlinkTime = MIN_BLINK_INTERVAL (1s) + + const vrm = createMockVRMCore() + const blink = useBlink() + + // Advance to just before the first blink (62 * 0.016s = 0.992s < 1s) + for (let i = 0; i < 62; i++) + blink.update(vrm, 0.016) + expect(vrm.expressionManager?.setValue).not.toHaveBeenCalled() + + // Cross the interval: blink starts, sine value written + blink.update(vrm, 0.016) + expect(lastBlinkValue(vrm)).toBeGreaterThan(0) + + // Emote begins mid-blink: suppressed updates hold the lid at 0 + blink.update(vrm, 0.016, { suppress: true }) + expect(lastBlinkValue(vrm)).toBe(0) + + // Run out the remainder of the 0.2s cycle under suppression + for (let i = 0; i < 12; i++) + blink.update(vrm, 0.016, { suppress: true }) + expect(lastBlinkValue(vrm)).toBe(0) + + // After the cycle completes the controller is not stuck: the emote ends + // and a later blink writes a real sine value again. random=0 keeps the + // next interval at 1s, so advance past it. + for (let i = 0; i < 64; i++) + blink.update(vrm, 0.016) + expect(lastBlinkValue(vrm)).toBeGreaterThan(0) + }) + + it('releases the lid when suppression begins between blinks', () => { + const vrm = createMockVRMCore() + const blink = useBlink() + + blink.update(vrm, 0.016, { suppress: true }) + expect(vrm.expressionManager?.setValue).toHaveBeenCalledWith('blink', 0) + }) + + it('does not reveal a partially closed lid when suppression ends during a blink cycle', () => { + vi.spyOn(Math, 'random').mockReturnValue(0) // nextBlinkTime = 1s + + const vrm = createMockVRMCore() + const blink = useBlink() + + // Advance to just before the first blink (0.992s < 1s) + for (let i = 0; i < 62; i++) + blink.update(vrm, 0.016) + + // Blink starts under suppression (e.g. emote is active) + blink.update(vrm, 0.016, { suppress: true }) + expect(lastBlinkValue(vrm)).toBe(0) + + // Advance halfway into the blink cycle under suppression (progress ~ 0.5, peak sine = 1.0) + blink.update(vrm, 0.084, { suppress: true }) + expect(lastBlinkValue(vrm)).toBe(0) + + // Emote ends mid-cycle (suppress released while progress is ~0.58) + // The hidden cycle must continue holding 0 instead of popping to sine value. + blink.update(vrm, 0.016, { suppress: false }) + expect(lastBlinkValue(vrm)).toBe(0) + + // Complete the remainder of the 0.2s cycle unsuppressed + for (let i = 0; i < 8; i++) + blink.update(vrm, 0.016) + expect(lastBlinkValue(vrm)).toBe(0) + + // Next scheduled blink (after 1s) runs normally and produces real sine values + for (let i = 0; i < 64; i++) + blink.update(vrm, 0.016) + expect(lastBlinkValue(vrm)).toBeGreaterThan(0) + }) +}) diff --git a/packages/stage-ui-three/src/composables/vrm/animation.ts b/packages/stage-ui-three/src/composables/vrm/animation.ts index 7f1028dfb8..483657554f 100644 --- a/packages/stage-ui-three/src/composables/vrm/animation.ts +++ b/packages/stage-ui-three/src/composables/vrm/animation.ts @@ -93,6 +93,7 @@ export function useBlink() { */ const isBlinking = ref(false) const blinkProgress = ref(0) + const isCycleSuppressed = ref(false) const timeSinceLastBlink = ref(0) const BLINK_DURATION = 0.2 // Duration of a single blink in seconds const MIN_BLINK_INTERVAL = 1 // Minimum time between blinks @@ -100,7 +101,7 @@ export function useBlink() { const nextBlinkTime = ref(Math.random() * (MAX_BLINK_INTERVAL - MIN_BLINK_INTERVAL) + MIN_BLINK_INTERVAL) // Function to handle blinking animation - function update(vrm: VRMCore | undefined, delta: number) { + function update(vrm: VRMCore | undefined, delta: number, options?: { suppress?: boolean }) { if (!vrm?.expressionManager) return @@ -110,14 +111,22 @@ export function useBlink() { if (!isBlinking.value && timeSinceLastBlink.value >= nextBlinkTime.value) { isBlinking.value = true blinkProgress.value = 0 + isCycleSuppressed.value = Boolean(options?.suppress) } // Handle blinking animation if (isBlinking.value) { blinkProgress.value += delta / BLINK_DURATION + if (options?.suppress) + isCycleSuppressed.value = true - // Calculate blink value using sine curve for smooth animation - const blinkValue = Math.sin(Math.PI * blinkProgress.value) + // Calculate blink value using sine curve for smooth animation. + // While an emote owns the eye area, the controller keeps advancing so a + // blink interrupted mid-cycle cannot leave the lid stuck closed. + // If the cycle started under suppression or was suppressed mid-blink, + // hold the morph at 0 until the cycle completes so releasing suppression + // cannot pop the eyelid into a partially closed pose. + const blinkValue = isCycleSuppressed.value ? 0 : Math.sin(Math.PI * blinkProgress.value) // Apply blink expression vrm.expressionManager.setValue('blink', blinkValue) @@ -125,11 +134,16 @@ export function useBlink() { // Reset blink when animation is complete if (blinkProgress.value >= 1) { isBlinking.value = false + isCycleSuppressed.value = false timeSinceLastBlink.value = 0 vrm.expressionManager.setValue('blink', 0) // Reset blink value to 0 nextBlinkTime.value = Math.random() * (MAX_BLINK_INTERVAL - MIN_BLINK_INTERVAL) + MIN_BLINK_INTERVAL } } + else if (options?.suppress) { + // Release the lid when an emote begins between blinks. + vrm.expressionManager.setValue('blink', 0) + } } return { update } diff --git a/packages/stage-ui-three/src/composables/vrm/expression.test.ts b/packages/stage-ui-three/src/composables/vrm/expression.test.ts new file mode 100644 index 0000000000..3408539ee4 --- /dev/null +++ b/packages/stage-ui-three/src/composables/vrm/expression.test.ts @@ -0,0 +1,249 @@ +import type { VRMCore } from '@pixiv/three-vrm-core' + +import { describe, expect, it, vi } from 'vitest' + +import { useVRMEmote } from './expression' + +function createMockVRMCore() { + const values = new Map() + return { + expressionManager: { + expressionMap: { + happy: {}, + aa: {}, + ee: {}, + ih: {}, + oh: {}, + ou: {}, + sad: {}, + angry: {}, + surprised: {}, + neutral: {}, + think: {}, + relaxed: {}, + // Not owned by any emotion state; driven by the blink controller. + blink: {}, + }, + getValue: vi.fn((name: string) => values.get(name) ?? 0), + setValue: vi.fn((name: string, val: number) => { + values.set(name, val) + }), + }, + } as unknown as VRMCore +} + +describe('useVRMEmote', () => { + it('updates expression weights during transition and maintains hold after transition completes', () => { + const vrm = createMockVRMCore() + const emote = useVRMEmote(vrm) + + expect(emote.isEmoteActive.value).toBe(false) + + emote.setEmotion('happy', 1) + expect(emote.currentEmotion.value).toBe('happy') + expect(emote.isTransitioning.value).toBe(true) + expect(emote.isEmoteActive.value).toBe(true) + + // Blend duration is 0.4s for happy + emote.update(0.2) // Halfway + expect(emote.isTransitioning.value).toBe(true) + expect(vrm.expressionManager?.setValue).toHaveBeenCalledWith('happy', expect.any(Number)) + + emote.update(0.3) // Transition finishes + expect(emote.isTransitioning.value).toBe(false) + expect(emote.isEmoteActive.value).toBe(true) + + // Clear calls to inspect hold behavior + vi.mocked(vrm.expressionManager!.setValue).mockClear() + + // Subsequent frame after transition completion should continue holding target weights + emote.update(0.016) + expect(vrm.expressionManager?.setValue).toHaveBeenCalledWith('happy', 0.7) + expect(vrm.expressionManager?.setValue).toHaveBeenCalledWith('aa', 0.2) + + // ROOT CAUSE: + // + // Previously the emote captured every entry of expressionMap with a + // default target of 0, so the hold branch overwrote blink and other + // animation-driven expressions on every frame after the first emotion. + // + // We fixed this by capturing only morphs owned by the emotion states. + expect(vrm.expressionManager?.setValue).not.toHaveBeenCalledWith('blink', expect.anything()) + }) + + it('does not mark the emote active for a mouth-only (viseme-only) custom state', () => { + // ROOT CAUSE: + // + // isEmoteActive treated any nonzero target as an eye conflict, so + // VRMModel suppressed procedural blinking for the entire hold period + // of *any* emotion — including a custom state added via + // addEmotionState whose targets are only viseme mouth morphs and + // never touch the eye region. That left the model unable to blink + // naturally while such a state was held, despite no real conflict. + // + // We fixed this by only counting non-viseme target names as + // eye-affecting for isEmoteActive. + const vrm = createMockVRMCore() + const emote = useVRMEmote(vrm) + + emote.addEmotionState('mouth-shape', { + expression: [{ name: 'aa', value: 0.5 }], + blendDuration: 0.2, + }) + + emote.setEmotion('mouth-shape', 1) + emote.update(0.2) // Settle the transition + + expect(emote.currentEmotion.value).toBe('mouth-shape') + expect(emote.isEmoteActive.value).toBe(false) + }) + + it('still zeroes a morph after its emotion state is removed before the reset settles', () => { + // ROOT CAUSE: + // + // The neutral transition captured which morphs to zero via + // ownedExpressionNames(), recomputed from the *current* emotionStates + // map every time setEmotion() ran. If a caller removed (or replaced) + // the departing emotion's state — e.g. removeEmotionState('happy') — + // before the scheduled reset fired, that morph silently dropped out of + // ownership and the neutral transition never wrote it back to 0, + // leaving the avatar stuck showing the removed expression forever. + // + // We fixed this by tracking active morph names in a set that survives + // emotionStates mutations, cleared only once the neutral transition + // fully settles. + const vrm = createMockVRMCore() + const emote = useVRMEmote(vrm) + + emote.setEmotion('happy', 1) + emote.update(0.4) // Settle happy: 'happy' and 'aa' held at 0.7 / 0.2 + + emote.removeEmotionState('happy') + + emote.setEmotion('neutral') + vi.mocked(vrm.expressionManager!.setValue).mockClear() + + emote.update(1.0) // > 0.6s blend duration for neutral + expect(emote.currentEmotion.value).toBeNull() + expect(vrm.expressionManager?.setValue).toHaveBeenCalledWith('happy', 0) + expect(vrm.expressionManager?.setValue).toHaveBeenCalledWith('aa', 0) + expect(vrm.expressionManager?.setValue).toHaveBeenCalledWith('neutral', 1) + }) + + it('keeps the state machine advancing during lip sync and blends deferred visemes when speech ends', () => { + const vrm = createMockVRMCore() + const emote = useVRMEmote(vrm) + + emote.setEmotion('happy', 1) + + // Lip sync active: visemes yield, but the transition still progresses. + emote.update(0.2, { skipVisemes: true }) + expect(vrm.expressionManager?.setValue).toHaveBeenCalledWith('happy', expect.any(Number)) + expect(vrm.expressionManager?.setValue).not.toHaveBeenCalledWith('aa', expect.anything()) + + emote.update(0.3, { skipVisemes: true }) + expect(emote.isTransitioning.value).toBe(false) + + vi.mocked(vrm.expressionManager!.setValue).mockClear() + + // Lip sync released: deferred visemes start blending smoothly instead of jumping immediately to target weight. + emote.update(0.016, { skipVisemes: false }) + expect(vrm.expressionManager?.setValue).toHaveBeenCalledWith('aa', expect.any(Number)) + const firstFrameValue = vi.mocked(vrm.expressionManager!.setValue).mock.calls.find(call => call[0] === 'aa')?.[1] + expect(firstFrameValue).toBeGreaterThan(0) + expect(firstFrameValue).toBeLessThan(0.2) + expect(emote.isVisemeTransitioning.value).toBe(true) + + // Run out the remainder of the blend duration (0.4s for happy) + emote.update(0.4, { skipVisemes: false }) + expect(vrm.expressionManager?.setValue).toHaveBeenCalledWith('aa', 0.2) + expect(emote.isVisemeTransitioning.value).toBe(false) + + vi.mocked(vrm.expressionManager!.setValue).mockClear() + + // Subsequent frames hold the target weight + emote.update(0.016, { skipVisemes: false }) + expect(vrm.expressionManager?.setValue).toHaveBeenCalledWith('aa', 0.2) + }) + + it('maintains emote active flag during neutral transition and resets when blend completes', () => { + const vrm = createMockVRMCore() + const emote = useVRMEmote(vrm) + + emote.setEmotion('sad', 1) + expect(emote.isEmoteActive.value).toBe(true) + + // Settle the 'sad' transition so values exist in expressionManager + emote.update(0.4) + expect(emote.isTransitioning.value).toBe(false) + expect(emote.isEmoteActive.value).toBe(true) + + // Resetting to neutral starts a transition (blendDuration: 0.6s) + emote.setEmotion('neutral') + expect(emote.isTransitioning.value).toBe(true) + // Emote should remain active while previous emotion weights fade out + expect(emote.isEmoteActive.value).toBe(true) + + // Complete the neutral transition + emote.update(0.6) + expect(emote.isTransitioning.value).toBe(false) + expect(emote.isEmoteActive.value).toBe(false) + expect(emote.currentEmotion.value).toBeNull() + + // Terminal neutral weights should have been written + expect(vrm.expressionManager?.setValue).toHaveBeenCalledWith('sad', 0) + expect(vrm.expressionManager?.setValue).toHaveBeenCalledWith('oh', 0) + expect(vrm.expressionManager?.setValue).toHaveBeenCalledWith('neutral', 1) + + vi.mocked(vrm.expressionManager!.setValue).mockClear() + + // Subsequent frames should not assert zero over expressions, releasing ownership + emote.update(0.016) + expect(vrm.expressionManager?.setValue).not.toHaveBeenCalled() + }) + + it('applies terminal neutral weights when transition completes in a single delta jump after pause', () => { + const vrm = createMockVRMCore() + const emote = useVRMEmote(vrm) + + emote.setEmotion('happy', 1) + emote.update(0.4) // Settle happy + + emote.setEmotion('neutral') + vi.mocked(vrm.expressionManager!.setValue).mockClear() + + // Delta jump of 1.0s (> 0.6s blend duration) + emote.update(1.0) + expect(emote.isTransitioning.value).toBe(false) + expect(emote.currentEmotion.value).toBeNull() + expect(vrm.expressionManager?.setValue).toHaveBeenCalledWith('happy', 0) + expect(vrm.expressionManager?.setValue).toHaveBeenCalledWith('aa', 0) + expect(vrm.expressionManager?.setValue).toHaveBeenCalledWith('neutral', 1) + + vi.mocked(vrm.expressionManager!.setValue).mockClear() + emote.update(0.016) + expect(vrm.expressionManager?.setValue).not.toHaveBeenCalled() + }) + + it('does not activate emote flag for no-op intensity or empty targets and avoids holding zero weights', () => { + const vrm = createMockVRMCore() + const emote = useVRMEmote(vrm) + + emote.setEmotion('happy', 0) + expect(emote.isEmoteActive.value).toBe(false) + expect(emote.currentEmotion.value).toBeNull() + expect(emote.isTransitioning.value).toBe(false) + + // Ensure update on subsequent frames is a no-op and does not overwrite mixer or other controllers with 0 + emote.update(0.016) + emote.update(0.5) + expect(vrm.expressionManager?.setValue).not.toHaveBeenCalled() + + // Test with setEmotionWithResetAfter as well + emote.setEmotionWithResetAfter('happy', 3000, 0) + expect(emote.isEmoteActive.value).toBe(false) + expect(emote.currentEmotion.value).toBeNull() + emote.update(0.016) + expect(vrm.expressionManager?.setValue).not.toHaveBeenCalled() + }) +}) diff --git a/packages/stage-ui-three/src/composables/vrm/expression.ts b/packages/stage-ui-three/src/composables/vrm/expression.ts index 195cd96d69..72369ccea1 100644 --- a/packages/stage-ui-three/src/composables/vrm/expression.ts +++ b/packages/stage-ui-three/src/composables/vrm/expression.ts @@ -1,6 +1,6 @@ import type { VRMCore } from '@pixiv/three-vrm-core' -import { ref } from 'vue' +import { computed, ref } from 'vue' interface EmotionState { expression?: { @@ -16,9 +16,22 @@ export function useVRMEmote(vrm: VRMCore) { const currentEmotion = ref(null) const isTransitioning = ref(false) const transitionProgress = ref(0) + const isVisemeTransitioning = ref(false) + const visemeTransitionProgress = ref(0) + const visemeStartValues = ref(new Map()) + const visemeBlendDuration = ref(0.3) const currentExpressionValues = ref(new Map()) const targetExpressionValues = ref(new Map()) const resetTimeout = ref() + let wasSkippingVisemes = false + // Morph names captured while entering an emotion, kept until the neutral + // transition fully settles. ownedExpressionNames() only reflects emotion + // states registered *right now*, so if a caller calls removeEmotionState + // (or replaces a state) for the departing emotion before its scheduled + // reset fires, its morph would otherwise drop out of tracking and never + // get the terminal zero write — leaving the avatar stuck in that + // expression. This set survives such mutations across emotionStates. + const trackedExpressionNames = new Set() // Utility functions const lerp = (start: number, end: number, t: number): number => { @@ -86,6 +99,25 @@ export function useVRMEmote(vrm: VRMCore) { }], ]) + // Morphs the emote owns: only expressions referenced by emotion states. + // Blink and animation-driven expressions stay under their own controllers, + // so the emote must never assert values (including 0) for them. + const ownedExpressionNames = (modelNames: string[]): string[] => { + const owned = new Set() + for (const state of emotionStates.values()) { + for (const expr of state.expression || []) { + const match = modelNames.find(n => n.toLowerCase() === expr.name.toLowerCase()) + if (match) + owned.add(match) + } + } + return [...owned] + } + + // Viseme mouth morphs driven by lip sync; the emote yields these while + // speech is active so phoneme weights are not overwritten. + const VISEME_NAMES = new Set(['aa', 'ee', 'ih', 'oh', 'ou']) + const clearResetTimeout = () => { if (resetTimeout.value) { clearTimeout(resetTimeout.value) @@ -94,17 +126,25 @@ export function useVRMEmote(vrm: VRMCore) { } const setEmotion = (emotionName: string, intensity = 1) => { - clearResetTimeout() + const normalizedIntensity = clampIntensity(intensity) + if (normalizedIntensity <= 0) + return if (!emotionStates.has(emotionName)) { console.warn(`Emotion ${emotionName} not found`) return } + clearResetTimeout() + const emotionState = emotionStates.get(emotionName)! currentEmotion.value = emotionName isTransitioning.value = true transitionProgress.value = 0 + isVisemeTransitioning.value = false + visemeTransitionProgress.value = 0 + visemeStartValues.value.clear() + wasSkippingVisemes = false // Store current expression values as starting point BEFORE resetting, // so the lerp transition starts from the actual displayed values @@ -112,17 +152,28 @@ export function useVRMEmote(vrm: VRMCore) { currentExpressionValues.value.clear() targetExpressionValues.value.clear() - const normalizedIntensity = clampIntensity(intensity) - if (vrm.expressionManager) { - // Capture current values for all expressions we'll be transitioning + // Capture current values only for morphs the emote owns, so the lerp + // transition starts from the actual displayed values instead of + // snapping to 0 first (fixes #590). Unrelated expressions (blink, + // animation-driven morphs) are left to their own controllers. + // + // Union with trackedExpressionNames rather than using + // ownedExpressionNames() alone: a still-pending morph from an + // emotion state removed since it was entered must keep being driven + // toward 0, even though it is no longer "owned" by any registered + // state. const expressionNames = Object.keys(vrm.expressionManager.expressionMap) - for (const name of expressionNames) { + const namesToCapture = new Set([...ownedExpressionNames(expressionNames), ...trackedExpressionNames]) + for (const name of namesToCapture) { const currentValue = vrm.expressionManager.getValue(name) || 0 currentExpressionValues.value.set(name, currentValue) - // Default target is 0 for expressions not in the target emotion + // Default target is 0 for owned expressions not in the target emotion targetExpressionValues.value.set(name, 0) } + trackedExpressionNames.clear() + for (const name of namesToCapture) + trackedExpressionNames.add(name) } // Override target values for specified expressions in the emotion state @@ -140,6 +191,9 @@ export function useVRMEmote(vrm: VRMCore) { } const setEmotionWithResetAfter = (emotionName: string, ms: number, intensity = 1) => { + if (clampIntensity(intensity) <= 0) + return + clearResetTimeout() setEmotion(emotionName, intensity) @@ -150,28 +204,153 @@ export function useVRMEmote(vrm: VRMCore) { }, ms) as unknown as number } - const update = (deltaTime: number) => { - if (!isTransitioning.value || !currentEmotion.value) - return + // isEmoteActive's only consumer (VRMModel.vue) uses it to suppress + // procedural blinking, which only matters for morphs that actually + // deform the eye region. Viseme mouth morphs (aa/ee/ih/oh/ou) never do, + // so an emotion state driving only visemes — e.g. a custom mouth-only + // state added via addEmotionState — must not suppress blinking just + // because those (mouth-only) targets are nonzero. + const hasNonzeroEyeAffectingTarget = (values: Map): boolean => + Array.from(values.entries()).some(([name, val]) => val > 0.001 && !VISEME_NAMES.has(name.toLowerCase())) - const emotionState = emotionStates.get(currentEmotion.value)! - const blendDuration = emotionState.blendDuration || 0.3 + const isEmoteActive = computed(() => { + const hasActiveTarget = currentEmotion.value !== null + && currentEmotion.value !== 'neutral' + && hasNonzeroEyeAffectingTarget(targetExpressionValues.value) - transitionProgress.value += deltaTime / blendDuration - if (transitionProgress.value >= 1.0) { - transitionProgress.value = 1.0 - isTransitioning.value = false + if (hasActiveTarget) + return true + + // When transitioning (e.g. returning to neutral), remain active while non-zero + // expression weights are still fading out to prevent procedural blink conflicts. + if (isTransitioning.value || isVisemeTransitioning.value) { + return hasNonzeroEyeAffectingTarget(currentExpressionValues.value) } - // Update all expressions - for (const [exprName, targetValue] of targetExpressionValues.value) { - const startValue = currentExpressionValues.value.get(exprName) || 0 - const currentValue = lerp( - startValue, - targetValue, - easeInOutCubic(transitionProgress.value), - ) - vrm.expressionManager?.setValue(exprName, currentValue) + return false + }) + + const update = (deltaTime: number, options?: { skipVisemes?: boolean }) => { + if (!currentEmotion.value) + return + + // While lip sync owns the mouth, viseme writes are skipped but the + // transition and reset lifecycle keep advancing, so emotions triggered + // during speech are not lost. + const isSkipping = Boolean(options?.skipVisemes) + const skip = isSkipping + ? (name: string) => VISEME_NAMES.has(name.toLowerCase()) + : () => false + + if (isSkipping) { + wasSkippingVisemes = true + isVisemeTransitioning.value = false + visemeStartValues.value.clear() + } + else if (wasSkippingVisemes) { + // Speech has ended and lip sync relinquished mouth ownership. + // If any target viseme differs from its current displayed weight, + // blend it smoothly into the emotion pose instead of snapping at once. + wasSkippingVisemes = false + visemeStartValues.value.clear() + if (vrm.expressionManager) { + for (const [exprName, targetValue] of targetExpressionValues.value) { + if (!VISEME_NAMES.has(exprName.toLowerCase())) + continue + const currentValue = vrm.expressionManager.getValue(exprName) || 0 + if (Math.abs(targetValue - currentValue) > 0.001) { + visemeStartValues.value.set(exprName, currentValue) + } + } + } + if (visemeStartValues.value.size > 0) { + isVisemeTransitioning.value = true + visemeTransitionProgress.value = 0 + const emotionState = emotionStates.get(currentEmotion.value) + visemeBlendDuration.value = emotionState?.blendDuration || 0.3 + } + } + + if (isVisemeTransitioning.value) { + visemeTransitionProgress.value += deltaTime / visemeBlendDuration.value + const isVisemeSettling = visemeTransitionProgress.value >= 1.0 + if (isVisemeSettling) { + visemeTransitionProgress.value = 1.0 + isVisemeTransitioning.value = false + } + + for (const [exprName, startValue] of visemeStartValues.value) { + const targetValue = targetExpressionValues.value.get(exprName) ?? 0 + const currentValue = isVisemeSettling + ? targetValue + : lerp( + startValue, + targetValue, + easeInOutCubic(visemeTransitionProgress.value), + ) + vrm.expressionManager?.setValue(exprName, currentValue) + currentExpressionValues.value.set(exprName, currentValue) + } + + if (isVisemeSettling && !isTransitioning.value && currentEmotion.value === 'neutral') { + currentEmotion.value = null + currentExpressionValues.value.clear() + targetExpressionValues.value.clear() + visemeStartValues.value.clear() + trackedExpressionNames.clear() + return + } + } + + if (!currentEmotion.value) + return + + if (isTransitioning.value) { + const emotionState = emotionStates.get(currentEmotion.value)! + const blendDuration = emotionState.blendDuration || 0.3 + + transitionProgress.value += deltaTime / blendDuration + const isSettling = transitionProgress.value >= 1.0 + if (isSettling) { + transitionProgress.value = 1.0 + isTransitioning.value = false + } + + // Update all expressions with lerp (or terminal targets if isSettling) + for (const [exprName, targetValue] of targetExpressionValues.value) { + if (skip(exprName)) + continue + if (isVisemeTransitioning.value && VISEME_NAMES.has(exprName.toLowerCase())) + continue + const startValue = currentExpressionValues.value.get(exprName) || 0 + const currentValue = isSettling + ? targetValue + : lerp( + startValue, + targetValue, + easeInOutCubic(transitionProgress.value), + ) + vrm.expressionManager?.setValue(exprName, currentValue) + } + + // Once the neutral transition completes and terminal weights are written, + // clear targets and reset currentEmotion to release morph ownership. + if (isSettling && !isVisemeTransitioning.value && currentEmotion.value === 'neutral') { + currentEmotion.value = null + currentExpressionValues.value.clear() + targetExpressionValues.value.clear() + trackedExpressionNames.clear() + } + } + else if (currentEmotion.value) { + // Hold target expression values across render frames so other updates don't clear them + for (const [exprName, targetValue] of targetExpressionValues.value) { + if (skip(exprName)) + continue + if (isVisemeTransitioning.value && VISEME_NAMES.has(exprName.toLowerCase())) + continue + vrm.expressionManager?.setValue(exprName, targetValue) + } } } @@ -191,6 +370,8 @@ export function useVRMEmote(vrm: VRMCore) { return { currentEmotion, isTransitioning, + isVisemeTransitioning, + isEmoteActive, setEmotion, setEmotionWithResetAfter, update, diff --git a/packages/stage-ui-three/src/composables/vrm/lip-sync.ts b/packages/stage-ui-three/src/composables/vrm/lip-sync.ts index f85d98ba41..7606d0963f 100644 --- a/packages/stage-ui-three/src/composables/vrm/lip-sync.ts +++ b/packages/stage-ui-three/src/composables/vrm/lip-sync.ts @@ -8,7 +8,7 @@ import { WLIP_SYNC_VOWELS, wlipsyncProfile, } from '@proj-airi/model-driver-lipsync/shared/wlipsync' -import { shallowRef, watch } from 'vue' +import { ref, shallowRef, watch } from 'vue' const VRM_EXPRESSION_BY_VOWEL: Record = { A: 'aa', @@ -28,12 +28,14 @@ export function useVRMLipSync( audioContext: Readonly>, audioSource: Readonly>, ) { + const isLipSyncActive = ref(false) const lipSyncNode = shallowRef() const vowelDriver = createWLipSyncVowelDriver() watch(audioContext, (context, _, onCleanup) => { lipSyncNode.value = undefined vowelDriver.reset() + isLipSyncActive.value = false if (!context) return @@ -83,13 +85,25 @@ export function useVRMLipSync( function update(vrm?: VRMCore, delta = 0.016) { const node = lipSyncNode.value - if (!vrm?.expressionManager || !node) + if (!vrm?.expressionManager || !node) { + isLipSyncActive.value = false return + } const weights = vowelDriver.update(node, delta) - for (const vowel of WLIP_SYNC_VOWELS) - vrm.expressionManager.setValue(VRM_EXPRESSION_BY_VOWEL[vowel], weights[vowel]) + let hasActiveVisemes = false + for (const vowel of WLIP_SYNC_VOWELS) { + const weight = weights[vowel] + if (weight > 0.01) + hasActiveVisemes = true + vrm.expressionManager.setValue(VRM_EXPRESSION_BY_VOWEL[vowel], weight) + } + + isLipSyncActive.value = hasActiveVisemes } - return { update } + return { + isLipSyncActive, + update, + } } diff --git a/packages/stage-ui/src/components/scenes/Stage.vue b/packages/stage-ui/src/components/scenes/Stage.vue index acee0620cb..7b857e234c 100644 --- a/packages/stage-ui/src/components/scenes/Stage.vue +++ b/packages/stage-ui/src/components/scenes/Stage.vue @@ -1021,6 +1021,11 @@ defineExpose({ */ captureFrame: captureCharacterFrame, readRenderTargetRegionAtClientPoint, + setExpression: async (expression: string, intensity = 1) => { + if (stageModelRenderer.value === 'vrm') { + await vrmViewerRef.value?.setExpression(expression, intensity) + } + }, }) diff --git a/vitest.config.ts b/vitest.config.ts index a0f888c1d3..aebf71c7db 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -26,6 +26,7 @@ export default defineConfig({ 'packages/server-runtime', 'packages/server-sdk', 'packages/stage-shared', + 'packages/stage-ui-three', 'packages/vitest-plugin-fakemic', ], },