From 464d15cebcae852463fa0c5430e41c299c3659f6 Mon Sep 17 00:00:00 2001 From: Neko Date: Wed, 21 Oct 2026 17:15:49 +0800 Subject: [PATCH] feat(core-agent,stage-ui): now speech formula, math, chem correctly (#3054) --- cspell.config.yaml | 1 + .../2026-10-20-speech-text-preprocessor.md | 293 +++ evals/tts-speech-formula/.env.example | 41 + evals/tts-speech-formula/README.md | 200 ++ .../eval/cases/formula-speech/cases.ts | 697 +++++++ .../eval/cases/formula-speech/index.ts | 2 + .../eval/cases/formula-speech/languages.ts | 24 + .../eval/cases/math-bridge/fetch.ts | 207 ++ .../eval/cases/math-bridge/index.test.ts | 30 + .../eval/cases/math-bridge/index.ts | 44 + .../eval/cases/math-bridge/sample.json | 1448 ++++++++++++++ .../eval/evaluators/grader/chem.ts | 136 ++ .../eval/evaluators/grader/currency.ts | 31 + .../eval/evaluators/grader/index.test.ts | 87 + .../eval/evaluators/grader/index.ts | 63 + .../eval/evaluators/grader/normalize.ts | 166 ++ .../eval/evaluators/grader/symbolic.ts | 95 + .../judge/calibration/calibration.json | 845 ++++++++ .../evaluators/judge/calibration/collect.ts | 146 ++ .../evaluators/judge/calibration/index.html | 74 + .../eval/evaluators/judge/calibration/main.ts | 266 +++ .../evaluators/judge/calibration/style.css | 548 ++++++ .../evaluators/judge/calibration/types.ts | 25 + .../judge/calibration/vite.config.ts | 90 + .../eval/evaluators/judge/index.test.ts | 32 + .../eval/evaluators/judge/index.ts | 107 ++ .../eval/evaluators/judge/judge.velin.md | 57 + .../eval/evaluators/listener/index.test.ts | 21 + .../eval/evaluators/listener/index.ts | 164 ++ .../evaluators/listener/listener.velin.md | 73 + .../eval/evaluators/notation/index.test.ts | 25 + .../eval/evaluators/notation/index.ts | 67 + .../speech/gemini-transcription.test.ts | 49 + .../evaluators/speech/gemini-transcription.ts | 102 + .../eval/evaluators/speech/index.ts | 86 + .../evaluators/speech/speech-models.test.ts | 157 ++ .../eval/evaluators/speech/speech-models.ts | 486 +++++ evals/tts-speech-formula/eval/index.ts | 16 + .../eval/shared/cache-key.ts | 12 + .../eval/shared/telemetry.test.ts | 9 + .../eval/shared/telemetry.ts | 44 + evals/tts-speech-formula/eval/shared/velin.ts | 27 + .../eval/tasks/read-aloud/config.ts | 214 +++ .../eval/tasks/read-aloud/index.ts | 300 +++ .../test-objects/llm/src/read-aloud.test.ts | 9 + .../eval/test-objects/llm/src/read-aloud.ts | 40 + .../test-objects/llm/src/rewrite.velin.md | 24 + .../read-aloud/formula-speech/en.eval.ts | 5 + .../read-aloud/formula-speech/ja.eval.ts | 5 + .../read-aloud/formula-speech/ko.eval.ts | 5 + .../read-aloud/formula-speech/zh.eval.ts | 5 + .../eval/test-objects/llm/vieval.config.ts | 30 + .../eval/test-objects/raw/src/read-aloud.ts | 4 + .../read-aloud/formula-speech/en.eval.ts | 5 + .../read-aloud/formula-speech/ja.eval.ts | 5 + .../read-aloud/formula-speech/ko.eval.ts | 5 + .../read-aloud/formula-speech/zh.eval.ts | 5 + .../eval/test-objects/raw/vieval.config.ts | 3 + .../test-objects/rule/src/read-aloud.test.ts | 23 + .../eval/test-objects/rule/src/read-aloud.ts | 7 + .../test-objects/rule/src/structure.test.ts | 123 ++ .../read-aloud/formula-speech/en.eval.ts | 5 + .../read-aloud/formula-speech/ja.eval.ts | 5 + .../read-aloud/formula-speech/ko.eval.ts | 5 + .../read-aloud/formula-speech/zh.eval.ts | 5 + .../eval/test-objects/rule/vieval.config.ts | 3 + .../eval/test-objects/sre/src/read-aloud.ts | 55 + .../read-aloud/formula-speech/en.eval.ts | 5 + .../read-aloud/formula-speech/ja.eval.ts | 5 + .../read-aloud/formula-speech/ko.eval.ts | 5 + .../read-aloud/formula-speech/zh.eval.ts | 5 + .../eval/test-objects/sre/vieval.config.ts | 3 + evals/tts-speech-formula/package.json | 63 + evals/tts-speech-formula/tsconfig.json | 24 + evals/tts-speech-formula/vieval.config.ts | 14 + evals/tts-speech-formula/vitest.config.ts | 8 + packages/core-agent/README.md | 25 +- packages/core-agent/package.json | 4 + packages/core-agent/src/voice/index.ts | 4 + .../src/voice/output/response.test.ts | 41 + .../core-agent/src/voice/output/response.ts | 131 +- .../src/voice/output/speech-text.test.ts | 41 - .../src/voice/output/speech-text.ts | 113 -- .../speech-transformation/index.test.ts | 143 ++ .../output/speech-transformation/index.ts | 223 +++ .../speech-transformation/language.test.ts | 25 + .../output/speech-transformation/language.ts | 33 + .../speech-transformation/markdown.test.ts | 11 + .../output/speech-transformation/markdown.ts | 18 + .../preprocessor/index.test.ts | 54 + .../preprocessor/index.ts | 3 + .../preprocessor/math/bare-latex.test.ts | 21 + .../preprocessor/math/bare-latex.ts | 89 + .../preprocessor/math/index.ts | 33 + .../preprocessor/math/speech.test.ts | 12 + .../preprocessor/math/speech.ts | 20 + .../preprocessor/money/index.ts | 19 + .../preprocessor/money/locales.ts | 24 + .../preprocessor/money/prices.test.ts | 22 + .../preprocessor/money/prices.ts | 76 + .../preprocessor/types.ts | 37 + .../speech-transformation/segmenter.test.ts | 60 + .../output/speech-transformation/segmenter.ts | 240 +++ packages/electron-screen-capture/package.json | 2 +- packages/mathjson/README.md | 30 + packages/mathjson/package.json | 35 + packages/mathjson/src/index.ts | 58 + packages/mathjson/tsconfig.json | 14 + packages/mathjson/tsdown.config.ts | 6 + .../src/processors/tts-chunker.test.ts | 14 + .../src/processors/tts-chunker.ts | 37 +- packages/speech-transformer-chem/README.md | 47 + packages/speech-transformer-chem/package.json | 45 + .../speech-transformer-chem/src/chem.test.ts | 20 + packages/speech-transformer-chem/src/chem.ts | 287 +++ packages/speech-transformer-chem/src/index.ts | 50 + .../speech-transformer-chem/src/locale.ts | 49 + .../src/locales/en.test.ts | 44 + .../speech-transformer-chem/src/locales/en.ts | 29 + .../src/locales/ja.test.ts | 51 + .../speech-transformer-chem/src/locales/ja.ts | 31 + .../src/locales/ko.test.ts | 39 + .../speech-transformer-chem/src/locales/ko.ts | 31 + .../src/locales/names-ja.ts | 134 ++ .../src/locales/names-ko.ts | 134 ++ .../src/locales/names-zh.ts | 133 ++ .../src/locales/names.test.ts | 12 + .../src/locales/zh.test.ts | 50 + .../speech-transformer-chem/src/locales/zh.ts | 29 + .../speech-transformer-chem/tsconfig.json | 14 + .../speech-transformer-chem/tsdown.config.ts | 6 + .../speech-transformer-chem/vitest.config.ts | 8 + packages/speech-transformer-math/README.md | 65 + packages/speech-transformer-math/package.json | 39 + .../speech-transformer-math/src/extension.ts | 44 + packages/speech-transformer-math/src/heads.ts | 67 + packages/speech-transformer-math/src/index.ts | 17 + .../speech-transformer-math/src/locale.ts | 389 ++++ .../src/locales/en.test.ts | 263 +++ .../speech-transformer-math/src/locales/en.ts | 219 +++ .../src/locales/ja.test.ts | 151 ++ .../speech-transformer-math/src/locales/ja.ts | 214 +++ .../src/locales/ko.test.ts | 161 ++ .../speech-transformer-math/src/locales/ko.ts | 217 +++ .../src/locales/zh.test.ts | 190 ++ .../speech-transformer-math/src/locales/zh.ts | 218 +++ .../speech-transformer-math/src/lower.test.ts | 112 ++ packages/speech-transformer-math/src/lower.ts | 1104 +++++++++++ .../src/numbers.test.ts | 25 + .../speech-transformer-math/src/numbers.ts | 132 ++ .../speech-transformer-math/src/parse.test.ts | 45 + packages/speech-transformer-math/src/parse.ts | 431 +++++ .../src/render.test.ts | 77 + .../speech-transformer-math/src/render.ts | 462 +++++ .../src/speech.bench.ts | 49 + .../src/speech.test.ts | 70 + .../speech-transformer-math/src/speech.ts | 138 ++ packages/speech-transformer-math/src/tex.ts | 83 + .../speech-transformer-math/src/units.test.ts | 62 + packages/speech-transformer-math/src/units.ts | 62 + .../speech-transformer-math/tsconfig.json | 14 + .../speech-transformer-math/tsdown.config.ts | 6 + .../speech-transformer-math/vitest.config.ts | 8 + .../components/markdown/markdown-renderer.vue | 7 +- .../chat/components/assistant-item.vue | 2 +- .../src/composables/chat-math.test.ts | 30 + .../stage-ui/src/composables/chat-math.ts | 37 + .../stage-ui/src/stores/modules/speech.ts | 5 +- packages/testing-audio/README.md | 26 + .../cases/formula-speech/case.audio.test.ts | 108 ++ pnpm-lock.yaml | 1695 ++++++++++------- pnpm-workspace.yaml | 19 +- vitest.config.ts | 3 + 173 files changed, 17229 insertions(+), 918 deletions(-) create mode 100644 docs/ai/adr/2026-10-20-speech-text-preprocessor.md create mode 100644 evals/tts-speech-formula/.env.example create mode 100644 evals/tts-speech-formula/README.md create mode 100644 evals/tts-speech-formula/eval/cases/formula-speech/cases.ts create mode 100644 evals/tts-speech-formula/eval/cases/formula-speech/index.ts create mode 100644 evals/tts-speech-formula/eval/cases/formula-speech/languages.ts create mode 100644 evals/tts-speech-formula/eval/cases/math-bridge/fetch.ts create mode 100644 evals/tts-speech-formula/eval/cases/math-bridge/index.test.ts create mode 100644 evals/tts-speech-formula/eval/cases/math-bridge/index.ts create mode 100644 evals/tts-speech-formula/eval/cases/math-bridge/sample.json create mode 100644 evals/tts-speech-formula/eval/evaluators/grader/chem.ts create mode 100644 evals/tts-speech-formula/eval/evaluators/grader/currency.ts create mode 100644 evals/tts-speech-formula/eval/evaluators/grader/index.test.ts create mode 100644 evals/tts-speech-formula/eval/evaluators/grader/index.ts create mode 100644 evals/tts-speech-formula/eval/evaluators/grader/normalize.ts create mode 100644 evals/tts-speech-formula/eval/evaluators/grader/symbolic.ts create mode 100644 evals/tts-speech-formula/eval/evaluators/judge/calibration/calibration.json create mode 100644 evals/tts-speech-formula/eval/evaluators/judge/calibration/collect.ts create mode 100644 evals/tts-speech-formula/eval/evaluators/judge/calibration/index.html create mode 100644 evals/tts-speech-formula/eval/evaluators/judge/calibration/main.ts create mode 100644 evals/tts-speech-formula/eval/evaluators/judge/calibration/style.css create mode 100644 evals/tts-speech-formula/eval/evaluators/judge/calibration/types.ts create mode 100644 evals/tts-speech-formula/eval/evaluators/judge/calibration/vite.config.ts create mode 100644 evals/tts-speech-formula/eval/evaluators/judge/index.test.ts create mode 100644 evals/tts-speech-formula/eval/evaluators/judge/index.ts create mode 100644 evals/tts-speech-formula/eval/evaluators/judge/judge.velin.md create mode 100644 evals/tts-speech-formula/eval/evaluators/listener/index.test.ts create mode 100644 evals/tts-speech-formula/eval/evaluators/listener/index.ts create mode 100644 evals/tts-speech-formula/eval/evaluators/listener/listener.velin.md create mode 100644 evals/tts-speech-formula/eval/evaluators/notation/index.test.ts create mode 100644 evals/tts-speech-formula/eval/evaluators/notation/index.ts create mode 100644 evals/tts-speech-formula/eval/evaluators/speech/gemini-transcription.test.ts create mode 100644 evals/tts-speech-formula/eval/evaluators/speech/gemini-transcription.ts create mode 100644 evals/tts-speech-formula/eval/evaluators/speech/index.ts create mode 100644 evals/tts-speech-formula/eval/evaluators/speech/speech-models.test.ts create mode 100644 evals/tts-speech-formula/eval/evaluators/speech/speech-models.ts create mode 100644 evals/tts-speech-formula/eval/index.ts create mode 100644 evals/tts-speech-formula/eval/shared/cache-key.ts create mode 100644 evals/tts-speech-formula/eval/shared/telemetry.test.ts create mode 100644 evals/tts-speech-formula/eval/shared/telemetry.ts create mode 100644 evals/tts-speech-formula/eval/shared/velin.ts create mode 100644 evals/tts-speech-formula/eval/tasks/read-aloud/config.ts create mode 100644 evals/tts-speech-formula/eval/tasks/read-aloud/index.ts create mode 100644 evals/tts-speech-formula/eval/test-objects/llm/src/read-aloud.test.ts create mode 100644 evals/tts-speech-formula/eval/test-objects/llm/src/read-aloud.ts create mode 100644 evals/tts-speech-formula/eval/test-objects/llm/src/rewrite.velin.md create mode 100644 evals/tts-speech-formula/eval/test-objects/llm/tasks/read-aloud/formula-speech/en.eval.ts create mode 100644 evals/tts-speech-formula/eval/test-objects/llm/tasks/read-aloud/formula-speech/ja.eval.ts create mode 100644 evals/tts-speech-formula/eval/test-objects/llm/tasks/read-aloud/formula-speech/ko.eval.ts create mode 100644 evals/tts-speech-formula/eval/test-objects/llm/tasks/read-aloud/formula-speech/zh.eval.ts create mode 100644 evals/tts-speech-formula/eval/test-objects/llm/vieval.config.ts create mode 100644 evals/tts-speech-formula/eval/test-objects/raw/src/read-aloud.ts create mode 100644 evals/tts-speech-formula/eval/test-objects/raw/tasks/read-aloud/formula-speech/en.eval.ts create mode 100644 evals/tts-speech-formula/eval/test-objects/raw/tasks/read-aloud/formula-speech/ja.eval.ts create mode 100644 evals/tts-speech-formula/eval/test-objects/raw/tasks/read-aloud/formula-speech/ko.eval.ts create mode 100644 evals/tts-speech-formula/eval/test-objects/raw/tasks/read-aloud/formula-speech/zh.eval.ts create mode 100644 evals/tts-speech-formula/eval/test-objects/raw/vieval.config.ts create mode 100644 evals/tts-speech-formula/eval/test-objects/rule/src/read-aloud.test.ts create mode 100644 evals/tts-speech-formula/eval/test-objects/rule/src/read-aloud.ts create mode 100644 evals/tts-speech-formula/eval/test-objects/rule/src/structure.test.ts create mode 100644 evals/tts-speech-formula/eval/test-objects/rule/tasks/read-aloud/formula-speech/en.eval.ts create mode 100644 evals/tts-speech-formula/eval/test-objects/rule/tasks/read-aloud/formula-speech/ja.eval.ts create mode 100644 evals/tts-speech-formula/eval/test-objects/rule/tasks/read-aloud/formula-speech/ko.eval.ts create mode 100644 evals/tts-speech-formula/eval/test-objects/rule/tasks/read-aloud/formula-speech/zh.eval.ts create mode 100644 evals/tts-speech-formula/eval/test-objects/rule/vieval.config.ts create mode 100644 evals/tts-speech-formula/eval/test-objects/sre/src/read-aloud.ts create mode 100644 evals/tts-speech-formula/eval/test-objects/sre/tasks/read-aloud/formula-speech/en.eval.ts create mode 100644 evals/tts-speech-formula/eval/test-objects/sre/tasks/read-aloud/formula-speech/ja.eval.ts create mode 100644 evals/tts-speech-formula/eval/test-objects/sre/tasks/read-aloud/formula-speech/ko.eval.ts create mode 100644 evals/tts-speech-formula/eval/test-objects/sre/tasks/read-aloud/formula-speech/zh.eval.ts create mode 100644 evals/tts-speech-formula/eval/test-objects/sre/vieval.config.ts create mode 100644 evals/tts-speech-formula/package.json create mode 100644 evals/tts-speech-formula/tsconfig.json create mode 100644 evals/tts-speech-formula/vieval.config.ts create mode 100644 evals/tts-speech-formula/vitest.config.ts delete mode 100644 packages/core-agent/src/voice/output/speech-text.test.ts delete mode 100644 packages/core-agent/src/voice/output/speech-text.ts create mode 100644 packages/core-agent/src/voice/output/speech-transformation/index.test.ts create mode 100644 packages/core-agent/src/voice/output/speech-transformation/index.ts create mode 100644 packages/core-agent/src/voice/output/speech-transformation/language.test.ts create mode 100644 packages/core-agent/src/voice/output/speech-transformation/language.ts create mode 100644 packages/core-agent/src/voice/output/speech-transformation/markdown.test.ts create mode 100644 packages/core-agent/src/voice/output/speech-transformation/markdown.ts create mode 100644 packages/core-agent/src/voice/output/speech-transformation/preprocessor/index.test.ts create mode 100644 packages/core-agent/src/voice/output/speech-transformation/preprocessor/index.ts create mode 100644 packages/core-agent/src/voice/output/speech-transformation/preprocessor/math/bare-latex.test.ts create mode 100644 packages/core-agent/src/voice/output/speech-transformation/preprocessor/math/bare-latex.ts create mode 100644 packages/core-agent/src/voice/output/speech-transformation/preprocessor/math/index.ts create mode 100644 packages/core-agent/src/voice/output/speech-transformation/preprocessor/math/speech.test.ts create mode 100644 packages/core-agent/src/voice/output/speech-transformation/preprocessor/math/speech.ts create mode 100644 packages/core-agent/src/voice/output/speech-transformation/preprocessor/money/index.ts create mode 100644 packages/core-agent/src/voice/output/speech-transformation/preprocessor/money/locales.ts create mode 100644 packages/core-agent/src/voice/output/speech-transformation/preprocessor/money/prices.test.ts create mode 100644 packages/core-agent/src/voice/output/speech-transformation/preprocessor/money/prices.ts create mode 100644 packages/core-agent/src/voice/output/speech-transformation/preprocessor/types.ts create mode 100644 packages/core-agent/src/voice/output/speech-transformation/segmenter.test.ts create mode 100644 packages/core-agent/src/voice/output/speech-transformation/segmenter.ts create mode 100644 packages/mathjson/README.md create mode 100644 packages/mathjson/package.json create mode 100644 packages/mathjson/src/index.ts create mode 100644 packages/mathjson/tsconfig.json create mode 100644 packages/mathjson/tsdown.config.ts create mode 100644 packages/speech-transformer-chem/README.md create mode 100644 packages/speech-transformer-chem/package.json create mode 100644 packages/speech-transformer-chem/src/chem.test.ts create mode 100644 packages/speech-transformer-chem/src/chem.ts create mode 100644 packages/speech-transformer-chem/src/index.ts create mode 100644 packages/speech-transformer-chem/src/locale.ts create mode 100644 packages/speech-transformer-chem/src/locales/en.test.ts create mode 100644 packages/speech-transformer-chem/src/locales/en.ts create mode 100644 packages/speech-transformer-chem/src/locales/ja.test.ts create mode 100644 packages/speech-transformer-chem/src/locales/ja.ts create mode 100644 packages/speech-transformer-chem/src/locales/ko.test.ts create mode 100644 packages/speech-transformer-chem/src/locales/ko.ts create mode 100644 packages/speech-transformer-chem/src/locales/names-ja.ts create mode 100644 packages/speech-transformer-chem/src/locales/names-ko.ts create mode 100644 packages/speech-transformer-chem/src/locales/names-zh.ts create mode 100644 packages/speech-transformer-chem/src/locales/names.test.ts create mode 100644 packages/speech-transformer-chem/src/locales/zh.test.ts create mode 100644 packages/speech-transformer-chem/src/locales/zh.ts create mode 100644 packages/speech-transformer-chem/tsconfig.json create mode 100644 packages/speech-transformer-chem/tsdown.config.ts create mode 100644 packages/speech-transformer-chem/vitest.config.ts create mode 100644 packages/speech-transformer-math/README.md create mode 100644 packages/speech-transformer-math/package.json create mode 100644 packages/speech-transformer-math/src/extension.ts create mode 100644 packages/speech-transformer-math/src/heads.ts create mode 100644 packages/speech-transformer-math/src/index.ts create mode 100644 packages/speech-transformer-math/src/locale.ts create mode 100644 packages/speech-transformer-math/src/locales/en.test.ts create mode 100644 packages/speech-transformer-math/src/locales/en.ts create mode 100644 packages/speech-transformer-math/src/locales/ja.test.ts create mode 100644 packages/speech-transformer-math/src/locales/ja.ts create mode 100644 packages/speech-transformer-math/src/locales/ko.test.ts create mode 100644 packages/speech-transformer-math/src/locales/ko.ts create mode 100644 packages/speech-transformer-math/src/locales/zh.test.ts create mode 100644 packages/speech-transformer-math/src/locales/zh.ts create mode 100644 packages/speech-transformer-math/src/lower.test.ts create mode 100644 packages/speech-transformer-math/src/lower.ts create mode 100644 packages/speech-transformer-math/src/numbers.test.ts create mode 100644 packages/speech-transformer-math/src/numbers.ts create mode 100644 packages/speech-transformer-math/src/parse.test.ts create mode 100644 packages/speech-transformer-math/src/parse.ts create mode 100644 packages/speech-transformer-math/src/render.test.ts create mode 100644 packages/speech-transformer-math/src/render.ts create mode 100644 packages/speech-transformer-math/src/speech.bench.ts create mode 100644 packages/speech-transformer-math/src/speech.test.ts create mode 100644 packages/speech-transformer-math/src/speech.ts create mode 100644 packages/speech-transformer-math/src/tex.ts create mode 100644 packages/speech-transformer-math/src/units.test.ts create mode 100644 packages/speech-transformer-math/src/units.ts create mode 100644 packages/speech-transformer-math/tsconfig.json create mode 100644 packages/speech-transformer-math/tsdown.config.ts create mode 100644 packages/speech-transformer-math/vitest.config.ts create mode 100644 packages/stage-ui/src/composables/chat-math.test.ts create mode 100644 packages/testing-audio/cases/formula-speech/case.audio.test.ts diff --git a/cspell.config.yaml b/cspell.config.yaml index deb68e8a31..fc8b6772af 100644 --- a/cspell.config.yaml +++ b/cspell.config.yaml @@ -118,6 +118,7 @@ words: - Equirectangular - esaxx - esbuild + - evals - eventa - Factorio - fakemic diff --git a/docs/ai/adr/2026-10-20-speech-text-preprocessor.md b/docs/ai/adr/2026-10-20-speech-text-preprocessor.md new file mode 100644 index 0000000000..3017fa6a65 --- /dev/null +++ b/docs/ai/adr/2026-10-20-speech-text-preprocessor.md @@ -0,0 +1,293 @@ +# Speech text preprocessor with a MathJSON layer + +Status: accepted + +## Context + +`toSpeechText` in `@proj-airi/core-agent` turns a markdown reply into the text that a TTS provider reads. It reads +TeX formulas, chemistry, units, and prices as words in Chinese, English, Japanese, and Korean. + +The formula reader has one hand-written pipeline: + +```text +TeX ──parseTex──▶ TexNode (syntax) ──reader (guesses meaning and builds words) ──▶ speech +``` + +The reader mixes two jobs. It guesses what the notation means, for example that `\frac{d}{dx}` is a derivative or +that `0^{+}` is a side of a limit. It also builds the words of each language. A new language or a new reading touches +the same functions as a new rule of meaning. The chemistry reader parses a small part of mhchem by itself. + +The formula speech eval (`evals/tts-speech-formula`) showed what goes wrong when a voice gets the text: a digit, a +letter, a unit, or a structure that the voice reads in another language or that the listener cannot parse. The fixes +of 2026-10-19 raised the rule engine from 89.5% to 98.5% without TTS, and from 70% to 94% with the official TTS. Each +fix added another guess to the reader. + +Popular libraries already parse TeX and mhchem. KaTeX parses TeX for the renderer of the stage apps, about 17.8 +million downloads each week. mhchemParser is the mhchem parser of KaTeX and MathJax. Compute Engine parses TeX into +MathJSON, but it is a computer algebra system. It changes what the writer wrote: `50\%` becomes `0.5`, `3\times10^8` +becomes one number, `AB` becomes a product, and `\overline{AB}` becomes a complex conjugate. + +## Decision + +### The goal is a reading for a person who listens + +AIRI reads a reply aloud. A reading must keep the meaning of the formula and sound like a teacher who explains it. It +is not a dictation: it does not name details that only matter to a person who writes the formula down, such as the end +of a subscript or the case of a lone capital letter. A reading that changes the meaning, as silent brackets after a +minus sign do, is a defect. A reading that adds words only for a dictation is a defect too. + +### Three layers for formulas + +```text +TeX ──KaTeX parse + normalize──▶ syntax tree ──lower──▶ MathJSON (meaning) ──render + locale──▶ speech +``` + +- **Parse.** `katex.__parse` with `throwOnError: false` is the syntax parser. KaTeX is the parser of the formulas that + the user sees, so speech and display agree: an override gives rehype-katex the catalog KaTeX. The catalog pins the + exact version, because `__parse` is not a public API. KaTeX returns a command that it does not know as text, so `\foo{x}` still reads. Unbalanced braces are balanced + first, and input that KaTeX still cannot parse falls back to the source text without backslashes. +- **Normalize.** The KaTeX tree is a layout tree. Normalization joins what KaTeX splits for display, for example the + digits of `3.14` and of `10^{8}`, unwraps the groups around scripts and arguments, reads negations from the MathML + branch, and pairs delimiters. The result is the small syntax tree of `tex.ts`. +- **Lower.** One module turns the syntax tree into MathJSON. It holds every rule of meaning, for example `d/dx` as + `D`, a one-sided limit, an interval after `\in`, a product with a parenthesized sum, and a function signature. + Lowering keeps what the writer wrote. It never simplifies or evaluates. +- **Render.** One module reads MathJSON. It chooses words only by the head of each expression and the locale. It has + no rule of meaning. + +### MathJSON, extended for speech + +The tree uses the MathJSON format: a number, a string symbol, or an array whose first item is the head. Standard heads +keep their MathJSON meaning, so the eval can compare them with Compute Engine. Speech needs more than standard +MathJSON keeps, so the tree also has these heads: + +| Head | Meaning | Why standard MathJSON is not enough | +| --- | --- | --- | +| `Percent` | `50\%` | Standard parsing turns it into `0.5`. | +| `Name` | `AB`, `ABC` of geometry | Standard parsing reads juxtaposition as a product. | +| `Segment`, `Arc` | `\overline{AB}`, `\overset{\frown}{AB}` | Standard parsing has a conjugate or no meaning. | +| `Unit` | `m`, `km/h`, `m/s^2` | The unit keeps its symbol, denominator, and power. Standard MathJSON has no unit reading. | +| `Celsius` | `25^\circ\mathrm{C}` | Standard parsing turns degrees into radians. | +| `Chemistry` | the content of `\ce{}` | MathJSON has no chemistry. | +| `Delimiter`, `Group` | parentheses that the writer wrote | Lowering decides whether they are read: `Group` is read as a quantity, `Delimiter` is silent. | +| `Mapping`, `Clauses` | `f: X \to Y, x \mapsto y` | A signature and comma-separated statements read as one sentence each. | +| `Apply` | `f(x)`, `H_k(M, N)` | Standard parsing calls only known functions. Other names and their parentheses become a product. | +| `Sequence` | `r, s \in \Sigma` | The items of a list read with a pause. Without it, `r, s` reads as the product `r s`. | +| `Arrow` | `H_k(M) \to H_k(M, N)` | `To` reads "approaches". An arrow between two structures is a map and reads "to". | +| `Superscript` | `\mathbb{N}^{+}`, `f^{(n)}` | A sign or a label in parentheses is a mark, not a power. | +| `Bracket`, `Placeholder` | `[M]`, `p(\cdot)` | Square brackets around one item and an empty argument slot are read. | +| `Expectation`, `Probability`, `Variance`, `Covariance` | `\mathbb{E}_{x \sim p}[f(x)]` | Standard parsing reads `\mathbb{E}` as a set, and its brackets as a factor. The subscript is the condition. | +| `Distributed` | `x \sim p_t` | Standard `\sim` is similarity. On the right of `\sim`, a calligraphic name or a density letter is a distribution. | +| `BigUnion`, `BigIntersection` | `\bigcup_{i=1}^{n} A_i` | Standard parsing has no bounds for them, so the upper bound reads as a power. | +| `Compose`, `Assign`, `GreaterApprox`, `LessApprox`, `NotDivides`, `Proves` | `f \circ g`, `x \coloneqq y`, `n \gtrsim 10` | Lowering keeps the meaning that the renderer needs. `Ceil` and `Floor` are standard heads. | +| `Inverse`, `Ordinal` | `f^{-1}(x)`, `\cos^{-1}`, `n^{\text{th}}` | Standard parsing reads them as powers, and the listener hears 1/f. | +| `Cases`, `Case`, `Matrix`, `Determinant`, `Row` | `\begin{cases}`, `\begin{pmatrix}` | Each case reads with its condition, and each row with its cells. Without them, the cells run together. | +| `Raw` | `\ce{H2O}` before its extension lowers it | The parser keeps the source of an extension command. | + +TeX writes a call and a product the same way, as in `f(x)` and `n(n+1)`. Lowering reads parentheses after a name as a +call when they hold a comma, hold no operator, or follow a name with scripts, an accent, or a usual function letter. +Parentheses after a function command, as `\ln(x+1)`, or after upright text, as `\operatorname{tr}(A)`, are a call too. +Other parentheses after a letter multiply. A call reads "f of x" in English. In Chinese, Japanese, and Korean a simple +call keeps the reading of a product, and a call with more than one argument or with an operator says its parentheses. + +`1e-3` written without spaces is scientific notation, and the parser rewrites it as `1\times 10^{-3}`. `2e - 1` with +spaces keeps Euler's number. Digits grouped as `1,000,000` or `1{,}000` join, but one plain group, as `[1,100]`, stays a +list. + +`3\times10^{8}` stays `Multiply` and `Power`. Standard parsing folds it into one number, and the speech tree keeps +it as the writer wrote it. + +A symbol keeps its letter case. The renderer says the case only when one formula uses both cases of a letter. + +### Chemistry with mhchemParser + +`\ce{}` and `\pu{}` go through `mhchemParser.toTex`. KaTeX parses its output, and lowering turns it into `Chemistry` +nodes: species, coefficients, charges, states, hydrate dots, and arrows. The hand-written mhchem grammar is removed. + +### Streamed segments + +A speech producer used to hold the whole reply until `end()`, so an answer started to speak only after the model +ended. Markdown cannot be parsed one token at a time: a formula, a code block, a link, or a table that is cut in two +reads as its source. The pipeline now has a segmenter before the transformation: + +```text +model text ──MarkdownSegmenter──▶ segment ──toSpeechText──▶ spoken text ──TTS chunker──▶ TTS requests +``` + +- `MarkdownSegmenter` ends a segment after a sentence end or a line, only where nothing is open: fenced code, + `$$...$$`, `\[...\]`, inline code, `$...$`, `\(...\)`, link brackets, emphasis, or a table. A `$` before a price does + not open a formula. +- A line ends a segment only when the next line has started. An indented line can continue a list item, and parsed + alone it would be a code block. A line of `=` makes a heading. +- `toSpeechText` reads each segment. `before` is the spoken text of the segment before it, so a display formula that + starts a segment reads in the language of the text before it. +- A flush follows each segment, because the chunker reads one character past a sentence end. +- Constructs that refer to later text, as reference link definitions, read as text. Model replies rarely use them. + +### A preprocessor for all spoken text + +`toSpeechText` becomes a host for preprocessor plugins. A plugin owns one kind of content, for example formulas, +money, or numbers in prose. The words come from locale packs, so a contributor adds the reading of a language without +changing a plugin. + +```text +markdown ──\( \[ to $──▶ prepare hooks ──remark──▶ mdast ──formula and prose hooks──▶ spoken text + │ + ├─ money (prepare: escape prices, prose: read prices) + ├─ math (formula: KaTeX → MathJSON → render, + │ prose: bare TeX such as \ce{H2O}) + └─ future: rewrites, special pronunciations, prose numbers +``` + +A plugin has three optional hooks: + +| Hook | Input | Rule | +| --- | --- | --- | +| `prepare` | the markdown before parsing | Each plugin gets the output of the plugin before it. | +| `formula` | one TeX formula and its language | The first plugin that returns words wins. The source is kept when none does. | +| `prose` | a run of text or inline code, its language, and `readFormula` | Each plugin gets the output of the plugin before it. | + +- A plugin is trusted in-process code, as voice input plugins are, see + [voice plugins watch and finalize](2026-10-16-voice-plugins-watch-and-finalize.md). +- Installation order decides which plugin sees a node first. There is no dependency graph. The default order is + money, then math, so a price reads before the math plugin looks for bare TeX. +- `options.preprocessors` replaces the default list. A caller adds a plugin by putting it in front of + `defaultSpeechPreprocessors()` or after it. +- The rules of formulas are a rule system without the speech pipeline, so other projects can use them. They are in + three packages: + - `@proj-airi/mathjson`: the MathJSON types and helpers. + - `@proj-airi/speech-transformer-math`: parse, lower, render, units, and the locale packs. + - `@proj-airi/speech-transformer-chem`: mhchem chemistry, as an extension of the math reader. +- `MathSpeech` is a class. Each instance owns its locale packs and extensions, so two callers, or two tests, can read + differently in one process. core-agent creates one instance, `mathSpeech`, with chemistry. +- An extension (`MathSpeechExtension`) names raw commands, for example `\ce`. KaTeX does not parse them. The parser keeps + their source in a `Raw` node, the extension lowers it to its own heads, and its render handlers read those heads. +- Chemistry and units are not preprocessor plugins. They appear only inside formulas, and the formula hook already + selects the first reader. +- A locale pack (`MathLocale`) holds the words of one language for formulas and units. A chemistry locale + (`ChemLocale`) holds the words of chemistry for the same language. Money is not a formula rule, so its words are in + the money plugin of core-agent. `mathSpeech.register` + adds a pack or replaces one, and returns a function that restores the earlier pack. A language without a pack uses + English. The math plugin uses the pack of its sentence. +- The tests of each language are next to its locale pack, for example `locales/ko.test.ts`. +- Unit packs name each SI prefix and unit once. `speakUnit` composes them, so a pack does not list each pair. + +## Scope + +- `packages/core-agent/src/voice/output/`: the preprocessor host, and the math and money plugins. +- `packages/mathjson`, `packages/speech-transformer-math`, and `packages/speech-transformer-chem`: the rules of + formulas, units, and chemistry, and their locale packs for `zh`, `en`, `ja`, and `ko`. +- `evals/tts-speech-formula`: a structural comparison of the lowered MathJSON with Compute Engine, a speed benchmark, + and an optional MathBridge dataset. + +Non-goals: + +- Compute Engine in `@proj-airi/core-agent`. It stays a dependency of the eval only. +- Evaluation, simplification, or type inference of formulas. +- Thai and other languages without a locale pack. +- Changes to the TTS providers or to the speech stream. + +## Module dependencies + +```mermaid +graph LR + SpeechText[toSpeechText] --> Host[preprocessor host] + Host --> Math[math plugin] + Host --> Money[money plugin] + Math --> Instance[MathSpeech instance] + Instance --> KaTeX[katex __parse] + Instance --> Lower[lower to MathJSON] + Instance --> Chem[chemistry extension] + Chem --> Mhchem[mhchemParser] + Chem --> KaTeX + Money --> MoneyLocales[money words] + Money --> Numbers[spellNumber] + Lower --> Render[render] + Render --> Locales[locale packs] + Eval[formula speech eval] --> Lower + Eval --> CE[Compute Engine] +``` + +## Affected files + +```text +packages/core-agent/src/voice/output/speech-transformation/ + index.ts `toSpeechText`, the host of the preprocessor plugins + segmenter.ts `MarkdownSegmenter`, the safe ends of streamed markdown + language.ts primary language subtags, and the language of the words around a formula + markdown.ts code spans, and `\(...\)` and `\[...\]` delimiters + preprocessor/types.ts plugin contracts + preprocessor/math/ the mathSpeech instance, the math plugin, and bare TeX + preprocessor/money/ the money plugin, prices, and money words +packages/mathjson/src/index.ts types and helpers of the speech MathJSON +packages/speech-transformer-math/src/ + speech.ts MathSpeech: locale packs, extensions, and the call stack + extension.ts the extension contract + parse.ts KaTeX parse, raw commands, unknown commands, fallback + lower.ts KaTeX tree to MathJSON + render.ts MathJSON to words + locales/{zh,en,ja,ko}.ts locale packs, with their tests next to them +packages/speech-transformer-chem/src/ + chem.ts mhchemParser to Chemistry nodes, and their reading + index.ts the chemistry extension + locales/{zh,en,ja,ko}.ts chemistry words and compound names, with their tests next to them +evals/tts-speech-formula/ + eval/test-objects/rule/src/ structural comparison with Compute Engine + eval/cases/math-bridge/ optional MathBridge cases +``` + +## Sequence + +```mermaid +sequenceDiagram + participant Reply as markdown reply + participant Host as preprocessor host + participant Math as math plugin + participant KaTeX + participant Lower + participant Render + Reply->>Host: toSpeechText(markdown, language) + Host->>Host: prepare hooks + Host->>Math: formula hook with the TeX and the sentence language + Math->>KaTeX: __parse(tex) + alt KaTeX cannot parse the source + KaTeX-->>Math: ParseError + Math->>Math: the source text without backslashes + else + KaTeX-->>Math: KaTeX tree, with unknown commands as text + end + Math->>Lower: lower(tree) + Lower-->>Math: MathJSON + Math->>Render: render(MathJSON, locale pack) + Render-->>Host: words + Host-->>Reply: spoken text +``` + +## Measurements + +Measured on 2026-10-20 with `vitest bench` on 14 formulas of the eval: + +| Layer | Time for all formulas | +| --- | --- | +| Parse (KaTeX and normalize) | 0.25 ms | +| Lower | 0.06 ms | +| Render | 0.10 ms | +| Whole pipeline | 0.52 ms | +| The replaced hand-written parser and reader | 0.22 ms | + +The pipeline is about 2.4 times slower than the replaced reader, about 37 µs for each formula. KaTeX parsing is the +largest layer. A reply has few formulas, so the cost is small next to TTS. + +Lowering was first the slowest layer, because each symbol scanned all alias groups. An index of the spellings made it +eight times faster. + +## Test plan + +- Unit tests for each layer: parsing and fallback, lowering of each head, rendering of each head in each locale, and + the host order of plugins. +- The 188 existing speech tests stay unchanged as a regression net. A change of their expected text needs a reason in + the commit. +- The eval compares lowered MathJSON with Compute Engine on the standard heads, and lists the differences in a + snapshot. +- A benchmark measures the old reader and the new pipeline on the formulas of the eval, and reports each layer. +- The full eval runs without TTS for every test object. Runs with TTS need account credit. diff --git a/evals/tts-speech-formula/.env.example b/evals/tts-speech-formula/.env.example new file mode 100644 index 0000000000..08124f9d57 --- /dev/null +++ b/evals/tts-speech-formula/.env.example @@ -0,0 +1,41 @@ +# Endpoints and keys only. The matrix, the model names, and the voices are in +# eval/tasks/read-aloud/config.ts, and the rewrite model is in eval/test-objects/llm/vieval.config.ts. + +# The listener writes down what it heard as LaTeX. Any OpenAI-compatible endpoint works. +LISTENER_BASE_URL='https://api.openai.com/v1/' +LISTENER_API_KEY='' + +# CLIProxyAPI as provider +# LISTENER_BASE_URL='http://localhost:8317/v1' +# LISTENER_API_KEY='local-key' + +# The rewrite model of the `llm` test object. +REWRITE_BASE_URL='https://api.openai.com/v1/' +REWRITE_API_KEY='' + +# CLIProxyAPI as provider +# REWRITE_BASE_URL='http://localhost:8317/v1' +# REWRITE_API_KEY='local-key' + +# Speech recognition, needed when the `tts` axis has a provider. +# A whisper.cpp server started with `--inference-path /v1/audio/transcriptions` also works. +ASR_BASE_URL='https://api.openai.com/v1/' +ASR_API_KEY='' + +# Gemini as an ASR model of the `asr` axis. It uses the Gemini `generateContent` API, not an OpenAI-compatible endpoint. +# Set the API root: Google itself, or a proxy route that forwards the Gemini API. +ASR_GEMINI_BASE_URL='https://generativelanguage.googleapis.com/v1beta/' +ASR_GEMINI_API_KEY='' + +# One pair per TTS provider in the `tts` axis: TTS__BASE_URL and TTS__API_KEY. +TTS_OFFICIAL_BASE_URL='https://api.airi.build/api/v1/' +TTS_OFFICIAL_API_KEY='' + +TTS_AZURE_BASE_URL='http://127.0.0.1:5933/v1/' +TTS_AZURE_API_KEY='' + +# Optional, for development: case ids and languages to run. Empty runs all 67 cases in all four languages. +EVAL_CASES='' +EVAL_LANGUAGES='' +# Optional: ASR models to run instead of the default `asr` axis, for example `gemini-3.8-flash`. +EVAL_ASR='' diff --git a/evals/tts-speech-formula/README.md b/evals/tts-speech-formula/README.md new file mode 100644 index 0000000000..acfad429c7 --- /dev/null +++ b/evals/tts-speech-formula/README.md @@ -0,0 +1,200 @@ +# @proj-airi/evals-tts-speech-formula + +A [vieval](https://vieval.dev/) comparison that measures how well a listener understands the formulas that AIRI reads aloud. It covers LaTeX math, chemistry, units, and dollar amounts in Chinese, English, Japanese, and Korean. + +For each case, a test object turns a reply into spoken text. The text optionally goes through TTS and ASR. Then a listener model writes down what it heard as LaTeX, and a deterministic grader compares that answer with the reference. The listener model never grades. + +## What the suite measures + +AIRI reads a reply aloud to a person. The person must understand what the formula says, as from a teacher. The person +does not write it down. So the goal of a reading is that it keeps the meaning and sounds natural: + +- A reading that changes the meaning is wrong, for example `v - (x - \epsilon)` read as "v minus x minus epsilon". +- A reading that spells markup is wrong, for example "open parenthesis x close parenthesis" for `f(x)`. +- A reading does not need to name every detail that a dictation needs. A lone `\Sigma` can be "sigma", and a compound + subscript does not need an end word. + +The listener and the grader measure whether the meaning survives. They are a proxy: a listener model writes LaTeX, so +`formula.correct` also rewards words that only help a dictation. Read `formula.correct` as the floor of meaning, and +`formula.natural` as the reading that a person wants to hear. A Chinese, Japanese, or Korean equation reads its +substances without coefficients, as a textbook does, so `formula.correct` is 0 for it although the listener heard it. + +A judge model also reads the spoken text. It checks that the text says the meaning of the formula instead of spelling its markup. A listener model can write down "open parenthesis x close parenthesis" as easily as "of x", so correctness alone rewards spelled readings. + +```text +markdown -> test object -> spoken text -> [TTS -> ASR] -> listener model (3 votes) -> LaTeX -> grader -> formula.correct: 1 or 0 + \-> judge model (3 votes) -> natural, mixed, or spelled -> formula.natural: 1, 0.5, or 0 +``` + +The task records two named scores, and vieval derives the headline score from them: + +| Score | Grader | Value | +| --- | --- | --- | +| `formula.correct` | listener votes and the grader | 1 when a strict majority writes the formula back correctly | +| `formula.natural` | judge votes | 1 for natural, 0.5 for mixed, 0 for spelled, the median | +| `formula.pass` (primary) | derived | 1 when the reading is correct and at least mixed | + +Compare `formula.correct` for correctness and `formula.pass` for correct readings that a person wants to hear. Reports of vieval 0.9 and older cannot be read, so run them again. + +## How to use it + +1. Copy `.env.example` to `.env` and set the endpoints and keys. The file holds nothing else. +2. Set the matrix in `eval/tasks/read-aloud/config.ts`: the `tts`, `asr`, and `listener` axes, the TTS models and their voices, and the languages. +3. Compare all test objects: + +```shell +pnpm -F @proj-airi/evals-tts-speech-formula eval:compare +``` + +The script builds `@proj-airi/core-agent` first, so the `rule` test object reads the code in your working tree. It writes the comparison to `.vieval/comparison.json`. + +To run one test object and keep its full report, run its workspace config: + +```shell +pnpm -F @proj-airi/core-agent build +cd evals/tts-speech-formula +pnpm exec vieval run --config ./eval/test-objects/rule/vieval.config.ts --report-out .vieval/reports/rule +``` + +Each case writes `spoken.txt` and `result.json` as artifacts. A case with a TTS provider also writes `speech.mp3`. To compare two retained reports case by case, use `vieval report compare `. Cases match by `benchmark.case.id`, which is `||` and leaves the test object out. + +To add a TTS provider, add a `speechProviderFrom` and a `speechModelFrom` entry to `modelPlugins` in `eval/tasks/read-aloud/config.ts`, add the model id to the `tts` axis, and set the provider env in `.env`. Every provider takes the same OpenAI-compatible `/audio/speech` request. Thus, the official AIRI provider, unspeech, and other services differ only in their endpoint, model, and voices. Set `sendsLanguage: true` on a model whose endpoint takes the language of the text in `extra_body.language`, as the official AIRI provider does. The clip then gets the BCP 47 tag of the case language, for example `en-US`, and the tag is part of its cache key. + +## Structure + +```text +vieval.config.ts the comparison +eval/ + cases/formula-speech/ the dataset: 70 cases and the four languages + evaluators/ + speech/ TTS and ASR. `speech-models.ts` is a vieval plugin, see below + listener/ the listener model: Velin prompt, votes, and cached answers + judge/ the judge model: Velin prompt, votes, and cached verdicts of the reading style + grader/ the deterministic grader and its unit tests + tasks/read-aloud/ the shared task that runs one test object through the evaluators, and the shared config + shared/ environment variables, cache keys, and Velin rendering + test-objects// + vieval.config.ts the workspace of the test object + src/read-aloud.ts the adapter: reply in, spoken text out + tasks/read-aloud/formula-speech/.eval.ts + index.ts the package entry that the test objects import +``` + +| Vieval layer | In this suite | +| --- | --- | +| Comparison | `vieval.config.ts`, on the `formula-speech` dataset. | +| Workspace | One test object under `eval/test-objects/`: `raw`, `rule`, `llm`, or `sre`. | +| Project | One per test object, named `formula-speech-`. | +| Run matrix | `tts`: `none`, then TTS model ids. The `llm` object adds `rewrite`, its rewrite model. | +| Eval matrix | `listener`, the listener model, `judge`, the judge model, and `asr`, the ASR model when `tts` has a provider. They are part of the measurement, not of the test object. | +| Task | `read-aloud` in one language. During development, `EVAL_LANGUAGES=,` limits a run to some languages. | +| Case | One of the 70 cases. During development, `EVAL_CASES=,` limits a run to some cases. | + +The test objects are: + +- `raw`: the markdown as the model wrote it. This is AIRI without formula speech. +- `rule`: `toSpeechText` of `@proj-airi/core-agent`, the rule engine that AIRI ships. +- `llm`: a model rewrites the reply for speech. It is a reference for the upper bound. It is too slow for the live path. Its rewrite prompt is in its own `src/`. +- `sre`: speech-rule-engine ClearSpeak. It has no Chinese or Japanese locale, so those languages get the English reading. + +The prompts are [Velin](https://github.com/moeru-ai/velin) markdown. Keep each LaTeX example of a prompt in a code span: Velin escapes backslashes and underscores in plain text. + +To add a test object, add a directory under `eval/test-objects/` with a `ReadAloud` adapter, a config that calls `defineFormulaSpeechConfig`, and one task file per language. The comparison finds it automatically. + +`tts: none` sends the spoken text straight to the listener. It measures only what AIRI hands to TTS, independent of the provider. This is the floor that AIRI controls. A real provider shows which parts of that text the provider then reads wrong. + +## ASR + +The `asr` axis selects the speech recognizer of every TTS clip. Two are registered: + +- `apple-speech`: Apple Speech on macOS, through `@xsai-apple-speech/transcription-native`. It runs locally and needs no key. It uses one recognizer per language: `zh-CN`, `en-US`, `ja-JP`, or `ko-KR`. Install these locales in the macOS speech settings. It is the default, because `whisper-1` fell into repetition loops on short clips. +- `whisper-1`: any OpenAI-compatible `/audio/transcriptions` endpoint, set with `ASR_BASE_URL` and `ASR_API_KEY`. +- `gemini-3.8-flash`: Gemini through its `generateContent` API, set with `ASR_GEMINI_BASE_URL` and `ASR_GEMINI_API_KEY`. The base URL is the Gemini API root, for example `https://generativelanguage.googleapis.com/v1beta/`. Gemini has no `/audio/transcriptions` endpoint, so `evaluators/speech/gemini-transcription.ts` is a transcription executor that sends the clip as inline audio. + +Put several in the `asr` axis to compare them on the same clips. `EVAL_ASR=apple-speech,gemini-3.8-flash` selects them for one run. + +## Speech models plugin + +`eval/evaluators/speech/speech-models.ts` registers TTS and ASR models the way `vieval/plugins/chat-models` registers chat models. It uses only the public vieval API, so it can move into vieval as `vieval/plugins/speech-models`. + +| Chat models | Speech models | +| --- | --- | +| `ChatProviders`, `chatProviderFrom` | `SpeechProviders`, `speechProviderFrom` | +| `ChatModels`, `chatModelFrom` | `SpeechModels`, `speechModelFrom`, `transcriptionModelFrom` | +| `modelFromRun`, `modelFromEval` | `speechModelFromRun`, `speechModelFromEval`, `transcriptionModelFromRun`, `transcriptionModelFromEval` | +| `openaiFromRunContext` | `openaiSpeechFromRunContext`, `openaiTranscriptionFromRunContext`, `voiceFor` | + +The plugins register every declared model, and the matrix only selects them by id, as with chat models. The models go into the shared `config.models` registry. Provider endpoints and keys are optional at load time, so an unused provider needs no env. A selected model without its base URL fails when the task resolves it, and the error names its provider. `parameters.capability` is `speech` or `transcription`, and a lookup rejects a model with the wrong capability. A TTS model has a default `voice` and a `voices` map by language. A transcription provider can also be a `TranscriptionExecutor`: an object that returns xsai request options with its own `fetch` for each call, for example a native recognizer. Like the chat plugin, it does not send requests. The suite sends them with `@xsai/generate-speech` and `@xsai/generate-transcription`. + +## Grading + +| Mode | A case passes when | +| --- | --- | +| `symbolic` | The answer is equal after normalization, or symbolically equivalent with [Compute Engine](https://cortexjs.io/compute-engine/). `6/π²` does not pass for `π²/6`. Limits, sums, and integrals must match operator by operator. | +| `norm` | The answer is equal after normalization. | +| `chem` | The answer has the same species, coefficients, and arrow class. The order of terms is free. | +| `currency` | The answer has the same dollar amounts and the same inline math. | + +Speech cannot carry letter case, so the grader folds letter case. When the reference uses both cases of one letter, for example `a \in A`, the grader keeps case. + +Each case gets three listener answers. The case passes with a strict majority. + +Each answer also has a hearing status: `ok`, `partial`, or `unclear`. The listener sets it through structured output. An `unclear` vote scores 0 with the grading rule `unclear`, so a report tells an unintelligible clip apart from a wrong reading. The `tts_speech_formula.hearing` metric is the status of most votes. + +Each case also gets three judge verdicts on the spoken text: `natural`, `mixed`, or `spelled`. The judge never sees the answer of the listener and never judges correctness. The case takes the median verdict as the `tts_speech_formula.reading` metric, and records its score as the vieval `judge` score. The judge reads the spoken text, not the ASR transcript, so ASR errors do not count against the test object. + +### Judge calibration + +A person labels a fixed set of readings, and the share of labels that the judge matches measures the judge. The set is +`eval/evaluators/judge/calibration/calibration.json`: 15 readings for each language, spread over the test objects and +the verdicts of the judge. + +1. After a change of the judge prompt, run the eval on the cases of the set, then refresh the verdicts with + `pnpm -F @proj-airi/evals-tts-speech-formula run judge:collect`. It keeps the set. `-- --pick` picks a new one. +2. Label with `pnpm -F @proj-airi/evals-tts-speech-formula run judge:label`. It opens a local page on port 5299. Keys + 1, 2, and 3 choose natural, mixed, and spelled. The verdict of the judge shows after the label. + +On 2026-10-20 the judge matched 31 of 43 labels (72%). After the prompt took the labels into account, it matched 34 (79%). + +The `tts_speech_formula.leftover_notation` metric counts the symbols and LaTeX command names that a test object left in its spoken text, for example `^`, `/`, `$`, or `mapsto`. It needs no TTS. The test `eval/test-objects/rule/src/read-aloud.test.ts` lists these leftovers of the rule engine for every case, without any model call. + +The test `eval/test-objects/rule/src/structure.test.ts` compares the MathJSON of the rule engine (`MathSpeech.toMathJson` of `@proj-airi/speech-transformer-math`) with Compute Engine on the standard heads, for example `Divide`, `Limit`, `D`, and `Interval`. Its snapshot lists each formula where the two trees differ. It also runs without any model call. + +Answers are cached by model, prompt, transcript, and vote number. Thus, an unchanged transcript gets the same answer in every run, and run-to-run changes come only from the test object. + +## History + +The suite was first written in Python for Goal task T-157, with a sympy grader. The cached answers of the two full T-157 runs were graded again with this grader. It agrees with the sympy grader on 99.88% (edge-tts) and 99.94% (official StepFun) of the 3216 votes of each run. The remaining differences are these: + +- `\sum (1/2)^k` against `\sum 1/2^k`: Compute Engine does not prove that the two summands are equal, so this grader fails the answer. +- An answer with a trailing `?`: sympy cannot parse it, but Compute Engine ignores the character. + +The prompts were also rewritten as structured Velin markdown. The scores are therefore not comparable with the T-157 baseline. Use a new baseline from this suite. + +## Usage + +The suite records what each run spends. It records only calls that reach a provider: a cached answer makes no call, so a rerun records almost nothing. + +| Call | Record | Where | +| --- | --- | --- | +| Listener, judge, and `llm` rewrite | Token usage and latency, as vieval `inference.*` events | The inference panel of each case in the viewer, and `events.jsonl` | +| TTS | `tts_speech_formula.tts.characters_sent` | Case metrics | +| ASR | `tts_speech_formula.asr.requests` | Case metrics | + +TTS and ASR use metrics because the vieval inference events accept only chat calls. The suite records no prices. + +## When to use it + +- Use it before and after you change `math-speech` or `toSpeechText` in `@proj-airi/core-agent`. +- Use it to compare TTS providers on the same spoken text. + +## When not to use it + +- Do not use it as a unit test. It calls paid models. The unit tests of the rule engine are in `packages/core-agent`. +- Do not use its score as a measure of human understanding. The listener is a model that reads an ASR transcript, not a person who hears audio. + +## Known issues + +- In vieval 0.4.0, two `describeTask` calls in one eval file get the same task id, and the second task overwrites the cases of the first. Thus, each language has its own task file. +- `vieval compare` writes the report of each test object to a temporary directory. To keep the cases and artifacts of one test object, run its workspace config with `--report-out`. +- Vieval loads config files with Node type stripping, which does not resolve a relative import without its extension. Thus, the test object configs import the shared config through the separate package export `@proj-airi/evals-tts-speech-formula/config`. diff --git a/evals/tts-speech-formula/eval/cases/formula-speech/cases.ts b/evals/tts-speech-formula/eval/cases/formula-speech/cases.ts new file mode 100644 index 0000000000..0f61291b3f --- /dev/null +++ b/evals/tts-speech-formula/eval/cases/formula-speech/cases.ts @@ -0,0 +1,697 @@ +import type { Language } from './languages' + +/** + * How a case is graded. Each mode compares the reference with what a listener wrote down. + * + * - `symbolic`: equal after normalization, or equivalent by symbolic math. `6/π²` is not `π²/6`. + * - `norm`: equal after normalization only. Use it when two different forms must not pass. + * - `chem`: the same species, coefficients, and arrow class. + * - `currency`: the same dollar amounts, and the same inline math. + */ +export type GradingMode = 'symbolic' | 'norm' | 'chem' | 'currency' + +/** One expected item of a currency sentence: an amount in US dollars, or an inline formula. */ +export type CurrencyItem = readonly ['USD', number] | readonly ['MATH', string] + +export interface FormulaSpeechCase { + /** Stable id. Reports use it to match the same case across runs. */ + id: string + /** The group of notation that the case covers, for example `limit` or `chem`. */ + stratum: string + /** Where the case comes from: a Goal task, a textbook, or a dataset. */ + source: string + mode: GradingMode + /** Reference LaTeX, or the expected items of a currency sentence. */ + reference: string | readonly CurrencyItem[] + /** The markdown that the methods get: one text for all languages, or one sentence per language. */ + input: string | Readonly> + /** + * Overrides the letter case policy of the grader. + * + * @default derived from the reference: `sensitive` when it uses both cases of one letter + */ + letterCase?: 'fold' | 'sensitive' + note: string +} + +/** + * The stratified regression set of formula speech, 70 cases in 12 strata. + * It was first built for Goal task T-157 and is read in Chinese, English, Japanese, and Korean. + */ +export const formulaSpeechCases: readonly FormulaSpeechCase[] = [ + { + id: 'lim-basel', + stratum: 'limit', + source: 'goal/T-150/T-151/T-153', + mode: 'symbolic', + reference: String.raw`\lim_{n \to \infty}\sum_{i=1}^{n}\frac{1}{i^2}=\frac{\pi^2}{6}`, + input: String.raw`$\lim_{n \to \infty}\sum_{i=1}^{n}\frac{1}{i^2}=\frac{\pi^2}{6}$`, + note: '目标公式；6/π² 倒置陷阱', + }, + { + id: 'lim-sinx', + stratum: 'limit', + source: 'textbook', + mode: 'symbolic', + reference: String.raw`\lim_{x \to 0}\frac{\sin x}{x}=1`, + input: String.raw`$\lim_{x \to 0}\frac{\sin x}{x}=1$`, + note: '', + }, + { + id: 'lim-e', + stratum: 'limit', + source: 'textbook', + mode: 'symbolic', + reference: String.raw`\lim_{n \to \infty}\left(1+\frac{1}{n}\right)^{n}=e`, + input: String.raw`$\lim_{n \to \infty}\left(1+\frac{1}{n}\right)^{n}=e$`, + note: '', + }, + { + id: 'lim-recip', + stratum: 'limit', + source: 'textbook', + mode: 'symbolic', + reference: String.raw`\lim_{x \to \infty}\frac{1}{x}=0`, + input: String.raw`$\lim_{x \to \infty}\frac{1}{x}=0$`, + note: '', + }, + { + id: 'lim-onesided', + stratum: 'limit', + source: 'textbook', + mode: 'norm', + reference: String.raw`\lim_{x \to 0^{+}}\ln x=-\infty`, + input: String.raw`$\lim_{x \to 0^{+}}\ln x=-\infty$`, + note: '单侧极限 0^+', + }, + { + id: 'lim-fx-inf', + stratum: 'limit', + source: 'T-153', + mode: 'norm', + reference: String.raw`f(x) \to \infty`, + input: String.raw`$f(x) \to \infty$`, + note: '趋于 → 关系', + }, + { + id: 'sum-gauss', + stratum: 'sum', + source: 'textbook', + mode: 'symbolic', + reference: String.raw`\sum_{k=1}^{n} k=\frac{n(n+1)}{2}`, + input: String.raw`$\sum_{k=1}^{n} k=\frac{n(n+1)}{2}$`, + note: '', + }, + { + id: 'sum-geo', + stratum: 'sum', + source: 'textbook', + mode: 'symbolic', + reference: String.raw`\sum_{k=0}^{\infty} \frac{1}{2^{k}}=2`, + input: String.raw`$\sum_{k=0}^{\infty} \frac{1}{2^{k}}=2$`, + note: '', + }, + { + id: 'sum-sq', + stratum: 'sum', + source: 'textbook', + mode: 'symbolic', + reference: String.raw`\sum_{i=1}^{n} i^{2}`, + input: String.raw`$\sum_{i=1}^{n} i^{2}$`, + note: '', + }, + { + id: 'sum-mb-ux', + stratum: 'sum', + source: 'MathBridge(style)', + mode: 'symbolic', + reference: String.raw`\sum_{i=1}^{n} x_{i}^{2}`, + input: String.raw`$\sum_{i=1}^{n} x_{i}^{2}$`, + note: '', + }, + { + id: 'sum-prod', + stratum: 'sum', + source: 'textbook', + mode: 'symbolic', + reference: String.raw`\prod_{k=1}^{n} k=n!`, + input: String.raw`$\prod_{k=1}^{n} k=n!$`, + note: '连乘 ∏', + }, + { + id: 'frac-half', + stratum: 'fraction', + source: 'textbook', + mode: 'symbolic', + reference: String.raw`\frac{1}{2}+\frac{1}{3}=\frac{5}{6}`, + input: String.raw`$\frac{1}{2}+\frac{1}{3}=\frac{5}{6}$`, + note: '', + }, + { + id: 'frac-pi2-6', + stratum: 'fraction', + source: 'textbook', + mode: 'symbolic', + reference: String.raw`\frac{\pi^2}{6}`, + input: String.raw`$\frac{\pi^2}{6}$`, + note: '分子分母次序陷阱（韩/日/中“分之”语序）', + }, + { + id: 'frac-6-pi2', + stratum: 'fraction', + source: 'textbook', + mode: 'symbolic', + reference: String.raw`\frac{6}{\pi^2}`, + input: String.raw`$\frac{6}{\pi^2}$`, + note: '与上一条互为倒数，判分必须区分', + }, + { + id: 'frac-nested', + stratum: 'fraction', + source: 'textbook', + mode: 'symbolic', + reference: String.raw`\frac{1}{1+\frac{1}{x}}`, + input: String.raw`$\frac{1}{1+\frac{1}{x}}$`, + note: '', + }, + { + id: 'frac-mb-7-4', + stratum: 'fraction', + source: 'MathBridge#200057(简化)', + mode: 'symbolic', + reference: String.raw`q \geq \frac{7}{4}`, + input: String.raw`$q \geq \frac{7}{4}$`, + note: '', + }, + { + id: 'frac-mb-sigma', + stratum: 'fraction', + source: 'MathBridge#3000060(简化)', + mode: 'symbolic', + reference: String.raw`\sigma_{0}=\frac{f-1}{2}`, + input: String.raw`$\sigma_{0}=\frac{f-1}{2}$`, + note: '(f-1)/2 vs f-1/2 分组', + }, + { + id: 'root-quadratic', + stratum: 'root', + source: 'T-153', + mode: 'norm', + reference: String.raw`x=\frac{-b\pm\sqrt{b^2-4ac}}{2a}`, + input: String.raw`$x=\frac{-b\pm\sqrt{b^2-4ac}}{2a}$`, + note: '±', + }, + { + id: 'root-sqrt2', + stratum: 'root', + source: 'textbook', + mode: 'norm', + reference: String.raw`\sqrt{2}\approx 1.414`, + input: String.raw`$\sqrt{2}\approx 1.414$`, + note: '', + }, + { + id: 'root-cube', + stratum: 'root', + source: 'airi math-speech test', + mode: 'symbolic', + reference: String.raw`\sqrt[3]{8}=2`, + input: String.raw`$\sqrt[3]{8}=2$`, + note: '', + }, + { + id: 'root-sum', + stratum: 'root', + source: 'textbook', + mode: 'symbolic', + reference: String.raw`\sqrt{a^2+b^2}`, + input: String.raw`$\sqrt{a^2+b^2}$`, + note: '', + }, + { + id: 'root-mb-s', + stratum: 'root', + source: 'MathBridge#1200092', + mode: 'symbolic', + reference: String.raw`\sqrt{s}=14`, + input: String.raw`$\sqrt{s}=14$`, + note: '', + }, + { + id: 'rel-perp', + stratum: 'relation', + source: 'T-150/T-153', + mode: 'norm', + reference: String.raw`AB \perp CD`, + input: String.raw`$AB \perp CD$`, + note: '垂直', + }, + { + id: 'rel-parallel', + stratum: 'relation', + source: 'airi math-speech test', + mode: 'norm', + reference: String.raw`l_1 \parallel l_2`, + input: String.raw`$l_1 \parallel l_2$`, + note: '', + }, + { + id: 'rel-leq', + stratum: 'relation', + source: 'T-153', + mode: 'symbolic', + reference: String.raw`x \leq 3`, + input: String.raw`$x \leq 3$`, + note: '', + }, + { + id: 'rel-geq', + stratum: 'relation', + source: 'airi math-speech test', + mode: 'symbolic', + reference: String.raw`|x| \geq 0`, + input: String.raw`$|x| \geq 0$`, + note: '', + }, + { + id: 'rel-neq', + stratum: 'relation', + source: 'textbook', + mode: 'norm', + reference: String.raw`a \neq b`, + input: String.raw`$a \neq b$`, + note: '', + }, + { + id: 'rel-approx', + stratum: 'relation', + source: 'textbook', + mode: 'norm', + reference: String.raw`\pi \approx 3.14`, + input: String.raw`$\pi \approx 3.14$`, + note: '', + }, + { + id: 'rel-sim', + stratum: 'relation', + source: 'airi math-speech test', + mode: 'norm', + reference: String.raw`\triangle ABC \sim \triangle DEF`, + input: String.raw`$\triangle ABC \sim \triangle DEF$`, + note: '', + }, + { + id: 'rel-mb-chain', + stratum: 'relation', + source: 'MathBridge#600008', + mode: 'norm', + reference: String.raw`x_1 \leq \cdots \leq x_n`, + input: String.raw`$x_1 \leq \cdots \leq x_n$`, + note: '', + }, + { + id: 'rel-angle', + stratum: 'relation', + source: 'airi math-speech test', + mode: 'norm', + reference: String.raw`\angle ABC=90^\circ`, + input: String.raw`$\angle ABC=90^\circ$`, + note: '', + }, + { + id: 'set-in', + stratum: 'set', + source: 'T-153', + mode: 'norm', + reference: String.raw`x \in A`, + input: String.raw`$x \in A$`, + note: '', + }, + { + id: 'set-notin', + stratum: 'set', + source: 'textbook', + mode: 'norm', + reference: String.raw`\sqrt{2} \notin \mathbb{Q}`, + input: String.raw`$\sqrt{2} \notin \mathbb{Q}$`, + note: '', + }, + { + id: 'set-subseteq', + stratum: 'set', + source: 'T-153', + mode: 'norm', + reference: String.raw`A \subseteq B`, + input: String.raw`$A \subseteq B$`, + note: '', + }, + { + id: 'set-cup', + stratum: 'set', + source: 'textbook', + mode: 'norm', + reference: String.raw`A \cup B=\{1,2,3\}`, + input: String.raw`$A \cup B=\{1,2,3\}$`, + note: '', + }, + { + id: 'set-cap-empty', + stratum: 'set', + source: 'textbook', + mode: 'norm', + reference: String.raw`A \cap B=\varnothing`, + input: String.raw`$A \cap B=\varnothing$`, + note: String.raw`空集写法 \emptyset/\varnothing 等价`, + }, + { + id: 'set-interval', + stratum: 'set', + source: 'MathBridge#3000013', + mode: 'norm', + reference: String.raw`x \in (0, b]`, + input: String.raw`$x \in (0, b]$`, + note: '半开区间', + }, + { + id: 'set-builder', + stratum: 'set', + source: 'textbook', + mode: 'norm', + reference: String.raw`\{x \mid x > 0\}`, + input: String.raw`$\{x \mid x > 0\}$`, + note: '', + }, + { + id: 'chem-water', + stratum: 'chem', + source: 'T-150/T-153', + mode: 'chem', + reference: String.raw`\ce{2H2 + O2 -> 2H2O}`, + input: String.raw`$\ce{2H2 + O2 -> 2H2O}$`, + note: '', + }, + { + id: 'chem-co2', + stratum: 'chem', + source: 'textbook', + mode: 'chem', + reference: String.raw`\ce{C + O2 -> CO2}`, + input: String.raw`$\ce{C + O2 -> CO2}$`, + note: 'CO 与 CO2 必须区分', + }, + { + id: 'chem-co', + stratum: 'chem', + source: 'textbook', + mode: 'chem', + reference: String.raw`\ce{2C + O2 -> 2CO}`, + input: String.raw`$\ce{2C + O2 -> 2CO}$`, + note: '与上一条对照', + }, + { + id: 'chem-h2o2', + stratum: 'chem', + source: 'textbook', + mode: 'chem', + reference: String.raw`\ce{2H2O2 -> 2H2O + O2}`, + input: String.raw`$\ce{2H2O2 -> 2H2O + O2}$`, + note: 'H2O2 vs H2O', + }, + { + id: 'chem-lime', + stratum: 'chem', + source: 'textbook', + mode: 'chem', + reference: String.raw`\ce{CaCO3 -> CaO + CO2}`, + input: String.raw`$\ce{CaCO3 -> CaO + CO2}$`, + note: '', + }, + { + id: 'chem-ca-oh', + stratum: 'chem', + source: 'T-156', + mode: 'chem', + reference: String.raw`\ce{Ca(OH)2 + CO2 -> CaCO3 + H2O}`, + input: String.raw`$\ce{Ca(OH)2 + CO2 -> CaCO3 + H2O}$`, + note: '括号原子团', + }, + { + id: 'chem-haber', + stratum: 'chem', + source: 'textbook', + mode: 'chem', + reference: String.raw`\ce{N2 + 3H2 <=> 2NH3}`, + input: String.raw`$\ce{N2 + 3H2 <=> 2NH3}$`, + note: '可逆箭头', + }, + { + id: 'chem-hydrate', + stratum: 'chem', + source: 'T-150', + mode: 'chem', + reference: String.raw`\ce{CuSO4*5H2O}`, + input: String.raw`$\ce{CuSO4*5H2O}$`, + note: '结晶水', + }, + { + id: 'unit-g', + stratum: 'unit', + source: 'textbook', + mode: 'norm', + reference: String.raw`9.8\,\mathrm{m/s^2}`, + input: String.raw`$9.8\,\mathrm{m/s^2}$`, + note: '', + }, + { + id: 'unit-c', + stratum: 'unit', + source: 'textbook', + mode: 'norm', + reference: String.raw`3\times10^{8}\,\mathrm{m/s}`, + input: String.raw`$3\times10^{8}\,\mathrm{m/s}$`, + note: '', + }, + { + id: 'unit-temp', + stratum: 'unit', + source: 'textbook', + mode: 'norm', + reference: String.raw`25^\circ\mathrm{C}`, + input: String.raw`$25^\circ\mathrm{C}$`, + note: '', + }, + { + id: 'unit-kg', + stratum: 'unit', + source: 'textbook', + mode: 'norm', + reference: String.raw`5\,\mathrm{kg}`, + input: String.raw`$5\,\mathrm{kg}$`, + note: '', + }, + { + id: 'unit-pct', + stratum: 'unit', + source: 'airi math-speech test', + mode: 'norm', + reference: String.raw`50\%`, + input: String.raw`$50\%$`, + note: '', + }, + { + id: 'cur-two', + stratum: 'currency', + source: 'textbook', + mode: 'currency', + reference: [['USD', 3], ['USD', 5]], + input: { + zh: '苹果 $3，香蕉 $5。', + en: 'Apples cost $3 and bananas cost $5.', + ja: 'りんごは $3、バナナは $5 です。', + ko: '사과는 $3, 바나나는 $5입니다.', + }, + note: '两个 $ 会被误配成一对数学定界符', + }, + { + id: 'cur-decimal', + stratum: 'currency', + source: 'textbook', + mode: 'currency', + reference: [['USD', 19.99]], + input: { + zh: '这本书售价 $19.99。', + en: 'The book costs $19.99.', + ja: 'この本は $19.99 です。', + ko: '이 책은 $19.99입니다.', + }, + note: '', + }, + { + id: 'cur-range', + stratum: 'currency', + source: 'textbook', + mode: 'currency', + reference: [['USD', 10], ['USD', 20]], + input: { + zh: '预算在 $10 到 $20 之间。', + en: 'The budget is between $10 and $20.', + ja: '予算は $10 から $20 の間です。', + ko: '예산은 $10에서 $20 사이입니다.', + }, + note: '', + }, + { + id: 'cur-thousand', + stratum: 'currency', + source: 'textbook', + mode: 'currency', + reference: [['USD', 1200]], + input: { + zh: '月租 $1,200。', + en: 'The rent is $1,200 a month.', + ja: '家賃は月 $1,200 です。', + ko: '월세는 $1,200입니다.', + }, + note: '千分位逗号', + }, + { + id: 'cur-mixed', + stratum: 'currency', + source: 'textbook', + mode: 'currency', + reference: [['USD', 5], ['MATH', 'x^2']], + input: { + zh: '门票 $5，面积是 $x^2$。', + en: 'A ticket is $5, and the area is $x^2$.', + ja: 'チケットは $5、面積は $x^2$ です。', + ko: '입장권은 $5이고 넓이는 $x^2$입니다.', + }, + note: '货币与数学定界符混排', + }, + { + id: 'case-fold-tri', + stratum: 'case', + source: 'textbook', + mode: 'norm', + reference: String.raw`\triangle ABC`, + input: String.raw`$\triangle ABC$`, + note: '全大写，语音无法区分大小写 → 折叠', + }, + { + id: 'case-fold-ema', + stratum: 'case', + source: 'textbook', + mode: 'symbolic', + reference: 'E=mc^2', + input: '$E=mc^2$', + note: 'E/m/c 各不同字母 → 折叠', + }, + { + id: 'case-fold-fma', + stratum: 'case', + source: 'textbook', + mode: 'symbolic', + reference: 'F=ma', + input: '$F=ma$', + note: '', + }, + { + id: 'case-sens-aA', + stratum: 'case', + source: 'textbook', + mode: 'norm', + reference: String.raw`a \in A`, + input: String.raw`$a \in A$`, + note: '同一字母大小写同时出现 → 必须区分（需读出“大写”）', + }, + { + id: 'case-sens-xX', + stratum: 'case', + source: 'textbook', + mode: 'norm', + reference: String.raw`f: X \to Y, x \mapsto y`, + input: String.raw`$f: X \to Y, x \mapsto y$`, + note: 'x/X、y/Y 同时出现 → 必须区分', + }, + { + id: 'case-sens-nN', + stratum: 'case', + source: 'textbook', + mode: 'symbolic', + reference: 'N=n+1', + input: '$N=n+1$', + note: 'N 与 n 同时出现 → 必须区分', + }, + { + id: 'case-sens-omega', + stratum: 'case', + source: 'textbook', + mode: 'norm', + reference: String.raw`\Omega \neq \omega`, + input: String.raw`$\Omega \neq \omega$`, + note: '希腊字母大小写同时出现 → 必须区分', + }, + { + id: 'misc-int', + stratum: 'misc', + source: 'airi math-speech test', + mode: 'symbolic', + reference: String.raw`\int_0^1 x^2\,dx=\frac{1}{3}`, + input: String.raw`$\int_0^1 x^2\,dx=\frac{1}{3}$`, + note: '', + }, + { + id: 'misc-deriv', + stratum: 'misc', + source: 'textbook', + mode: 'norm', + reference: String.raw`\frac{d}{dx}x^{n}=nx^{n-1}`, + input: String.raw`$\frac{d}{dx}x^{n}=nx^{n-1}$`, + note: '', + }, + { + id: 'misc-neg-pow', + stratum: 'misc', + source: 'airi math-speech test', + mode: 'symbolic', + reference: String.raw`x^{-1}=\frac{1}{x}`, + input: String.raw`$x^{-1}=\frac{1}{x}$`, + note: '', + }, + { + id: 'misc-binom', + stratum: 'misc', + source: 'T-156', + mode: 'symbolic', + reference: '(a+b)^2=a^2+2ab+b^2', + input: '$(a+b)^2=a^2+2ab+b^2$', + note: '', + }, + { + id: 'prob-fm-straight', + stratum: 'probability', + source: 'AIRI reply, 2026-10-20 (straight-path Flow Matching)', + mode: 'symbolic', + reference: String.raw`x_t=(1-t)\epsilon+tx`, + input: String.raw`$$x_t=(1-t)\epsilon+tx$$`, + note: 'The brackets of (1-t) bind it to epsilon. Read silent, the formula became 1 - t epsilon.', + }, + { + id: 'prob-fm-target', + stratum: 'probability', + source: 'AIRI reply, 2026-10-20 (straight-path Flow Matching)', + mode: 'symbolic', + reference: String.raw`\mathcal{L}=\mathbb{E}\left[\left\|v_\theta(x_t,t)-(x-\epsilon)\right\|^2\right]`, + input: String.raw`$$\mathcal{L}=\mathbb{E}\left[\left\|v_\theta(x_t,t)-(x-\epsilon)\right\|^2\right]$$`, + note: 'The brackets after the minus decide the sign of epsilon. Read silent, the formula became v - x - epsilon.', + }, + { + id: 'prob-fm-loss', + stratum: 'probability', + source: 'AIRI reply, 2026-10-20 (Flow Matching, arXiv 2210.02747 eq. 5)', + mode: 'symbolic', + reference: String.raw`\mathcal{L}_{FM}(\theta) = \mathbb{E}_{t \sim \mathcal{U}[0,1],\, x \sim p_t}\left[\|v_\theta(x,t) - u_t(x)\|^2\right]`, + input: String.raw`$$\mathcal{L}_{FM}(\theta) = \mathbb{E}_{t \sim \mathcal{U}[0,1],\, x \sim p_t}\left[\|v_\theta(x,t) - u_t(x)\|^2\right]$$`, + note: '期望、范数与服从：方括号和范数线曾被读成"方括号""括号"，\\sim 曾被读成"相似于"', + }, +] diff --git a/evals/tts-speech-formula/eval/cases/formula-speech/index.ts b/evals/tts-speech-formula/eval/cases/formula-speech/index.ts new file mode 100644 index 0000000000..c8162f71c6 --- /dev/null +++ b/evals/tts-speech-formula/eval/cases/formula-speech/index.ts @@ -0,0 +1,2 @@ +export * from './cases' +export * from './languages' diff --git a/evals/tts-speech-formula/eval/cases/formula-speech/languages.ts b/evals/tts-speech-formula/eval/cases/formula-speech/languages.ts new file mode 100644 index 0000000000..b4b2315aa9 --- /dev/null +++ b/evals/tts-speech-formula/eval/cases/formula-speech/languages.ts @@ -0,0 +1,24 @@ +/** The languages that the suite reads every case in. */ +export const languages = [ + 'zh', + 'en', + 'ja', + 'ko', +] as const + +export type Language = typeof languages[number] + +/** BCP 47 code that AIRI uses for the voice of each language. */ +export const voiceLanguage: Readonly> = { + zh: 'zh-CN', + en: 'en-US', + ja: 'ja-JP', + ko: 'ko-KR', +} + +export const languageName: Readonly> = { + zh: 'Mandarin Chinese', + en: 'English', + ja: 'Japanese', + ko: 'Korean', +} diff --git a/evals/tts-speech-formula/eval/cases/math-bridge/fetch.ts b/evals/tts-speech-formula/eval/cases/math-bridge/fetch.ts new file mode 100644 index 0000000000..84c522242e --- /dev/null +++ b/evals/tts-speech-formula/eval/cases/math-bridge/fetch.ts @@ -0,0 +1,207 @@ +import process from 'node:process' + +import { createHash } from 'node:crypto' +import { createReadStream, createWriteStream } from 'node:fs' +import { access, mkdir, rename, writeFile } from 'node:fs/promises' +import { dirname } from 'node:path' +import { pipeline } from 'node:stream/promises' +import { fileURLToPath } from 'node:url' + +import katex from 'katex' + +import { DuckDBDataProtocol } from '@duckdb/duckdb-wasm' +import { connect } from '@proj-airi/duckdb-wasm' +import { getBundles } from '@proj-airi/duckdb-wasm/bundles/default-node' + +import 'katex/contrib/mhchem' + +/** + * Samples MathBridge (https://huggingface.co/datasets/Kyudan/MathBridge, MIT license) into `sample.json`. + * + * The dataset is one parquet file of 23 million rows. The script downloads the file at a pinned revision into the + * ignored `.cache/` folder, checks its SHA-256, and queries it with DuckDB. The same revision and seed always give the + * same sample. Only a new sample needs the file: the eval reads the committed `sample.json`. + * + * - The `uniform` stratum takes {@link uniformCases} gradable formulas in a seeded random order over all rows. + * - Each topic stratum takes {@link topicCases} formulas with one kind of notation, for example an expectation or a + * norm. Uniform rows rarely hold them, so the rules for them would have no cases without these strata. + * - A formula longer than {@link shortLength} characters gets a `-long` suffix, so a report can tell them apart. + * + * Run: pnpm -F @proj-airi/evals-tts-speech-formula run cases:math-bridge + * + * Call stack: + * + * main + * -> download, once per revision, then verify + * -> connect (@proj-airi/duckdb-wasm), registerFileURL with NODE_FS + * -> uniform query and topic query, see {@link gradableRows} + * -> sample.json + */ + +const dataset = 'Kyudan/MathBridge' +/** The commit of the dataset repository. Change it, and {@link sha256}, to sample a newer release. */ +const revision = '353fb2255f6959ad2940865346c38689cfce902a' +/** The SHA-256 of `train.parquet` at {@link revision}, as the `x-linked-etag` header of the LFS file reports it. */ +const sha256 = '49f5c87a7086de1130842fc80aab7bb11218bf020bf3b4e937682714a1c68b57' +const fileUrl = `https://huggingface.co/datasets/${dataset}/resolve/${revision}/train.parquet` +const cacheFile = fileURLToPath(new URL(`../../../.cache/mathbridge/train-${revision}.parquet`, import.meta.url)) + +/** The seed of the sample order. Change it to draw another sample. */ +const seed = 20261020 +/** Cases of the uniform stratum, so a run costs a known number of listener calls. */ +const uniformCases = 120 +/** Cases of each topic stratum. */ +const topicCases = 12 +/** + * Candidates per case that the query returns. Some candidates use macros of their paper and are dropped, see + * {@link usesStandardTex}, so the query returns more than it keeps. + */ +const candidateFactor = 4 +/** Longer formulas get a `-long` suffix. They are kept up to {@link maxLength}, because real replies have them. */ +const shortLength = 80 +const maxLength = 200 + +/** + * Notation that the uniform rows rarely hold, by stratum name. A formula goes to the first topic that matches. + * The patterns are RE2, the regular expressions of DuckDB. MathBridge writes a space after each token, so they allow + * spaces inside braces. + */ +const topics: readonly (readonly [string, string])[] = [ + ['expectation', String.raw`\\mathbb\s*\{\s*E\s*\}|\\mathbb\s+E\b|\\operatorname\s*\{\s*E\s*\}`], + ['variance', String.raw`\\(operatorname|mathrm)\s*\{\s*(Var|Cov)\s*\}|\\Pr\b|\\mathbb\s*\{\s*P\s*\}`], + ['norm', String.raw`\\\||\\[lr]?Vert\b`], + ['distribution', String.raw`\\sim\b`], + ['calligraphic-call', String.raw`\\mathcal\s*\{\s*[A-Z]\s*\}(\s*_\s*\{[^{}]*\})?(\s*\\left)?\s*\(`], +] + +/** One row of `sample.json`. ./index reads the same shape. */ +interface MathBridgeRow { + /** `uniform`, or a topic of {@link topics}. A `-long` suffix marks a formula longer than {@link shortLength}. */ + stratum: string + offset: number + contextBefore: string + equation: string + contextAfter: string + spokenEnglish: string +} + +/** Downloads the pinned file once. A partial download goes to a temporary name, so a broken run leaves no bad cache. */ +async function download(): Promise { + const exists = await access(cacheFile).then(() => true, () => false) + if (!exists) { + process.stdout.write(`downloading ${fileUrl}\n`) + await mkdir(dirname(cacheFile), { recursive: true }) + const response = await fetch(fileUrl) + if (!response.ok || !response.body) + throw new Error(`MathBridge download: ${response.status}`) + const partial = `${cacheFile}.partial` + await pipeline(response.body, createWriteStream(partial)) + await rename(partial, cacheFile) + } + + const hash = createHash('sha256') + await pipeline(createReadStream(cacheFile), hash) + const actual = hash.digest('hex') + if (actual !== sha256) + throw new Error(`MathBridge file ${cacheFile} has SHA-256 ${actual}, expected ${sha256}. Delete it and run again.`) +} + +/** + * The SQL of the rows that a listener can grade: notation without prose, a size that fits one spoken sentence, and a + * sentence before it. `priority` is the DuckDB hash of the row index and the seed. Sorting by it gives a random order + * that does not depend on the order of the file. The catalog pins the DuckDB version, so the same seed repeats it. + * About 88% of the first 16 million rows hold notation, but under 1% of the rest do: they are single letters, for + * example `$ A $`. Gradable rows therefore come from the first 16 million rows, and that is the data, not the hash. + */ +function gradableRows(): string { + return ` + WITH source AS ( + SELECT + file_row_number AS row_offset, + trim(regexp_replace(equation, '^\\s*\\$+|\\$+\\s*$', '', 'g')) AS tex, + equation, context_before, context_after, "spoken_English" AS spoken_english + FROM read_parquet('mathbridge.parquet', file_row_number = true) + ) + SELECT *, hash(row_offset, ${seed}) AS priority + FROM source + WHERE length(tex) BETWEEN 6 AND ${maxLength} + AND regexp_matches(tex, '[\\\\^_=<>]') + AND NOT regexp_matches(tex, '\\\\(text|mbox|label|ref|cite)|\\\\mathrm\\{[a-z]{4,}') + AND length(context_before) > 0` +} + +/** + * True when KaTeX renders the formula, so it uses only standard commands. A formula with a macro that its paper + * defines, for example `\\bxi` or `\\NN`, cannot be read by anyone without the paper, and a reply never writes it. + */ +function usesStandardTex(tex: string): boolean { + try { + katex.renderToString(tex, { throwOnError: true, strict: 'ignore' }) + return true + } + catch { + return false + } +} + +/** Escapes a value for a single-quoted SQL string. The patterns are constants, so this only doubles quotes. */ +function sqlString(value: string): string { + return `'${value.replaceAll('\'', '\'\'')}'` +} + +async function main() { + await download() + + const client = await connect({ bundles: getBundles() }) + try { + // NODE_FS reads the file from disk in pieces, so DuckDB never holds the whole gigabyte in WebAssembly memory. + await client.db.registerFileURL('mathbridge.parquet', cacheFile, DuckDBDataProtocol.NODE_FS, false) + + const topicCase = topics.map(([name, pattern]) => `WHEN regexp_matches(tex, ${sqlString(pattern)}) THEN ${sqlString(name)}`).join(' ') + const toRow = (row: Record, stratum: string): MathBridgeRow => ({ + stratum: String(row.tex).length > shortLength ? `${stratum}-long` : stratum, + offset: Number(row.row_offset), + contextBefore: String(row.context_before ?? ''), + equation: String(row.equation ?? ''), + contextAfter: String(row.context_after ?? ''), + spokenEnglish: String(row.spoken_english ?? ''), + }) + + const uniform = await client.query(`${gradableRows()} ORDER BY priority LIMIT ${uniformCases * candidateFactor}`) + const byTopic = await client.query(` + SELECT * FROM (SELECT *, CASE ${topicCase} END AS topic FROM (${gradableRows()})) + WHERE topic IS NOT NULL + QUALIFY row_number() OVER (PARTITION BY topic ORDER BY priority) <= ${topicCases * candidateFactor} + ORDER BY topic, priority`) + + // Candidates come in the seeded order. Each stratum takes the first standard formulas up to its quota. A formula + // in both strata stays in the uniform one, so the uniform stratum keeps its size. + const kept = new Set() + const sample: MathBridgeRow[] = [] + const taken = new Map() + let dropped = 0 + for (const [rows, stratumOf, quota] of [[uniform, () => 'uniform', uniformCases], [byTopic, (row: Record) => String(row.topic), topicCases]] as const) { + for (const row of rows as Record[]) { + const stratum = stratumOf(row) + if ((taken.get(stratum) ?? 0) >= quota || kept.has(Number(row.row_offset))) + continue + if (!usesStandardTex(String(row.tex))) { + dropped++ + continue + } + kept.add(Number(row.row_offset)) + taken.set(stratum, (taken.get(stratum) ?? 0) + 1) + sample.push(toRow(row, stratum)) + } + } + + await writeFile(new URL('./sample.json', import.meta.url), `${JSON.stringify({ source: `https://huggingface.co/datasets/${dataset}`, license: 'MIT', revision, seed, rows: sample }, null, 2)}\n`) + const counts = Object.entries(Object.groupBy(sample, row => row.stratum)).map(([name, rows]) => `${name} ${rows?.length ?? 0}`) + process.stdout.write(`kept ${sample.length} rows: ${counts.join(', ')}. Dropped ${dropped} with macros of their paper.\n`) + } + finally { + await client.close() + } +} + +void main() diff --git a/evals/tts-speech-formula/eval/cases/math-bridge/index.test.ts b/evals/tts-speech-formula/eval/cases/math-bridge/index.test.ts new file mode 100644 index 0000000000..1edfbd76c0 --- /dev/null +++ b/evals/tts-speech-formula/eval/cases/math-bridge/index.test.ts @@ -0,0 +1,30 @@ +import { toSpeechText } from '@proj-airi/core-agent' +import { describe, expect, it } from 'vitest' + +import { mathBridgeCases } from '.' +import { leftoverNotation } from '../../evaluators/notation' + +describe('mathBridgeCases', () => { + it('loads the committed sample with unique ids', () => { + expect(mathBridgeCases.length).toBeGreaterThan(0) + expect(new Set(mathBridgeCases.map(item => item.id)).size).toBe(mathBridgeCases.length) + }) + + // Runs the rule engine on arXiv notation without any model call. The snapshot lists what it still leaves as + // notation. `\left\ {` and `\deltat` are artifacts of the MathBridge tokenizer, not notation that a reply writes. + it('lists the notation that the rule engine leaves in arXiv formulas', () => { + const leftovers: Record = {} + for (const item of mathBridgeCases) { + const markdown = item.input as string + const found = leftoverNotation(toSpeechText(markdown, { language: 'en-US' }), markdown) + if (found.length > 0) + leftovers[item.id] = `${item.reference as string} => ${found.join(' ')}` + } + + expect(leftovers).toMatchInlineSnapshot(` + { + "mb-4602321": "\\mathbb { P } \\Gamma\\backslash \\mathbb { CH } ^9 =M => backslash", + } + `) + }) +}) diff --git a/evals/tts-speech-formula/eval/cases/math-bridge/index.ts b/evals/tts-speech-formula/eval/cases/math-bridge/index.ts new file mode 100644 index 0000000000..d4c7c402af --- /dev/null +++ b/evals/tts-speech-formula/eval/cases/math-bridge/index.ts @@ -0,0 +1,44 @@ +import type { FormulaSpeechCase } from '../formula-speech/cases' + +import { readFileSync } from 'node:fs' + +/** One row of `sample.json`, written by ./fetch. */ +interface MathBridgeRow { + /** `uniform`, or a topic such as `expectation`. A `-long` suffix marks a formula longer than 80 characters. */ + stratum: string + offset: number + contextBefore: string + equation: string + contextAfter: string + spokenEnglish: string +} + +const sample = JSON.parse(readFileSync(new URL('./sample.json', import.meta.url), 'utf8')) as { rows: MathBridgeRow[] } + +/** + * Repairs what the MathBridge tokenizer split. It writes `\{` as `\ {` and `\%` as `\ %`. In TeX `\ ` is a space and + * `%` starts a comment, so `68.3\ %` loses its percent sign and `\ { 1, 2 \ }` loses its set braces. + */ +function untokenize(tex: string): string { + return tex.replace(/\\ ([{}%])/g, '\\$1') +} + +/** + * Formulas of MathBridge (MIT license), a sample of formulas from arXiv papers of 2023 and open textbooks. + * + * The cases add realistic notation to the textbook set. Each input is the formula alone, because the listener writes + * down one formula and the grader compares formulas. The sentence around it stays in `note`. The dataset is English, + * so a run selects it with `EVAL_DATASET=math-bridge EVAL_LANGUAGES=en`. + */ +export const mathBridgeCases: readonly FormulaSpeechCase[] = sample.rows.map((row) => { + const tex = untokenize(row.equation.replace(/^\s*\$+|\$+\s*$/g, '').trim()) + return { + id: `mb-${row.offset}`, + stratum: `math-bridge-${row.stratum}`, + source: `MathBridge#${row.offset}`, + mode: 'symbolic', + reference: tex, + input: `$${tex}$`, + note: `${row.contextBefore} [formula] ${row.contextAfter}`.slice(0, 200), + } +}) diff --git a/evals/tts-speech-formula/eval/cases/math-bridge/sample.json b/evals/tts-speech-formula/eval/cases/math-bridge/sample.json new file mode 100644 index 0000000000..e51c339d75 --- /dev/null +++ b/evals/tts-speech-formula/eval/cases/math-bridge/sample.json @@ -0,0 +1,1448 @@ +{ + "source": "https://huggingface.co/datasets/Kyudan/MathBridge", + "license": "MIT", + "revision": "353fb2255f6959ad2940865346c38689cfce902a", + "seed": 20261020, + "rows": [ + { + "stratum": "uniform", + "offset": 10614451, + "contextBefore": "10 & AlO A", + "equation": "$ ^2\\Pi_ { 3/2 } $", + "contextAfter": "-- X", + "spokenEnglish": "two pi sub three halves" + }, + { + "stratum": "uniform", + "offset": 2545766, + "contextBefore": "and", + "equation": "$ \\hat { y } $", + "contextAfter": "to an explanation map", + "spokenEnglish": "y-hat." + }, + { + "stratum": "uniform", + "offset": 4200163, + "contextBefore": "Let", + "equation": "$ \\beta < 1 $", + "contextAfter": ". We introduce the exit time :", + "spokenEnglish": "Beta is less than one." + }, + { + "stratum": "uniform", + "offset": 3021182, + "contextBefore": "equipped with smooth maps", + "equation": "$ u_ { n } : \\Sigma_ { n } \\to W $", + "contextAfter": ". In", + "spokenEnglish": "u sub n, colon, Sigma sub n to W" + }, + { + "stratum": "uniform", + "offset": 12016537, + "contextBefore": "Then the root of", + "equation": "$ F ( H , A , B , k_0 , T , { \\bf B } , X ) $", + "contextAfter": "is associated with", + "spokenEnglish": "F of H, A, B, k sub 0, T, bold B, X" + }, + { + "stratum": "uniform", + "offset": 1289939, + "contextBefore": "if", + "equation": "$ \\hat { Y } =Y $", + "contextAfter": ". By the assumption on", + "spokenEnglish": "Y hat equals Y." + }, + { + "stratum": "uniform", + "offset": 1285100, + "contextBefore": "for", + "equation": "$ x \\in \\left ( M_ { \\sigma } /M^1 \\right ) ^ { \\vee } $", + "contextAfter": "and", + "spokenEnglish": "x belongs to the dual space of the quotient of the space M sub sigma by M raised to the power of 1." + }, + { + "stratum": "uniform", + "offset": 3961190, + "contextBefore": "but that", + "equation": "$ u ' ( \\theta ) $", + "contextAfter": "is either equal to", + "spokenEnglish": "you prime of theta" + }, + { + "stratum": "uniform", + "offset": 9711756, + "contextBefore": "which gives the explicit expression for the inverse", + "equation": "$ \\psi ( y ) $", + "contextAfter": "", + "spokenEnglish": "Psi of y." + }, + { + "stratum": "uniform", + "offset": 12735629, + "contextBefore": "Evaluate the fitness of individuals in", + "equation": "$ P_ { t } $", + "contextAfter": "by using the proposed cell-based weight inheritance", + "spokenEnglish": "P sub t." + }, + { + "stratum": "uniform", + "offset": 6128864, + "contextBefore": "such as", + "equation": "$ \\mathbf { E } $", + "contextAfter": ", for a fair comparison . Naturally , poisoning more items will make the attack goal easier to achieve , however , it will also cause too many side effects", + "spokenEnglish": "E bold." + }, + { + "stratum": "uniform", + "offset": 5164016, + "contextBefore": "The restricted convex term has lower bounded curvature within a particular neighborhood of", + "equation": "$ \\Theta^ * $", + "contextAfter": ", motivating us to impose a constraint on the neighborhood radius", + "spokenEnglish": "Theta star." + }, + { + "stratum": "uniform", + "offset": 6684591, + "contextBefore": "&", + "equation": "$ 45.1_ { \\pm0.5 } $", + "contextAfter": "", + "spokenEnglish": "four one, subscript plus or minus zero point five." + }, + { + "stratum": "uniform", + "offset": 9393813, + "contextBefore": "Let", + "equation": "$ \\Sigma $", + "contextAfter": "be a surface of negative Euler characteristic , homeomorphic to a closed surface , possibly with a finite number of points removed . In this paper , we present a", + "spokenEnglish": "Sigma." + }, + { + "stratum": "uniform", + "offset": 15529913, + "contextBefore": "Let", + "equation": "$ \\hat \\mu $", + "contextAfter": "be the transverse measure on", + "spokenEnglish": "mu hat." + }, + { + "stratum": "uniform", + "offset": 2508752, + "contextBefore": "and", + "equation": "$ \\alpha_t < 1 $", + "contextAfter": ", thus :", + "spokenEnglish": "Alpha sub t less than one." + }, + { + "stratum": "uniform", + "offset": 4158564, + "contextBefore": "Its presence is independent of individual magnitudes of both", + "equation": "$ \\Omega_q $", + "contextAfter": "and", + "spokenEnglish": "Omega q." + }, + { + "stratum": "uniform", + "offset": 4634896, + "contextBefore": "and with the single particle dispersion relation", + "equation": "$ \\varepsilon_k $", + "contextAfter": "of the isolated SSH model defined by the magnitude of the Bloch vector ,", + "spokenEnglish": "epsilon sub k." + }, + { + "stratum": "uniform", + "offset": 6015283, + "contextBefore": "another possible Laplacian pyramid usage is to firstly smooth out the unwanted texture details from", + "equation": "$ L_ { k-1 } $", + "contextAfter": "by performing joint bilateral filtering using the initial upsampling output", + "spokenEnglish": "L sub k minus 1." + }, + { + "stratum": "uniform", + "offset": 519492, + "contextBefore": "Then for any", + "equation": "$ k=1 , \\dots , d $", + "contextAfter": ", we have", + "spokenEnglish": "k equals one comma dot dot dot comma d" + }, + { + "stratum": "uniform", + "offset": 6401833, + "contextBefore": "with a lift", + "equation": "$ \\tilde { x } $", + "contextAfter": "to", + "spokenEnglish": "x tilde." + }, + { + "stratum": "uniform", + "offset": 13433106, + "contextBefore": "where", + "equation": "$ \\eta \\in ( 0,1 ) $", + "contextAfter": "is the step size . The server then broadcasts the updated parameters to all", + "spokenEnglish": "Eta belongs to the open interval from zero to one." + }, + { + "stratum": "uniform", + "offset": 2534572, + "contextBefore": "morphism and", + "equation": "$ y_1 , \\dots , y_m $", + "contextAfter": "be a", + "spokenEnglish": "y sub 1, dot dot dot, y sub m." + }, + { + "stratum": "uniform", + "offset": 10183014, + "contextBefore": "The average value of the square-norm of", + "equation": "$ \\psi_n $", + "contextAfter": "is normalized to", + "spokenEnglish": "Psi sub n." + }, + { + "stratum": "uniform", + "offset": 9215908, + "contextBefore": "&", + "equation": "$ 68.3\\ % $", + "contextAfter": "&", + "spokenEnglish": "68.3 percent." + }, + { + "stratum": "uniform", + "offset": 7175591, + "contextBefore": "with", + "equation": "$ \\overline { x } $", + "contextAfter": ". Thus , we have that", + "spokenEnglish": "x bar." + }, + { + "stratum": "uniform", + "offset": 15452024, + "contextBefore": "where", + "equation": "$ \\rho_M $", + "contextAfter": "is defined as pull back of", + "spokenEnglish": "rho sub M." + }, + { + "stratum": "uniform", + "offset": 7443697, + "contextBefore": "which only allows edges", + "equation": "$ X_i \\rightarrow X_j $", + "contextAfter": ", where", + "spokenEnglish": "X sub i goes to X sub j." + }, + { + "stratum": "uniform", + "offset": 13478849, + "contextBefore": "the internal structure of", + "equation": "$ \\mathcal { S } $", + "contextAfter": "is encoded into the dynamics of each", + "spokenEnglish": "ess" + }, + { + "stratum": "uniform", + "offset": 15299684, + "contextBefore": "we identify state samples within", + "equation": "$ \\mathcal { Z } _s $", + "contextAfter": "that possess at least one control input leading to", + "spokenEnglish": "Z sub s." + }, + { + "stratum": "uniform", + "offset": 7472601, + "contextBefore": "and", + "equation": "$ \\beta=0.01 $", + "contextAfter": ". Without reaction mobility", + "spokenEnglish": "Beta equals zero point zero one." + }, + { + "stratum": "uniform", + "offset": 13092254, + "contextBefore": "on", + "equation": "$ \\Gamma $", + "contextAfter": "in Figure", + "spokenEnglish": "Gamma." + }, + { + "stratum": "uniform", + "offset": 221274, + "contextBefore": "We initialize", + "equation": "$ \\sigma $", + "contextAfter": "with the word", + "spokenEnglish": "Sigma." + }, + { + "stratum": "uniform", + "offset": 2794309, + "contextBefore": "/Ly", + "equation": "$ \\alpha $", + "contextAfter": "and", + "spokenEnglish": "Alpha." + }, + { + "stratum": "uniform", + "offset": 9071341, + "contextBefore": "of", + "equation": "$ \\cap_ { i=1 } ^r E_i $", + "contextAfter": ". Then we have local coordinates", + "spokenEnglish": "Intersection from i equals 1 to r of E sub i." + }, + { + "stratum": "uniform", + "offset": 1627293, + "contextBefore": "and tunneling current", + "equation": "$ I_ { \\rm t } = 100 $", + "contextAfter": "~pA .", + "spokenEnglish": "I sub t equals 100." + }, + { + "stratum": "uniform", + "offset": 11223418, + "contextBefore": "we see that the probability of", + "equation": "$ \\mathcal { Q } _n $", + "contextAfter": "occurring in", + "spokenEnglish": "Q sub n." + }, + { + "stratum": "uniform", + "offset": 8256122, + "contextBefore": "the early delivery policy schedules the items in", + "equation": "$ \\mathcal { S } _j $", + "contextAfter": "to be delivered at time", + "spokenEnglish": "S sub j." + }, + { + "stratum": "uniform", + "offset": 1162100, + "contextBefore": "to", + "equation": "$ \\mathbf { r } ' $", + "contextAfter": ". With respect to wavefunction", + "spokenEnglish": "r prime." + }, + { + "stratum": "uniform", + "offset": 4748410, + "contextBefore": "Shades indicate the", + "equation": "$ \\pm1\\sigma $", + "contextAfter": "confidence intervals . The fits are performed by minimizing the weighted , squared orthogonal residuals to the line . For the normal E sample , low-velocity-dispersion galaxies are discarded using", + "spokenEnglish": "Plus or minus one sigma." + }, + { + "stratum": "uniform", + "offset": 14831819, + "contextBefore": "on", + "equation": "$ \\Omega $", + "contextAfter": "; tensor fields", + "spokenEnglish": "Omega." + }, + { + "stratum": "uniform", + "offset": 7222018, + "contextBefore": "and", + "equation": "$ \\sigma_ { G } $", + "contextAfter": "describing the transit depth , central wavelength , and standard deviation wavelength of each excess absorption feature . We excluded telluric bands red-ward of the helium triplet , as well", + "spokenEnglish": "sigma G." + }, + { + "stratum": "uniform", + "offset": 352234, + "contextBefore": "and the optimal prior probabilities are", + "equation": "$ p ( a ) \\approx ( 0.2965,0.3518,0.3518 ) $", + "contextAfter": ". Additionally ,", + "spokenEnglish": "p of a approximately equals open parenthesis 0.2965, 0.3518, 0.3518 close parenthesis." + }, + { + "stratum": "uniform", + "offset": 7178784, + "contextBefore": "and an integer", + "equation": "$ k \\ge 1 $", + "contextAfter": ".", + "spokenEnglish": "k is greater than or equal to 1." + }, + { + "stratum": "uniform", + "offset": 10499854, + "contextBefore": "indicate", + "equation": "$ B_ { \\rm c2 } $", + "contextAfter": "at 85", + "spokenEnglish": "B sub c2." + }, + { + "stratum": "uniform", + "offset": 13555897, + "contextBefore": "with task weightings", + "equation": "$ \\lambda_i $", + "contextAfter": ".", + "spokenEnglish": "Lambda i." + }, + { + "stratum": "uniform", + "offset": 1128781, + "contextBefore": "we vary its value to", + "equation": "$ \\ { 1 , 2 , 3 , 4 , 5 ,6\\ } $", + "contextAfter": ". As shown in Fig.~", + "spokenEnglish": "Spoken English: The set containing the elements 1, 2, 3, 4, 5, and 6." + }, + { + "stratum": "uniform", + "offset": 9093279, + "contextBefore": "is negative still holds when", + "equation": "$ 1 \\leq t < \\eta $", + "contextAfter": ". Therefore", + "spokenEnglish": "One less than or equal to t, less than eta." + }, + { + "stratum": "uniform", + "offset": 7700814, + "contextBefore": "the full lines correspond to azimuthal angles", + "equation": "$ \\phi=0 $", + "contextAfter": ", the dotted line to", + "spokenEnglish": "phi equals zero." + }, + { + "stratum": "uniform", + "offset": 8790613, + "contextBefore": "We fix", + "equation": "$ T < \\infty $", + "contextAfter": ",", + "spokenEnglish": "T is less than infinity." + }, + { + "stratum": "uniform", + "offset": 8422271, + "contextBefore": "given a desired velocity for the end effector", + "equation": "$ \\dot { x } _ { ee } $", + "contextAfter": ". The second platform has access to both its kinematic chain and the model of the viscous fluidic model that it inhabits :", + "spokenEnglish": "dot x sub e e" + }, + { + "stratum": "uniform", + "offset": 14417290, + "contextBefore": "& head height &", + "equation": "$ x_ { 16 } $", + "contextAfter": "& head circumference", + "spokenEnglish": "x sub 16." + }, + { + "stratum": "uniform", + "offset": 5868723, + "contextBefore": "The reason why we insist on changing the norm of", + "equation": "$ H^ { -1 } $", + "contextAfter": "to", + "spokenEnglish": "H to the power of negative one." + }, + { + "stratum": "uniform", + "offset": 11293457, + "contextBefore": "decrease as", + "equation": "$ x_ { k-1 } $", + "contextAfter": "increases from one odd positive integer to another .", + "spokenEnglish": "x sub k minus one." + }, + { + "stratum": "uniform", + "offset": 262170, + "contextBefore": "which is the difference of two linear combinations of", + "equation": "$ \\boldsymbol { y } $", + "contextAfter": "and hence is itself a linear combination of", + "spokenEnglish": "y." + }, + { + "stratum": "uniform", + "offset": 13614478, + "contextBefore": "where", + "equation": "$ C_ { \\mu\\nu\\sigma\\rho } $", + "contextAfter": "is the", + "spokenEnglish": "C mu nu sigma rho: None" + }, + { + "stratum": "uniform", + "offset": 5901150, + "contextBefore": "there exists an integer", + "equation": "$ p_a ( x ) $", + "contextAfter": "such that", + "spokenEnglish": "p sub a of x." + }, + { + "stratum": "uniform", + "offset": 3748747, + "contextBefore": "of a graph~", + "equation": "$ G= ( V , E ) $", + "contextAfter": "is a map~", + "spokenEnglish": "G equals open parenthesis V comma E close parenthesis." + }, + { + "stratum": "uniform", + "offset": 552372, + "contextBefore": "Starting from the canonical basis", + "equation": "$ ( \\boldsymbol { e } _1 , \\dots , \\boldsymbol { e } _ { n+1 } ) $", + "contextAfter": "of", + "spokenEnglish": "Open parenthesis, bold e sub 1, comma, dot dot dot, bold e sub n plus 1, close parenthesis." + }, + { + "stratum": "uniform", + "offset": 937380, + "contextBefore": "target sentence and", + "equation": "$ y^ { ( k ) } _ { < i } $", + "contextAfter": "are the previously generated words of the target sentence", + "spokenEnglish": "y to the power of k sub i less than" + }, + { + "stratum": "uniform", + "offset": 7588942, + "contextBefore": "to", + "equation": "$ \\ket { g } $", + "contextAfter": ".", + "spokenEnglish": "Ket g." + }, + { + "stratum": "uniform", + "offset": 740038, + "contextBefore": "of", + "equation": "$ X_ { u } ( t+1 ) $", + "contextAfter": "for each of 101 uniformly spaced points", + "spokenEnglish": "X sub u, open parenthesis, t plus 1, close parenthesis." + }, + { + "stratum": "uniform", + "offset": 616205, + "contextBefore": "we introduce the notation", + "equation": "$ \\theta_ { k+1 } = \\theta_k + \\alpha_k g_k + \\alpha_k w_k $", + "contextAfter": ", where", + "spokenEnglish": "Theta sub k plus 1 equals theta sub k plus alpha sub k g sub k plus alpha sub k w sub k." + }, + { + "stratum": "uniform", + "offset": 4014641, + "contextBefore": "indexed by a mask supported outside of", + "equation": "$ \\gamma_ { s_\\beta } $", + "contextAfter": ", and for each such pair of masks", + "spokenEnglish": "gamma sub s sub beta" + }, + { + "stratum": "uniform", + "offset": 5565465, + "contextBefore": "&", + "equation": "$ \\frac { 7 } { 60 } \\mathbf { k } \\cdot \\mathbf { k } ^\\prime $", + "contextAfter": "", + "spokenEnglish": "Fraction 7 over 60 times bold k dot bold k prime." + }, + { + "stratum": "uniform", + "offset": 1115210, + "contextBefore": "where", + "equation": "$ m=\\lceil \\alpha \\rceil $", + "contextAfter": ". These derivatives differ , but obey some common general trends . Figures", + "spokenEnglish": "m equals the ceiling of alpha." + }, + { + "stratum": "uniform", + "offset": 2722416, + "contextBefore": "two upper thick solid curves are for", + "equation": "$ 1.4M_\\odot $", + "contextAfter": "neutron stars , with the lower curve for cooling only while the upper one includes the maximum vortex creep heating . Two curves between these thick solid curves show the", + "spokenEnglish": "1.4 times the mass of the Sun." + }, + { + "stratum": "uniform", + "offset": 8534820, + "contextBefore": "For a fiducial cluster redshift of", + "equation": "$ z = 0.5 $", + "contextAfter": ", we determined that 3 hours of integration time in", + "spokenEnglish": "z equals zero point five." + }, + { + "stratum": "uniform", + "offset": 1450389, + "contextBefore": "The assumptions for all of these results are that there is a jump from", + "equation": "$ \\alpha = ( a_1 , a_2 ) $", + "contextAfter": "to", + "spokenEnglish": "Alpha equals open parenthesis a sub 1, comma a sub 2 close parenthesis." + }, + { + "stratum": "uniform", + "offset": 3064229, + "contextBefore": "the following equalities are satisfied for all", + "equation": "$ k\\in [ K ] $", + "contextAfter": "and", + "spokenEnglish": "k belongs to the set of K." + }, + { + "stratum": "uniform", + "offset": 14001482, + "contextBefore": "Exchanging the two variables in the Whitney polynomial of collection of hypermaps", + "equation": "$ ( \\sigma , \\alpha ) $", + "contextAfter": "of genus zero yields the Whitney polynomial of its dual", + "spokenEnglish": "sigma, alpha." + }, + { + "stratum": "uniform", + "offset": 741475, + "contextBefore": "The combined results covering excitation energies up to 14 MeV clearly demonstrate the transition between the constant temperature and the Fermi gas regimes at", + "equation": "$ \\approx 7 $", + "contextAfter": "MeV . The experimental NLDs were used to estimate the role of the Porter-Thomas fluctuations in assessing the generalised Brink-Axel hypothesis below the neutron separation energy in", + "spokenEnglish": "Approximately seven." + }, + { + "stratum": "uniform", + "offset": 1548559, + "contextBefore": "Define", + "equation": "$ \\varphi\\coloneqq\\iota_\\infty\\circ\\mu_\\infty $", + "contextAfter": ". Then", + "spokenEnglish": "Phi is defined as iota infinity composed with mu infinity." + }, + { + "stratum": "uniform", + "offset": 1547557, + "contextBefore": "have moved to LN", + "equation": "$ _ { ( n,2 ) } ( P_t ) $", + "contextAfter": "at", + "spokenEnglish": "subscript n comma 2 of P sub t" + }, + { + "stratum": "uniform", + "offset": 13998683, + "contextBefore": "such that for all regular dominant element", + "equation": "$ \\lambda\\in \\Lambda $", + "contextAfter": ",", + "spokenEnglish": "Lambda belongs to the set of lambdas." + }, + { + "stratum": "uniform", + "offset": 4831787, + "contextBefore": "and set the sets", + "equation": "$ \\mathcal { R } _i $", + "contextAfter": "to include", + "spokenEnglish": "R sub i." + }, + { + "stratum": "uniform", + "offset": 6904738, + "contextBefore": "Whenever a theory has a collection of higher-form symmetries", + "equation": "$ G^ { ( p ) } $", + "contextAfter": ",", + "spokenEnglish": "G raised to the power of p." + }, + { + "stratum": "uniform", + "offset": 6655716, + "contextBefore": "If", + "equation": "$ \\hat { H } ( t ) $", + "contextAfter": "possesses the PT-symmetry ,", + "spokenEnglish": "H-hat of t." + }, + { + "stratum": "uniform", + "offset": 5041888, + "contextBefore": "~", + "equation": "$ \\omega_ { \\rm BD } > 2230 $", + "contextAfter": "and", + "spokenEnglish": "Omega sub R B, greater than 2 2 3." + }, + { + "stratum": "uniform", + "offset": 11432946, + "contextBefore": "over a sequence of time stamps", + "equation": "$ t=1 , \\cdots , T $", + "contextAfter": "with a window size of", + "spokenEnglish": "t equals 1, dot dot dot, T" + }, + { + "stratum": "uniform", + "offset": 12272241, + "contextBefore": "&", + "equation": "$ f_i , g_i $", + "contextAfter": "& none & none &", + "spokenEnglish": "f sub i, g sub i" + }, + { + "stratum": "uniform", + "offset": 11121926, + "contextBefore": "as", + "equation": "$ \\omega \\to 0 $", + "contextAfter": ".", + "spokenEnglish": "Omega goes to zero." + }, + { + "stratum": "uniform", + "offset": 12772538, + "contextBefore": "The tendencies of relative", + "equation": "$ T_ { 0i } $", + "contextAfter": "and", + "spokenEnglish": "T sub zero i." + }, + { + "stratum": "uniform", + "offset": 1913164, + "contextBefore": "we produce a noisy one by sampling from the distribution", + "equation": "$ vMF ( \\mu , \\kappa ) $", + "contextAfter": ", where", + "spokenEnglish": "vee-ef mu, kappawe can infer that" + }, + { + "stratum": "uniform", + "offset": 6279623, + "contextBefore": "with a non-trivial uncertainty interval is mandatory if and only if there is a hyperedge", + "equation": "$ S \\in E $", + "contextAfter": "with", + "spokenEnglish": "S belongs to E." + }, + { + "stratum": "uniform", + "offset": 1264807, + "contextBefore": "The following proposition says", + "equation": "$ \\hat { \\varphi } $", + "contextAfter": "is continuous .", + "spokenEnglish": "phi hat." + }, + { + "stratum": "uniform", + "offset": 3519845, + "contextBefore": "It follows from this and the series representation for", + "equation": "$ A ( s , z ; \\pi ) $", + "contextAfter": "given by the first equality in", + "spokenEnglish": "A of s, z, pi" + }, + { + "stratum": "uniform", + "offset": 1167649, + "contextBefore": "we may assume that every component of", + "equation": "$ \\Sigma_ { B_T } $", + "contextAfter": ", that is vertical over", + "spokenEnglish": "Sigma B sub T." + }, + { + "stratum": "uniform", + "offset": 4289099, + "contextBefore": "When a cloud instead has", + "equation": "$ \\gtrsim 10 $", + "contextAfter": "neighbours in close proximity , the median distance to the nearest baryon-rich satellite drops to", + "spokenEnglish": "Greater than or approximately equal to ten." + }, + { + "stratum": "uniform", + "offset": 982870, + "contextBefore": "Integrating out", + "equation": "$ \\widehat { a } $", + "contextAfter": "enforces", + "spokenEnglish": "a-hat." + }, + { + "stratum": "uniform", + "offset": 407118, + "contextBefore": "Set", + "equation": "$ a : = ( 1+ \\bar { \\kappa } _0 R_0 \\sqrt { \\delta } ) ^2 $", + "contextAfter": ". First , for", + "spokenEnglish": "1 plus kappa bar sub-zero times R sub-zero times the square root of delta." + }, + { + "stratum": "uniform", + "offset": 3599218, + "contextBefore": "where", + "equation": "$ c_ { \\theta } $", + "contextAfter": "is some constant in", + "spokenEnglish": "c sub theta" + }, + { + "stratum": "uniform", + "offset": 8969236, + "contextBefore": "when", + "equation": "$ \\theta > 1 $", + "contextAfter": ", when", + "spokenEnglish": "Theta is greater than one." + }, + { + "stratum": "uniform", + "offset": 13707144, + "contextBefore": "fixing an inner product on", + "equation": "$ \\mathfrak { g } $", + "contextAfter": ", such as the Frobenius inner product", + "spokenEnglish": "g in Fraktur font." + }, + { + "stratum": "uniform", + "offset": 11052164, + "contextBefore": "s", + "equation": "$ ^ { -1 } $", + "contextAfter": "throughout.", + "spokenEnglish": "to the power of negative one." + }, + { + "stratum": "uniform", + "offset": 2317842, + "contextBefore": "to", + "equation": "$ p\\vdash tr ( y ) $", + "contextAfter": "are removed , which means that", + "spokenEnglish": "p turnstile tr y" + }, + { + "stratum": "uniform", + "offset": 7107825, + "contextBefore": "fundamental Dirac fermions", + "equation": "$ \\Psi_i $", + "contextAfter": ",", + "spokenEnglish": "Psi i." + }, + { + "stratum": "uniform", + "offset": 1876665, + "contextBefore": "'' is consistent for", + "equation": "$ \\kappa=\\omega_1 $", + "contextAfter": "by", + "spokenEnglish": "kappa equals omega sub 1." + }, + { + "stratum": "uniform", + "offset": 14313471, + "contextBefore": "так що", + "equation": "$ f_n ( x ) = x^2 , \\ |x| \\leq n $", + "contextAfter": ", причому", + "spokenEnglish": "f sub n of x equals x squared, absolute value of x less than or equal to n." + }, + { + "stratum": "uniform", + "offset": 2123535, + "contextBefore": "The Dirac matrix", + "equation": "$ \\mathbf { D } _1 $", + "contextAfter": "and its eigenvalues and eigenvectors are calculated . As the non-zero eigenvalues of", + "spokenEnglish": "D sub 1." + }, + { + "stratum": "uniform", + "offset": 8715240, + "contextBefore": "a basic way is to choose respectively random row and column index sets", + "equation": "$ \\mathcal { I } $", + "contextAfter": "and", + "spokenEnglish": "capital I." + }, + { + "stratum": "uniform", + "offset": 11092729, + "contextBefore": "and", + "equation": "$ \\theta_ { \\rm view } $", + "contextAfter": "constrained by SED fitting in", + "spokenEnglish": "Theta sub view." + }, + { + "stratum": "uniform", + "offset": 1727433, + "contextBefore": "if and only if", + "equation": "$ \\Delta_k < 0 $", + "contextAfter": ", where", + "spokenEnglish": "Delta sub k is less than zero." + }, + { + "stratum": "uniform", + "offset": 10206034, + "contextBefore": "Note that this bound will only be valid when there exists", + "equation": "$ \\delta_1 , \\delta_2 $", + "contextAfter": "with low values of", + "spokenEnglish": "Delta one, delta two." + }, + { + "stratum": "uniform", + "offset": 608470, + "contextBefore": "This in particular implies", + "equation": "$ \\pi\\nmid m $", + "contextAfter": "for any", + "spokenEnglish": "pi does not divide m." + }, + { + "stratum": "uniform", + "offset": 1727034, + "contextBefore": "Each literal", + "equation": "$ l_ { i } $", + "contextAfter": "is associated with a positive integer coefficient", + "spokenEnglish": "l sub i." + }, + { + "stratum": "uniform", + "offset": 12663600, + "contextBefore": "em", + "equation": "$ a_t=a_ { t ' } $", + "contextAfter": "", + "spokenEnglish": "a sub t equals a sub t prime" + }, + { + "stratum": "uniform", + "offset": 6100476, + "contextBefore": "states that if", + "equation": "$ Q=\\mathbb { R } $", + "contextAfter": ", then", + "spokenEnglish": "Q equals the set of real numbers." + }, + { + "stratum": "uniform", + "offset": 12858590, + "contextBefore": "using Doppler parameter of", + "equation": "$ \\sim 50 $", + "contextAfter": "km s", + "spokenEnglish": "Approximately fifty." + }, + { + "stratum": "uniform", + "offset": 8649182, + "contextBefore": "the only unbounded quantifier in", + "equation": "$ \\varphi_1 $", + "contextAfter": "is the one existentially quantifying", + "spokenEnglish": "phi sub one." + }, + { + "stratum": "uniform", + "offset": 1047438, + "contextBefore": "equation with", + "equation": "$ \\sigma < 1 $", + "contextAfter": ". This form is chosen to mesh with previous work : if", + "spokenEnglish": "Sigma is less than one." + }, + { + "stratum": "uniform", + "offset": 7811323, + "contextBefore": "will be defined with a different value of", + "equation": "$ \\beta_k $", + "contextAfter": "so that price of the union bound is finite", + "spokenEnglish": "Beta sub k." + }, + { + "stratum": "uniform", + "offset": 13203154, + "contextBefore": "again about two orders of magnitude less that the variation in", + "equation": "$ \\gamma $", + "contextAfter": "or", + "spokenEnglish": "Gamma." + }, + { + "stratum": "uniform", + "offset": 15873220, + "contextBefore": "copies of", + "equation": "$ \\Sigma_1 $", + "contextAfter": "and use induction to complete the proof .", + "spokenEnglish": "Sigma one." + }, + { + "stratum": "uniform", + "offset": 14666412, + "contextBefore": "these secondary variables are the quadratic monomials", + "equation": "$ x_i x_j $", + "contextAfter": ", see Fig.~", + "spokenEnglish": "x sub i times x sub j." + }, + { + "stratum": "uniform", + "offset": 1546576, + "contextBefore": "then", + "equation": "$ P_ { 1 } = P_ { 0 } $", + "contextAfter": ", and hence , by the fairness of the scheduler , a robot such that it is not at", + "spokenEnglish": "P sub 1 equals P sub 0." + }, + { + "stratum": "uniform", + "offset": 2128356, + "contextBefore": "it is not a", + "equation": "$ \\lambda $", + "contextAfter": "-abstraction , so there is no", + "spokenEnglish": "Lambda." + }, + { + "stratum": "uniform", + "offset": 10480913, + "contextBefore": "the upper bound is tight for some choice of", + "equation": "$ \\ { p , q , r\\ } $", + "contextAfter": "whenever the Kapranov degree vanishes .", + "spokenEnglish": "curly brace p, comma q, comma r" + }, + { + "stratum": "uniform", + "offset": 15691531, + "contextBefore": "00", + "equation": "$ _ { ( +0.65 ) } $", + "contextAfter": "", + "spokenEnglish": "sub plus zero point six five." + }, + { + "stratum": "uniform", + "offset": 14640960, + "contextBefore": "by", + "equation": "$ \\lambda_1 , \\lambda_2 , \\ldots , \\lambda_N $", + "contextAfter": ", namely :", + "spokenEnglish": "Lambda one, lambda two, ellipsis, lambda N." + }, + { + "stratum": "calligraphic-call", + "offset": 4377213, + "contextBefore": "The local topological property of the characteristic function", + "equation": "$ \\mathcal { F } ( z ) $", + "contextAfter": "can be reflected by the local topological number at its", + "spokenEnglish": "F of z." + }, + { + "stratum": "calligraphic-call", + "offset": 13450953, + "contextBefore": "the bounds exhibit a decaying rate in the order of", + "equation": "$ \\mathcal { O } ( 1/\\sqrt { n } ) $", + "contextAfter": ", where", + "spokenEnglish": "big O of one over the square root of n." + }, + { + "stratum": "calligraphic-call", + "offset": 14336637, + "contextBefore": "we subtract the delayed features", + "equation": "$ \\mathcal { D } ( \\boldsymbol { f } ,1 ) $", + "contextAfter": "from the input visual features", + "spokenEnglish": "D of f, 1" + }, + { + "stratum": "calligraphic-call", + "offset": 3879866, + "contextBefore": "is", + "equation": "$ \\mathcal { B } ( \\mathcal { H } ) $", + "contextAfter": "-measurable if and only if", + "spokenEnglish": "B of H" + }, + { + "stratum": "calligraphic-call", + "offset": 9673989, + "contextBefore": "The space", + "equation": "$ \\mathcal { O } ( b_1 , \\dots , b_L ) $", + "contextAfter": "is a subspace of", + "spokenEnglish": "Big O of b sub 1, dot dot dot, b sub L." + }, + { + "stratum": "calligraphic-call", + "offset": 3406965, + "contextBefore": "the auto-tuning makes", + "equation": "$ \\mathcal { H } ( \\pi ) $", + "contextAfter": "converge to", + "spokenEnglish": "H of pi" + }, + { + "stratum": "calligraphic-call", + "offset": 2878229, + "contextBefore": "the quantum state transition can be modelled by quantum channel", + "equation": "$ \\mathcal { E } ( \\rho ) =\\sum_ { i\\in [ k ] } K_i\\rho K_i^\\dag $", + "contextAfter": ". We define the tensor product of quantum channels", + "spokenEnglish": "Mathematical calligraphic E of rho equals the sum over i in the set of k of K sub i rho K sub i dagger." + }, + { + "stratum": "calligraphic-call", + "offset": 8575803, + "contextBefore": "and", + "equation": "$ \\mathcal { B } ( B \\to \\ , K^ { * } \\ , \\nu\\ , \\bar { \\nu } ) $", + "contextAfter": ", respectively . It is evident that the allowed ranges of", + "spokenEnglish": "Mathcal B, open parenthesis, B goes to K star nu, bar nu, close parenthesis." + }, + { + "stratum": "calligraphic-call", + "offset": 14864258, + "contextBefore": "or", + "equation": "$ \\mathcal { O } ( 4 ) $", + "contextAfter": "and the", + "spokenEnglish": "big O of four." + }, + { + "stratum": "calligraphic-call", + "offset": 7789233, + "contextBefore": "computed from the VD observation is of the", + "equation": "$ \\mathcal { O } ( 1 ) $", + "contextAfter": ", suggesting the existence of a second galaxy lacking DM", + "spokenEnglish": "big O of one." + }, + { + "stratum": "calligraphic-call", + "offset": 10344114, + "contextBefore": "can not be executed in parallel and has a time complexity of", + "equation": "$ \\mathcal { O } ( \\mathcal { T } HW ) $", + "contextAfter": ", i.e . the number of pixels in", + "spokenEnglish": "big O of T H W" + }, + { + "stratum": "calligraphic-call", + "offset": 14431594, + "contextBefore": "&", + "equation": "$ \\mathcal { U } ( 1e-3 , 100 . ) $", + "contextAfter": "& RV jitter", + "spokenEnglish": "U one e minus three, one hundred dot." + }, + { + "stratum": "distribution", + "offset": 9416790, + "contextBefore": "By redshift of", + "equation": "$ z \\sim 1 $", + "contextAfter": ", the three structures have become approximately virialized galaxy clusters . One can also see the filamentary structure surrounding the clusters , particularly visible in the lensing maps at low", + "spokenEnglish": "z is distributed according to a standard normal distribution." + }, + { + "stratum": "distribution", + "offset": 2650564, + "contextBefore": "If the previous result can be extended to the range of", + "equation": "$ \\sim 10^ { 12 } $", + "contextAfter": "~m , the present work improves the existing upper bound by as much as 12 orders of magnitude . The obtained constraint on", + "spokenEnglish": "Approximately ten to the power of twelve." + }, + { + "stratum": "distribution", + "offset": 1280471, + "contextBefore": "implying that the depths of the considered bands would be reduced to", + "equation": "$ \\sim 60 $", + "contextAfter": "", + "spokenEnglish": "Approximately sixty." + }, + { + "stratum": "distribution", + "offset": 9159376, + "contextBefore": "which equate to roughly", + "equation": "$ \\sim 1 \\ \\mathrm { mm } $", + "contextAfter": "depth bins . The conversion from the DOI to the true depth is done by bounding the DOI values for the full-energy peak and the centroid . This is done", + "spokenEnglish": "sim one millimeter" + }, + { + "stratum": "distribution", + "offset": 8664719, + "contextBefore": "leads to a decrease of the merger efficiency by a factor", + "equation": "$ \\sim 2 $", + "contextAfter": ". Moreover , the merger efficiency increases with the cluster density following a power-law with slope", + "spokenEnglish": "Approximately two." + }, + { + "stratum": "distribution", + "offset": 5000487, + "contextBefore": "found that", + "equation": "$ \\sim 85\\ % $", + "contextAfter": "of SGRBs are observed in", + "spokenEnglish": "Approximately 85 percent." + }, + { + "stratum": "distribution", + "offset": 4867559, + "contextBefore": "% and", + "equation": "$ a\\sim b $", + "contextAfter": "if", + "spokenEnglish": "a is approximately equal to b" + }, + { + "stratum": "distribution", + "offset": 4996457, + "contextBefore": "converging in law to a complex normal random variable", + "equation": "$ Z\\sim\\mathcal { CN } ( 0 , \\frac { 1 } { 2\\lambda } ) $", + "contextAfter": "as", + "spokenEnglish": "Z is distributed as a normal distribution with mean zero and variance one over two lambda." + }, + { + "stratum": "distribution", + "offset": 11649356, + "contextBefore": "and reddening contribute an additional", + "equation": "$ \\sim 0.06 $", + "contextAfter": "~mag to the error budget , the final distance modulus becomes", + "spokenEnglish": "Approximately zero point zero six." + }, + { + "stratum": "distribution", + "offset": 14924247, + "contextBefore": "and", + "equation": "$ J_ { ij } \\sim Poi ( p_ { ij } ) $", + "contextAfter": ". Then , %", + "spokenEnglish": "J sub i j is distributed as Po sub i j." + }, + { + "stratum": "distribution", + "offset": 13959242, + "contextBefore": "at", + "equation": "$ \\sim 45 $", + "contextAfter": "", + "spokenEnglish": "Approximately 45." + }, + { + "stratum": "distribution", + "offset": 13012594, + "contextBefore": "we find that orbital circularization of disk particles results in a", + "equation": "$ \\sim 10\\ % $", + "contextAfter": "mean entropy gain to the disk , although our results show that this effect can produce most of the disk 's vapor as disk particles are shifted further into pure", + "spokenEnglish": "Approximately ten percent." + }, + { + "stratum": "expectation", + "offset": 5630930, + "contextBefore": "points of", + "equation": "$ \\mathbb { E } $", + "contextAfter": ", one of which is", + "spokenEnglish": "E" + }, + { + "stratum": "expectation", + "offset": 10788867, + "contextBefore": "Here", + "equation": "$ \\mathbb { E } $", + "contextAfter": "represents the conditional expectation of", + "spokenEnglish": "E" + }, + { + "stratum": "expectation", + "offset": 7588277, + "contextBefore": "we pick once and for all an", + "equation": "$ \\mathbb E $", + "contextAfter": "-extension", + "spokenEnglish": "E" + }, + { + "stratum": "expectation", + "offset": 9000673, + "contextBefore": "% % % % % % % % % % % Let", + "equation": "$ ( \\mathscr { C } , \\mathbb { E } , \\mathfrak { s } ) $", + "contextAfter": "be an extriangulated category . The axioms above ensure that any extriangle induce six-term exact sequences after application of some covariant or contravariant Hom-functor . In particular in any conflation", + "spokenEnglish": "Open parenthesis, script C, comma, blackboard bold E, comma, script s, close parenthesis." + }, + { + "stratum": "expectation", + "offset": 10853365, + "contextBefore": "as", + "equation": "$ B ( \\mathbb E ) $", + "contextAfter": ", and", + "spokenEnglish": "B of E" + }, + { + "stratum": "expectation", + "offset": 1672071, + "contextBefore": "let", + "equation": "$ \\mathbb { E } _ { p_ { G } } $", + "contextAfter": "be the expectation taken with respect to product measure with density", + "spokenEnglish": "Expectation with respect to the distribution p sub G." + }, + { + "stratum": "expectation", + "offset": 4346518, + "contextBefore": "where", + "equation": "$ \\mathbb { E } $", + "contextAfter": "is a contour path enclosing the whole region of the", + "spokenEnglish": "E" + }, + { + "stratum": "expectation", + "offset": 8846615, + "contextBefore": "three- or fourfold translative tile in", + "equation": "$ \\mathbb { E } ^2 $", + "contextAfter": ". Then what about the projection of a three-dimensional fivefold translative tile", + "spokenEnglish": "E squared." + }, + { + "stratum": "expectation", + "offset": 764409, + "contextBefore": "where", + "equation": "$ \\sigma_ { 2 } ^2 = \\mathbb { E } [ \\phi_ { d_ { \\eta^\\ast } } ^2 ] $", + "contextAfter": "is the semiparametric efficiency bound . Therefore we obtain in the end", + "spokenEnglish": "Sigma squared sub 2 equals the expected value of phi squared sub d sub eta asterisk." + }, + { + "stratum": "expectation", + "offset": 7855753, + "contextBefore": "it would be interesting to identify", + "equation": "$ \\mathbb E [ N_\\alpha^T ( v ) ] $", + "contextAfter": ". In the same remark , we saw that for", + "spokenEnglish": "Expectation of N sub alpha transpose of v." + }, + { + "stratum": "expectation", + "offset": 330087, + "contextBefore": "And", + "equation": "$ \\mathbb { L } ^\\infty ( X ) = \\mathbb { E } \\left [ X\\right ] $", + "contextAfter": ",", + "spokenEnglish": "L-infinity of X equals the expected value of X." + }, + { + "stratum": "expectation", + "offset": 9033551, + "contextBefore": "Note that", + "equation": "$ P_r = \\mathbb { E } \\ { \\lvert q_m ( t ) \\rvert^2 \\ } $", + "contextAfter": ". % Please refer to Fig . 3 in", + "spokenEnglish": "The value of P sub r is equal to the expectation of the absolute value of q sub m of t squared." + }, + { + "stratum": "norm", + "offset": 12096080, + "contextBefore": "where", + "equation": "$ \\left \\| \\cdot \\right \\| $", + "contextAfter": "denotes the 2-norm of a vector ,", + "spokenEnglish": "norm of dot." + }, + { + "stratum": "norm", + "offset": 842616, + "contextBefore": "we have also", + "equation": "$ \\Vert b - c \\Vert \\geq \\Delta $", + "contextAfter": ". Finally , we conclude", + "spokenEnglish": "The magnitude of the vector b minus c is greater than or equal to delta." + }, + { + "stratum": "norm", + "offset": 7440886, + "contextBefore": "We first prove that the Hilbert-Schmidt norm of", + "equation": "$ \\lVert D_ { \\mu_1 , \\mu_2 } ^1 \\rVert $", + "contextAfter": "is bounded uniformly in", + "spokenEnglish": "the norm of D sub mu one comma mu two superscript one" + }, + { + "stratum": "norm", + "offset": 6683662, + "contextBefore": "The prior analyses apply to generic", + "equation": "$ \\|\\cdot\\|_2 $", + "contextAfter": "-nonexpansive or contractive operators and", + "spokenEnglish": "norm of dot, subscript 2." + }, + { + "stratum": "norm", + "offset": 8274825, + "contextBefore": "where", + "equation": "$ \\|\\cdot\\| $", + "contextAfter": "is an appropriate norm measuring the distance between", + "spokenEnglish": "norm of dot." + }, + { + "stratum": "norm", + "offset": 12038442, + "contextBefore": "we see that the bound on", + "equation": "$ \\|E_1\\|_F $", + "contextAfter": "is cubic in", + "spokenEnglish": "Norm of E sub 1 sub F" + }, + { + "stratum": "norm", + "offset": 6027028, + "contextBefore": "We interchangeably use the notations", + "equation": "$ \\|x\\|_2 $", + "contextAfter": "and", + "spokenEnglish": "the norm of x subscript 2." + }, + { + "stratum": "norm", + "offset": 10691868, + "contextBefore": "where", + "equation": "$ \\|\\cdot\\| $", + "contextAfter": "denotes the norm induced by", + "spokenEnglish": "norm of dot." + }, + { + "stratum": "norm", + "offset": 2423990, + "contextBefore": "let", + "equation": "$ \\|\\bm { x } \\|_1 : =\\sum_ { k=1 } ^K |x_k| $", + "contextAfter": ",", + "spokenEnglish": "The norm of vector x, subscript 1, is equal to the sum from k equals 1 to K of the absolute value of x subscript k." + }, + { + "stratum": "norm", + "offset": 7859309, + "contextBefore": "inducing the norm", + "equation": "$ \\|\\cdot\\| $", + "contextAfter": ". Since", + "spokenEnglish": "norm of dot." + }, + { + "stratum": "norm", + "offset": 14468741, + "contextBefore": "the norm inequality", + "equation": "$ \\|x\\|_1\\leq\\|x\\|_p $", + "contextAfter": "holds , since %", + "spokenEnglish": "The norm of x sub 1 is less than or equal to the norm of x sub p." + }, + { + "stratum": "norm", + "offset": 10767879, + "contextBefore": "we can also obtain a local reliable bound for", + "equation": "$ \\|u-u_ { \\theta } \\|_U $", + "contextAfter": "through the following Assumption :", + "spokenEnglish": "Norm of u minus u sub theta, norm sub U." + }, + { + "stratum": "variance", + "offset": 4602321, + "contextBefore": "The point in", + "equation": "$ \\mathbb { P } \\Gamma\\backslash \\mathbb { CH } ^9 =M $", + "contextAfter": "corresponding to the triangulation is the", + "spokenEnglish": "Mathematical double-struck P, Gamma, backslash, mathematical double-struck CH raised to the power of 9, equals M." + }, + { + "stratum": "variance", + "offset": 12302759, + "contextBefore": "Let 's extend", + "equation": "$ \\bar { \\mathbb { P } } $", + "contextAfter": "again with", + "spokenEnglish": "bar P bold." + }, + { + "stratum": "variance", + "offset": 1433602, + "contextBefore": "where", + "equation": "$ \\mathbb { P } _ { m } ( \\cdot |H_ { i , t } ^ { m } ) $", + "contextAfter": "represents this probability is conditional on history", + "spokenEnglish": "Probability of m given the event H sub i, t superscript m." + }, + { + "stratum": "variance", + "offset": 13101408, + "contextBefore": "morphism to", + "equation": "$ \\mathbb { P } ^1 $", + "contextAfter": ", then it is also birational to a quintic in", + "spokenEnglish": "P one." + }, + { + "stratum": "variance", + "offset": 2991800, + "contextBefore": "in", + "equation": "$ \\mathcal { M } _ { \\mathcal { N } , \\mathbb { P } } $", + "contextAfter": ". % , including distinct extensions from a matroid", + "spokenEnglish": "M sub N, P" + }, + { + "stratum": "variance", + "offset": 1690856, + "contextBefore": "and in", + "equation": "$ L^p ( \\tilde { \\Omega } , \\tilde { \\mathbb { P } } ) $", + "contextAfter": ".", + "spokenEnglish": "L to the power of p, open parenthesis, tilde Omega, comma, tilde mathbb P, close parenthesis." + }, + { + "stratum": "variance", + "offset": 7146627, + "contextBefore": "denotes the total variation distance between", + "equation": "$ \\mathbb { P } $", + "contextAfter": "and", + "spokenEnglish": "P" + }, + { + "stratum": "variance", + "offset": 7984398, + "contextBefore": "of the sheaf cohomology of", + "equation": "$ \\mathbb { P } ^n $", + "contextAfter": ".", + "spokenEnglish": "P to the power of n." + }, + { + "stratum": "variance", + "offset": 4536903, + "contextBefore": "assuming that", + "equation": "$ X_h\\in L^1 ( \\mathbb { P } ) $", + "contextAfter": "and that the cdf of", + "spokenEnglish": "X sub h belongs to L to the power of 1, open parenthesis, capital P, close parenthesis." + }, + { + "stratum": "variance", + "offset": 2022535, + "contextBefore": "For this assumption it suffices to assume that", + "equation": "$ ( \\Omega , \\mathcal { F } , \\mathbb { P } ) $", + "contextAfter": "is separable see e.g . ,", + "spokenEnglish": "Open parenthesis, Omega, comma, script F, comma, double-struck P, close parenthesis." + }, + { + "stratum": "variance", + "offset": 11561649, + "contextBefore": "measures the volume of the", + "equation": "$ \\mathbb { P } ^5 $", + "contextAfter": "factor . In the limit", + "spokenEnglish": "P to the power of 5." + }, + { + "stratum": "variance", + "offset": 7928090, + "contextBefore": "Then", + "equation": "$ \\mathbb { P } $", + "contextAfter": "is a hyperfield , called the", + "spokenEnglish": "P" + } + ] +} diff --git a/evals/tts-speech-formula/eval/evaluators/grader/chem.ts b/evals/tts-speech-formula/eval/evaluators/grader/chem.ts new file mode 100644 index 0000000000..e8cd6de083 --- /dev/null +++ b/evals/tts-speech-formula/eval/evaluators/grader/chem.ts @@ -0,0 +1,136 @@ +import { stripWrappers } from './normalize' + +type Arrow = '<=>' | '->' + +/** The reversible class is tried first, because `<=>` also contains `=` and `->`-like text. */ +const arrows: readonly (readonly [RegExp, Arrow])[] = [ + [/<=>|<->|\\rightleftharpoons|\\leftrightarrow|⇌|↔|\\rightleftarrows/, '<=>'], + // `=` is the arrow of Chinese textbooks. + [/->|→|\\rightarrow|\\longrightarrow|\\to(?![a-z])|=(?!=)|\\xrightarrow\{[^}]*\}|-\[[^\]]*\]>/, '->'], +] + +const uppercase = /\p{Lu}/u +const lowercase = /\p{Ll}/u +const digit = /\d/ + +function readCount(text: string, start: number): [count: number, end: number] { + let end = start + while (end < text.length && digit.test(text[end]!)) + end++ + return [end > start ? Number(text.slice(start, end)) : 1, end] +} + +/** Counts the elements of a formula from `start` to the closing bracket of the current group. */ +function readGroup(text: string, start: number): [counts: Map, end: number] { + const counts = new Map() + let index = start + while (index < text.length) { + const char = text[index]! + if (char === '(' || char === '[') { + const [inner, afterGroup] = readGroup(text, index + 1) + const [multiplier, afterCount] = readCount(text, afterGroup) + for (const [element, count] of inner) + counts.set(element, (counts.get(element) ?? 0) + count * multiplier) + index = afterCount + } + else if (char === ')' || char === ']') { + return [counts, index + 1] + } + else if (uppercase.test(char)) { + const element = char + (index + 1 < text.length && lowercase.test(text[index + 1]!) ? text[index + 1]! : '') + const [count, afterCount] = readCount(text, index + element.length) + counts.set(element, (counts.get(element) ?? 0) + count) + index = afterCount + } + else { + index++ + } + } + return [counts, index] +} + +/** + * Keys one species by its element counts, so `Ca(OH)2` and `CaO2H2` are the same species. + * A hydrate such as `CuSO4*5H2O` adds the water of crystallization. + */ +function speciesKey(formula: string): string { + const total = new Map() + for (const part of formula.split(/\\cdot|\\bullet|[*·.•⋅]/)) { + const trimmed = part.trim() + const [multiplier, start] = readCount(trimmed, 0) + const [counts] = readGroup(trimmed.slice(start), 0) + for (const [element, count] of counts) + total.set(element, (total.get(element) ?? 0) + count * multiplier) + } + return [...total].map(([element, count]) => `${element}${count}`).sort().join(' ') +} + +/** Parses an equation into its arrow class and one multiset of `(coefficient, species)` per side. */ +function parseChem(source: string): { arrow: Arrow | undefined, sides: Map[] } { + let text = stripWrappers(source) + text = /\\ce\{(.*)\}/s.exec(text)?.[1] ?? text + // State marks, gas and precipitate marks, and text notes do not change the species. + text = text.replace(/\\text\{[^}]*\}|\^\{?[\\a-z]*\}?(?![0-9+-])|\s+v(?=\s|$)|↑|↓|\(g\)|\(s\)|\(aq\)|\(l\)/g, ' ') + text = text.replace(/_\{?(\d+)\}?/g, '$1') + + let arrow: Arrow | undefined + let parts = [text] + for (const [pattern, name] of arrows) { + const match = pattern.exec(text) + if (match) { + arrow = name + parts = [text.slice(0, match.index), text.slice(match.index + match[0].length)] + break + } + } + + const sides = parts.map((side) => { + const terms = new Map() + for (const term of side.split(/\s\+\s|\s*\+\s*(?=[0-9A-Z(])/)) { + const trimmed = term.trim() + if (!trimmed) + continue + const match = /^(\d+)\s*(?=[A-Z([])(.+)$/.exec(trimmed) + const coefficient = match ? Number(match[1]) : 1 + const formula = match ? match[2]! : trimmed + if (!/[A-Z]/.test(formula)) + continue + const key = `${coefficient}|${speciesKey(formula)}` + terms.set(key, (terms.get(key) ?? 0) + 1) + } + return terms + }) + return { arrow, sides } +} + +function sameMultiset(left: Map, right: Map): boolean { + return left.size === right.size && [...left].every(([key, count]) => right.get(key) === count) +} + +/** The species of a side without their coefficients. */ +function speciesOf(side: Map): Set { + return new Set([...side.keys()].map(key => key.split('|')[1]!)) +} + +/** + * Tells whether two chemical equations have the same species, coefficients, and arrow class. + * The order of terms is free. A wrong species (CO for CO2), coefficient, or arrow class fails. + * + * A hypothesis without any coefficient passes with the same species and arrow. Chinese, Japanese, and Korean readings + * name the substances of an equation without coefficients, as a textbook does, so the listener cannot hear them. A + * hypothesis that writes coefficients must write the right ones. + */ +export function sameChemistry(reference: string, hypothesis: string): boolean { + const expected = parseChem(reference) + const actual = parseChem(hypothesis) + if (expected.arrow !== actual.arrow || expected.sides.length !== actual.sides.length) + return false + if (expected.sides.every((side, index) => sameMultiset(side, actual.sides[index]!))) + return true + const withoutCoefficients = actual.sides.every(side => [...side.keys()].every(key => key.startsWith('1|'))) + return withoutCoefficients && expected.sides.every((side, index) => { + const left = speciesOf(side) + const right = speciesOf(actual.sides[index]!) + return left.size === right.size && [...left].every(species => right.has(species)) + }) +} diff --git a/evals/tts-speech-formula/eval/evaluators/grader/currency.ts b/evals/tts-speech-formula/eval/evaluators/grader/currency.ts new file mode 100644 index 0000000000..77e6f09a51 --- /dev/null +++ b/evals/tts-speech-formula/eval/evaluators/grader/currency.ts @@ -0,0 +1,31 @@ +import type { CurrencyItem } from '../../cases/formula-speech/cases' + +import { canonicalize } from './normalize' + +/** A dollar amount with a sign before it (`$5`, `US$5`, `USD 5`), or a word after it (`5 dollars`, `5美元`). */ +const amount = /(?:\$|USD|US\$)\s*(\d[\d,]*(?:\.\d+)?)|(\d[\d,]*(?:\.\d+)?)\s*(?:dollars?|USD|美元|美金|ドル|달러)/g + +/** + * Lists the US dollar amounts in a sentence, sorted. + * + * @example + * dollarAmounts('Apples cost \\$3, and rent is 1,200 dollars') + * // => [3, 1200] + */ +export function dollarAmounts(text: string): number[] { + const normalized = text.replaceAll('\\$', '$').replaceAll('＄', '$') + return [...normalized.matchAll(amount)] + .map(match => Number((match[1] ?? match[2]!).replaceAll(',', ''))) + .sort((left, right) => left - right) +} + +/** Tells whether a sentence keeps every dollar amount, and no other amount, and every inline formula of the reference. */ +export function sameCurrencySentence(reference: readonly CurrencyItem[], hypothesis: string): boolean { + const expected = reference.flatMap(item => item[0] === 'USD' ? [item[1]] : []).sort((left, right) => left - right) + const actual = dollarAmounts(hypothesis) + if (actual.length !== expected.length || actual.some((value, index) => value !== expected[index])) + return false + + const sentence = canonicalize(hypothesis) + return reference.every(item => item[0] !== 'MATH' || sentence.includes(canonicalize(item[1]))) +} diff --git a/evals/tts-speech-formula/eval/evaluators/grader/index.test.ts b/evals/tts-speech-formula/eval/evaluators/grader/index.test.ts new file mode 100644 index 0000000000..152cf4618c --- /dev/null +++ b/evals/tts-speech-formula/eval/evaluators/grader/index.test.ts @@ -0,0 +1,87 @@ +import type { FormulaSpeechCase } from '../../cases/formula-speech/cases' + +import { describe, expect, it } from 'vitest' + +import { grade, majorityScore } from '.' +import { formulaSpeechCases } from '../../cases/formula-speech/cases' + +const cases = new Map(formulaSpeechCases.map(item => [item.id, item])) + +describe('grade', () => { + // Ported from the T-157 scorer self test. It covers the three judge defects that the manual calibration found, + // letter case, strict chemistry, currency, and units. + it.each([ + ['lim-sinx', String.raw`\lim_{x \to 0}\frac{x}{\sin x}=1`, 0, 'review #9: symbolic evaluated both limits to 1; operators must match'], + ['lim-basel', String.raw`\lim_{n\to\infty}\sum_{i=1}^{n}\frac{1}{i}=\frac{\pi^2}{6}`, 0, 'summand 1/i is not 1/i^2'], + ['lim-onesided', String.raw`\lim_{x \to 0^+} \ln x = -\infty`, 1, 'review #6: 0^{+} vs 0^+ (natural `}}` was read as an escaped brace)'], + ['set-notin', String.raw`\sqrt{2}\notin\mathbb{Q}`, 1, 'review #96/#104: \\notin\\mathbb{Q} without a space'], + ['set-cup', String.raw`A \cup B = \{1,2,3\}`, 1, 'escaped set braces still compare'], + ['chem-hydrate', String.raw`\ce{CuSO4 \cdot 5H2O}`, 1, 'review B#7: \\cdot hydrate dot'], + ['lim-onesided', String.raw`\lim_{x \to +0} \ln x = -\infty`, 1, 'Japanese x→+0 is 0^+'], + ['lim-basel', String.raw`\lim_{n \to \infty}\sum_{i=1}^{n}\frac{1}{i^2}=\frac{6}{\pi^2}`, 0, 'T-153 ko 6/π² inversion must fail'], + ['lim-basel', String.raw`\lim_{n\to\infty}\sum_{i=1}^{n}\frac{1}{i^{2}}=\frac{\pi^{2}}{6}`, 1, 'same formula, other braces'], + ['frac-pi2-6', String.raw`\frac{6}{\pi^2}`, 0, 'inverse fraction'], + ['frac-6-pi2', String.raw`6/\pi^2`, 1, 'slash form is equivalent (symbolic)'], + ['sum-gauss', String.raw`\sum_{k=1}^{n} k = \frac{n^2+n}{2}`, 1, 'expanded numerator (symbolic)'], + ['frac-mb-sigma', String.raw`\sigma_0 = f - \frac{1}{2}`, 0, '(f-1)/2 vs f-1/2 grouping'], + ['lim-sinx', String.raw`\lim_{x\to 0} \frac{\sin(x)}{x} = 1`, 1, 'parenthesised argument'], + ['case-fold-tri', String.raw`\triangle abc`, 1, 'case folded: speech cannot carry case'], + ['rel-perp', String.raw`ab \perp cd`, 1, 'T-153 relations: case folded'], + ['case-fold-ema', String.raw`e = m c^2`, 1, 'case folded'], + ['prob-fm-loss', String.raw`l_{fm}(\theta)=\mathbb{E}_{t\sim U([0,1]),\,x\sim p_t}\left[\left\|v_\theta(x,t)-u_t(x)\right\|^2\right]`, 1, 'L has one case, and U([0,1]) is U[0,1]'], + ['prob-fm-loss', String.raw`l_{fm}(\theta)=\mathbb{E}_{t\sim u[0,1],\,x\sim p_t}\left[\left\|v_\theta(x,t)-u_t(x)\right\|^2\right]`, 0, 'U and u are both in the reference'], + ['case-sens-aA', String.raw`a \in a`, 0, 'a and A both in reference: listener must hear "capital"'], + ['case-sens-aA', String.raw`a \in A`, 1, 'distinction kept'], + ['case-sens-nN', String.raw`n = n + 1`, 0, 'N vs n lost'], + ['case-sens-omega', String.raw`\omega \neq \omega`, 0, 'Ω vs ω lost'], + ['case-sens-omega', String.raw`\Omega \ne \omega`, 1, 'kept; \\ne alias'], + ['chem-water', String.raw`\ce{H2 + O2 -> H2O}`, 1, 'no coefficients: the substances of a textbook reading'], + ['chem-water', String.raw`\ce{H2 + O2 -> H2O2}`, 0, 'no coefficients, but another product'], + ['chem-water', String.raw`\ce{2H2 + O2 -> H2O}`, 0, 'one coefficient dropped'], + ['chem-water', String.raw`\ce{O2 + 2H2 -> 2H2O}`, 1, 'reactant order is free'], + ['chem-water', String.raw`2H_2 + O_2 \rightarrow 2H_2O`, 1, 'plain LaTeX arrow form'], + ['chem-water', String.raw`\ce{2H2 + O2 = 2H2O}`, 1, '= is the Chinese textbook arrow'], + ['chem-water', String.raw`\ce{2H2 + O2 <=> 2H2O}`, 0, 'wrong arrow class'], + ['chem-water', String.raw`\ce{2H2 + O2 -> 2H2O2}`, 0, 'H2O2 is not H2O'], + ['chem-co2', String.raw`\ce{C + O2 -> CO}`, 0, 'CO is not CO2'], + ['chem-ca-oh', String.raw`\ce{CaOH2 + CO2 -> CaCO3 + H2O}`, 0, 'lost (OH) group changes the formula'], + ['chem-ca-oh', String.raw`\ce{Ca(OH)2 + CO2 -> CaCO3 v + H2O}`, 1, 'precipitate mark ignored'], + ['chem-hydrate', String.raw`\ce{CuSO4.5H2O}`, 1, 'hydrate dot form'], + ['chem-hydrate', String.raw`\ce{CuSO4}`, 0, 'water of crystallisation lost'], + ['chem-haber', String.raw`\ce{N2 + 3H2 -> 2NH3}`, 0, 'reversible arrow lost'], + ['cur-two', String.raw`苹果 \$3，香蕉 \$5`, 1, 'two USD amounts'], + ['cur-two', String.raw`苹果 3，香蕉 5`, 0, 'dollar sign lost'], + ['cur-thousand', String.raw`The rent is \$1,200 a month`, 1, 'thousands separator'], + ['cur-thousand', String.raw`The rent is \$1 200`, 0, 'read as 1 and 200'], + ['cur-mixed', String.raw`A ticket is \$5, and the area is $x^2$`, 1, 'money + math'], + ['cur-mixed', String.raw`A ticket is 5, and the area is x`, 0, 'both lost'], + ['unit-temp', String.raw`25°C`, 1, 'degree sign'], + ['unit-g', String.raw`9.8\ \text{m/s}^2`, 1, 'text unit'], + ['unit-g', String.raw`9.8\ \text{m/s}`, 0, 'square lost'], + ['set-cap-empty', String.raw`A \cap B = \emptyset`, 1, 'empty set alias'], + ['set-interval', String.raw`x \in [0, b]`, 0, 'closed interval is not half-open'], + ] as const)('grades %s as %s: %s', (id, hypothesis, expected, _reason) => { + expect(grade(cases.get(id)!, hypothesis).score).toBe(expected) + }) + + it('accepts the spaced braces of MathBridge references', () => { + // MathBridge writes a space after each token, as in `_ { u }`. The answer `_u` is the same formula. + const item = (reference: string): FormulaSpeechCase => ({ id: 'mb', stratum: 'math-bridge', source: 'MathBridge', mode: 'symbolic', reference, input: `$${reference}$`, note: '' }) + expect(grade(item(String.raw`\lambda_ { \mathrm { u } } \rightarrow\infty`), String.raw`\lambda_u \to \infty`).score).toBe(1) + expect(grade(item(String.raw`b\in\mathbb { N } ^ { + }`), String.raw`b\in\mathbb{N}^{+}`).score).toBe(1) + expect(grade(item(String.raw`R_ { 0 , \mathrm { b } } =H_ { \mathrm { b } }`), String.raw`R_{0,b}=H_b`).score).toBe(1) + }) + + it('scores an empty answer 0 and does not throw on malformed LaTeX', () => { + expect(grade(cases.get('lim-basel')!, '')).toEqual({ score: 0, rule: 'empty' }) + expect(grade(cases.get('lim-basel')!, String.raw`\frac{{{`).score).toBe(0) + }) +}) + +describe('majorityScore', () => { + it('passes only with a strict majority', () => { + expect(majorityScore([1, 0, 1])).toBe(1) + expect(majorityScore([1, 0])).toBe(0) + expect(majorityScore([0, 0, 1])).toBe(0) + }) +}) diff --git a/evals/tts-speech-formula/eval/evaluators/grader/index.ts b/evals/tts-speech-formula/eval/evaluators/grader/index.ts new file mode 100644 index 0000000000..0da1f13900 --- /dev/null +++ b/evals/tts-speech-formula/eval/evaluators/grader/index.ts @@ -0,0 +1,63 @@ +import type { FormulaSpeechCase } from '../../cases/formula-speech/cases' +import type { KeptCase } from './normalize' + +import { sameChemistry } from './chem' +import { sameCurrencySentence } from './currency' +import { canonicalize, caseSensitiveLetters } from './normalize' +import { symbolicallyEquivalent } from './symbolic' + +/** + * The grade of one written-down answer. + * + * `rule` names the check that decided it: `norm`, `symbolic`, `chem`, or `currency`. + * `empty` means the listener wrote nothing, and `error` means the grader failed on malformed input. + * `unclear` means the listener could not hear anything recoverable. The read-aloud task sets it, not {@link grade}. + */ +export interface Grade { + score: 0 | 1 + rule: 'norm' | 'symbolic' | 'chem' | 'currency' | 'empty' | 'error' | 'unclear' +} + +/** + * Grades what a listener wrote down against the reference of a case. The result is deterministic, 1 or 0. + * The listener model never grades. Malformed LaTeX scores 0 and never fails a run. + */ +export function grade(item: FormulaSpeechCase, hypothesis: string | undefined): Grade { + try { + return gradeUnsafe(item, hypothesis) + } + catch { + return { score: 0, rule: 'error' } + } +} + +function gradeUnsafe(item: FormulaSpeechCase, hypothesis: string | undefined): Grade { + if (!hypothesis?.trim()) + return { score: 0, rule: 'empty' } + + // Only currency cases have a list reference. + if (typeof item.reference !== 'string') + return { score: sameCurrencySentence(item.reference, hypothesis) ? 1 : 0, rule: 'currency' } + + const reference = item.reference + if (item.mode === 'chem') + return { score: sameChemistry(reference, hypothesis) ? 1 : 0, rule: 'chem' } + + const keep: KeptCase = item.letterCase === 'sensitive' ? 'all' : item.letterCase === 'fold' ? new Set() : caseSensitiveLetters(reference) + if (canonicalize(reference, keep) === canonicalize(hypothesis, keep)) + return { score: 1, rule: 'norm' } + if (item.mode === 'symbolic' && symbolicallyEquivalent(reference, hypothesis, keep)) + return { score: 1, rule: 'symbolic' } + return { score: 0, rule: item.mode } +} + +/** + * Combines the grades of several listener votes. The item passes only with a strict majority of passing votes. + * + * @example + * majorityScore([1, 0, 1]) + * // => 1 + */ +export function majorityScore(scores: readonly (0 | 1)[]): 0 | 1 { + return 2 * scores.reduce((sum, score) => sum + score, 0) > scores.length ? 1 : 0 +} diff --git a/evals/tts-speech-formula/eval/evaluators/grader/normalize.ts b/evals/tts-speech-formula/eval/evaluators/grader/normalize.ts new file mode 100644 index 0000000000..ae25af3011 --- /dev/null +++ b/evals/tts-speech-formula/eval/evaluators/grader/normalize.ts @@ -0,0 +1,166 @@ +const greekNames = new Set(['alpha', 'beta', 'gamma', 'delta', 'epsilon', 'zeta', 'eta', 'theta', 'iota', 'kappa', 'lambda', 'mu', 'nu', 'xi', 'omicron', 'pi', 'rho', 'sigma', 'tau', 'upsilon', 'phi', 'chi', 'psi', 'omega']) + +/** + * Rewrites that make two spellings of one notation compare equal. Order matters: a later rule sees the output of the earlier ones. + * Set braces become sentinels first, so that a natural closing `}}` is never read as an escaped brace. + */ +const aliases: readonly (readonly [RegExp, string])[] = [ + [/\\dfrac|\\tfrac/g, String.raw`\frac`], + [/\\leqslant|\\leqq|\\le(?![a-z])/g, String.raw`\leq`], + [/\\geqslant|\\geqq|\\ge(?![a-z])/g, String.raw`\geq`], + [/\\ne(?![a-z])|\\not=/g, String.raw`\neq`], + [/\\rightarrow|\\longrightarrow|\\to(?![a-z])/g, String.raw`\to`], + [/\\emptyset|\\varnothing|\\O(?![a-zA-Z])/g, String.raw`\emptyset`], + [/\\dots|\\ldots|\\cdots|\.\.\.|…/g, String.raw`\cdots`], + [/\\times/g, String.raw`\cdot`], + [/\\ast|\*/g, String.raw`\cdot`], + [/\\colon/g, ':'], + [/\\vert|\\mid|\\lvert|\\rvert/g, '|'], + [/\\varepsilon/g, String.raw`\epsilon`], + [/\\varphi/g, String.raw`\phi`], + [/\\degree|°|\\textdegree/g, String.raw`^\circ`], + [/\\lbrace|\\\{/g, '\x01'], + [/\\rbrace|\\\}/g, '\x02'], + [/\\mapsto|\\longmapsto/g, String.raw`\mapsto`], + [/\\(?:mathbf|mathbb|mathcal|mathrm|text|textrm|operatorname|mathit|boldsymbol)\s*\{([^{}]*)\}/g, ' $1 '], + [/\\(?:mathbf|mathbb|mathcal|mathrm|text|textrm|operatorname|mathit|boldsymbol)(?![a-z])/g, ''], + [/\\Delta(?=\s*ABC)|\\bigtriangleup/g, String.raw`\triangle`], + [/π/g, String.raw`\pi`], + [/∞/g, String.raw`\infty`], + [/≤/g, String.raw`\leq`], + [/≥/g, String.raw`\geq`], + [/≠/g, String.raw`\neq`], + [/≈/g, String.raw`\approx`], + [/∈/g, String.raw`\in`], + [/∉/g, String.raw`\notin`], + [/⊆/g, String.raw`\subseteq`], + [/⊥/g, String.raw`\perp`], + [/∥/g, String.raw`\parallel`], + [/√/g, String.raw`\sqrt`], + [/[×·⋅]/g, String.raw`\cdot`], + [/[−–]/g, '-'], + [/∠/g, String.raw`\angle`], + [/△/g, String.raw`\triangle`], + // Japanese textbooks write a one-sided limit as `x→+0`. + [/\\to\s*\+\s*0(?![0-9.])/g, String.raw`\to 0^+`], + [/\\to\s*-\s*0(?![0-9.])/g, String.raw`\to 0^-`], + [/\\%/g, '%'], + [/\\lim\\limits/g, String.raw`\lim`], + [/\\sum\\limits/g, String.raw`\sum`], +] + +/** Commands and characters that only change spacing or size. */ +const spacing = /\\left|\\right|\\bigg|\\Bigg|\\big|\\Big|\\displaystyle|\\,|\\;|\\:|\\!|\\quad|\\qquad|\\ |~|\\limits|\$/g + +/** + * Removes the wrappers around an answer: math delimiters, code fences, equation environments, and final punctuation. + * A listener who heard raw LaTeX read aloud writes "dollar backslash triangle"; that spoken markup is not content. + * + * @example + * stripWrappers('$$\\backslash triangle ABC$$.') + * // => '\\triangle ABC' + */ +export function stripWrappers(source: string): string { + let text = source.trim() + text = text.replace(/\\backslash\s*(?=\\)/g, '') + text = text.replace(/\\backslash\s*([A-Z]+)/gi, (_, word: string) => `\\${word}`) + text = text.trim().replace(/^\\\$\s*|\s*\\\$$/g, '') + text = text.replace(/^```(?:latex|tex)?|```$/g, '').trim() + text = text.replace(/^\$\$?|\$\$?$/g, '').trim() + text = text.replace(/^\\\[|\\\]$|^\\\(|\\\)$/g, '').trim() + text = text.replace(/\\begin\{(?:equation|align)\*?\}|\\end\{(?:equation|align)\*?\}/g, '') + return text.replace(/[.。,，]+$/, '').trim() +} + +/** + * The letters whose case a comparison keeps: `all`, or a set of small Latin letters and small Greek names. A letter + * outside the set is folded. + */ +export type KeptCase = 'all' | ReadonlySet + +/** + * Lowers the case of every letter outside a command name, except the letters in `keep`. A capital Greek command becomes + * the small one. + * + * @example + * foldLetterCase('\\Omega = E \\cdot X') + * // => '\\omega = e \\cdot x' + */ +export function foldLetterCase(source: string, keep: ReadonlySet = new Set()): string { + const text = source.replace(/\\([A-Z][a-z]+)/g, (match, name: string) => greekNames.has(name.toLowerCase()) && !keep.has(name.toLowerCase()) ? `\\${name.toLowerCase()}` : match) + let result = '' + for (let index = 0; index < text.length;) { + if (text[index] === '\\') { + const command = /^\\(?:[A-Z]+|.)?/i.exec(text.slice(index))![0] + result += command + index += command.length + continue + } + const char = text[index]! + result += keep.has(char.toLowerCase()) ? char : char.toLowerCase() + index++ + } + return result +} + +/** + * The letters whose case must survive speech for this reference. + * + * Speech carries letter case only as a word, as "capital A", and a reading says it only for a letter that the formula + * uses in both cases, for example `a \\in A` or `\\Omega \\neq \\omega`. Those letters keep their case in the + * comparison. The other letters are folded, so `\\mathcal{L}_{FM}` read as "L sub F M" matches `l_{fm}`. + */ +export function caseSensitiveLetters(reference: string): Set { + // A calligraphic or a blackboard letter is still a letter, as `\\mathcal{U}` beside `u_t`. Upright words are names. + const body = reference.replace(/\\(?:mathbb|mathcal)\{([^}]*)\}/g, '$1').replace(/\\(?:mathrm|text|operatorname)\{[^}]*\}/g, '') + const commands = [...body.matchAll(/\\([A-Z]+)/gi)].map(match => match[1]!) + const sensitive = new Set() + for (const name of commands) { + const small = name.toLowerCase() + if (greekNames.has(small) && commands.includes(small) && commands.includes(small[0]!.toUpperCase() + small.slice(1))) + sensitive.add(small) + } + + const plain = body.replace(/\\[A-Z]+/gi, ' ') + const letters = new Set([ + // The first letter of each run, so `\\sin` and `ABC` do not count as many single letters. + ...[...plain.matchAll(/(? match[1]!), + // Every capital inside a run of capitals, for example each letter of `ABC`. + ...[...body.matchAll(/[A-Z](?=[A-Z])|(?<=[A-Z])[A-Z]/g)].map(match => match[0]), + ]) + for (const letter of letters) { + if (letters.has(letter.toLowerCase()) && letters.has(letter.toUpperCase())) + sensitive.add(letter.toLowerCase()) + } + return sensitive +} + +/** + * Brings an answer to one canonical LaTeX string: aliases, no spacing, single-token braces removed, thousands separators removed. + * One space stays after each command, so `\triangle ABC` does not become the command `\triangleABC`. + * + * @example + * canonicalize('$\\left( x^{2} \\right) \\le \\dfrac 1 2$') + * // => '(x^2)\\leq \\frac{1}{2}' + */ +export function canonicalize(source: string, keep: KeptCase = new Set()): string { + let text = stripWrappers(source) + for (const [pattern, replacement] of aliases) + text = text.replace(pattern, replacement) + text = text.replace(spacing, ' ') + text = text.replace(/\\infty/g, String.raw`\infty `) + text = text.replaceAll('\x01', '\\{').replaceAll('\x02', '\\}') + // Three passes unwrap nested single tokens, for example `x^{{2}}`. MathBridge writes its references with a space + // after each token, as in `\lambda_ { u }`, so a space can come between the script mark and its brace. + for (let pass = 0; pass < 3; pass++) + text = text.replace(/([\^_])\s*\{\s*([A-Z0-9+-]|\\[A-Z]+)\s*\}/gi, '$1$2') + text = text.replace(/\\frac\s*([0-9A-Z])\s*([0-9A-Z])/gi, String.raw`\frac{$1}{$2}`) + text = text.replace(/\\sqrt\s*([0-9A-Z])(?![A-Z])/gi, String.raw`\sqrt{$1}`) + text = text.replace(/\^\\circ\s*/g, String.raw`^\circ `) + text = text.replace(/(\d),(\d{3})(?!\d)/g, '$1$2') + // A distribution on an interval is written `U[0,1]` or `U([0,1])`: parentheses around one bracketed interval add nothing. + text = text.replace(/\(\s*(\[[^()[\]]*[\])])\s*\)/g, '$1') + text = text.replace(/(\\[A-Z]+)\s*/gi, '$1\x00') + text = text.replace(/\s+/g, '').replaceAll('\x00', ' ') + return keep === 'all' ? text : foldLetterCase(text, keep) +} diff --git a/evals/tts-speech-formula/eval/evaluators/grader/symbolic.ts b/evals/tts-speech-formula/eval/evaluators/grader/symbolic.ts new file mode 100644 index 0000000000..5449547f84 --- /dev/null +++ b/evals/tts-speech-formula/eval/evaluators/grader/symbolic.ts @@ -0,0 +1,95 @@ +import type { Expression } from '@cortex-js/compute-engine' + +import type { KeptCase } from './normalize' + +import { ComputeEngine, isFunction } from '@cortex-js/compute-engine' + +import { foldLetterCase, stripWrappers } from './normalize' + +const engine = new ComputeEngine() + +const relations = new Set(['Equal', 'NotEqual', 'Less', 'LessEqual', 'Greater', 'GreaterEqual']) + +/** + * Operators that the engine evaluates to a value. `lim sin x / x` and `lim x / sin x` both evaluate to 1, + * so these operators must match one by one before values are compared. + */ +const evaluatedOperators = new Set(['Limit', 'Sum', 'Product', 'Integrate', 'D', 'Derivative']) + +/** Prepares LaTeX for the parser: no sizing commands, one spelling of each relation, and products written out. */ +function prepare(source: string, keep: KeptCase): string { + let text = stripWrappers(source) + text = text.replace(/\\left|\\right|\\,|\\;|\\!|\\displaystyle|\\limits/g, '') + text = text.replace(/\\dfrac|\\tfrac/g, String.raw`\frac`) + text = text.replace(/\\le(?![a-z])|\\leqslant/g, String.raw`\leq`) + text = text.replace(/\\ge(?![a-z])|\\geqslant/g, String.raw`\geq`) + text = text.replaceAll('π', String.raw`\pi`).replaceAll('∞', String.raw`\infty`).replaceAll('−', '-') + // `n(n+1)` is a product here. Only f, g, and h name functions in the cases. + text = text.replace(/(? JSON.stringify(expression.json) + const leftParts = evaluatedParts(left).sort((a, b) => byJson(a).localeCompare(byJson(b))) + const rightParts = evaluatedParts(right).sort((a, b) => byJson(a).localeCompare(byJson(b))) + if (leftParts.length !== rightParts.length) + return false + + return leftParts.every((part, index) => { + const counterpart = rightParts[index]! + if (part.operator !== counterpart.operator) + return false + if (part.isSame(counterpart)) + return true + const ops = operandsOf(part) + const counterpartOps = operandsOf(counterpart) + return ops.length === counterpartOps.length && ops.every((op, opIndex) => same(op, counterpartOps[opIndex]!)) + }) +} + +/** + * Tells whether a hypothesis is symbolically equivalent to the reference. + * + * Relations must have the same operator and equivalent sides. An equation may also swap its sides. + * Unparsable input is not equivalent; it never throws. + */ +export function symbolicallyEquivalent(reference: string, hypothesis: string, keep: KeptCase): boolean { + try { + const expected = engine.parse(prepare(reference, keep)) + const actual = engine.parse(prepare(hypothesis, keep)) + if (!expected.isValid || !actual.isValid) + return false + if (expected.isSame(actual)) + return true + if (!evaluatedPartsMatch(expected, actual)) + return false + + if (relations.has(expected.operator) || relations.has(actual.operator)) { + if (expected.operator !== actual.operator || !isFunction(expected) || !isFunction(actual)) + return false + if (same(expected.op1, actual.op1) && same(expected.op2, actual.op2)) + return true + return expected.operator === 'Equal' && same(expected.op1, actual.op2) && same(expected.op2, actual.op1) + } + return same(expected, actual) + } + catch { + return false + } +} diff --git a/evals/tts-speech-formula/eval/evaluators/judge/calibration/calibration.json b/evals/tts-speech-formula/eval/evaluators/judge/calibration/calibration.json new file mode 100644 index 0000000000..c372bee062 --- /dev/null +++ b/evals/tts-speech-formula/eval/evaluators/judge/calibration/calibration.json @@ -0,0 +1,845 @@ +{ + "seed": 20261020, + "items": [ + { + "id": "rule|mb-8715240|en", + "object": "rule", + "caseId": "mb-8715240", + "language": "en", + "reference": "\\mathcal { I }", + "spoken": "I", + "judge": { + "reading": "natural", + "reason": "No part is spelled; “I” naturally reads the letter in the formula." + }, + "label": "natural", + "note": "" + }, + { + "id": "llm|mb-3510988|en", + "object": "llm", + "caseId": "mb-3510988", + "language": "en", + "reference": "\\|e_ { \\vect u_3 } \\|_ { \\infty }", + "spoken": "The norm, with subscript infinity, of e, with subscript vector u, subscript three.", + "judge": { + "reading": "natural", + "reason": "No part is spelled; the subscripts are read in words." + }, + "label": "natural", + "note": "" + }, + { + "id": "sre|mb-7855753|en", + "object": "sre", + "caseId": "mb-7855753", + "language": "en", + "reference": "\\mathbb E [ N_\\alpha^T ( v ) ]", + "spoken": "double struck E of N sub alpha to the T power of v", + "judge": { + "reading": "natural", + "reason": "No part is spelled; the reading uses standard words for the expectation, subscript, and power." + }, + "label": "natural", + "note": "Check whether \"N sub alpha to the T power of v\" keeps the grouping of N_alpha^T(v)." + }, + { + "id": "rule|mb-519492|en", + "object": "rule", + "caseId": "mb-519492", + "language": "en", + "reference": "k=1 , \\dots , d", + "spoken": "k equals one, dot dot dot, d", + "judge": { + "reading": "natural", + "reason": "No part is spelled; “and so on, up to” is a natural way to read the ellipsis." + }, + "label": "mixed", + "note": "Read the ellipsis as \"and so on\" or \"up to\", not \"dot dot dot\"." + }, + { + "id": "sre|frac-6-pi2|en", + "object": "sre", + "caseId": "frac-6-pi2", + "language": "en", + "reference": "\\frac{6}{\\pi^2}", + "spoken": "the fraction with numerator 6 and denominator pi squared", + "judge": { + "reading": "natural", + "reason": "No part is spelled; the numerator-and-denominator phrasing naturally preserves the grouping of pi squared." + }, + "label": "mixed", + "note": "A simple fraction reads shorter: \"six over pi squared\"." + }, + { + "id": "rule|mb-3064229|en", + "object": "rule", + "caseId": "mb-3064229", + "language": "en", + "reference": "k\\in [ K ]", + "spoken": "lowercase k is an element of open bracket capital K close bracket", + "judge": { + "reading": "mixed", + "reason": "“Lowercase” and “capital” spell out letter case, and “open bracket” and “close bracket” name brackets around the set." + }, + "label": "mixed", + "note": "" + }, + { + "id": "sre|misc-binom|en", + "object": "sre", + "caseId": "misc-binom", + "language": "en", + "reference": "(a+b)^2=a^2+2ab+b^2", + "spoken": "open paren a plus b close paren squared equals a squared plus 2 a b plus b squared", + "judge": { + "reading": "spelled", + "reason": "The reading names the parentheses around the compound expression instead of reading its meaning naturally." + }, + "label": "spelled", + "note": "Do not read the parentheses. Say \"the square of the sum of a and b\"." + }, + { + "id": "llm|mb-1727433|en", + "object": "llm", + "caseId": "mb-1727433", + "language": "en", + "reference": "\\Delta_k < 0", + "spoken": "Delta sub k is less than zero.", + "judge": { + "reading": "natural", + "reason": "No part is spelled; “Delta sub k is less than zero” is a natural reading." + }, + "label": "natural", + "note": "Good reading." + }, + { + "id": "rule|mb-8575803|en", + "object": "rule", + "caseId": "mb-8575803", + "language": "en", + "reference": "\\mathcal { B } ( B \\to \\ , K^ { * } \\ , \\nu\\ , \\bar { \\nu } )", + "spoken": "B of B approaches, K superscript star, nu, nu bar", + "judge": { + "reading": "mixed", + "reason": "“Superscript” spells out the markup for the star rather than reading it as “star,” and “approaches” is not a spelled part." + }, + "label": "mixed", + "note": "The reading of the decay and of \"nu, nu bar\" is unclear." + }, + { + "id": "llm|mb-6904738|en", + "object": "llm", + "caseId": "mb-6904738", + "language": "en", + "reference": "G^ { ( p ) }", + "spoken": "G superscript, open parenthesis, p, close parenthesis.", + "judge": { + "reading": "spelled", + "reason": "“Open parenthesis” and “close parenthesis” spell out the grouping instead of reading the superscript as G to the power of p." + }, + "label": "mixed", + "note": "" + }, + { + "id": "sre|mb-5565465|en", + "object": "sre", + "caseId": "mb-5565465", + "language": "en", + "reference": "\\frac { 7 } { 60 } \\mathbf { k } \\cdot \\mathbf { k } ^\\prime", + "spoken": "7 over 60 bold k times bold k prime", + "judge": { + "reading": "mixed", + "reason": "“Bold” names formatting markup for both k’s." + }, + "label": "mixed", + "note": "Only one k appears, so \"bold\" is not needed: \"k times k prime\"." + }, + { + "id": "llm|chem-water|ja", + "object": "llm", + "caseId": "chem-water", + "language": "ja", + "reference": "\\ce{2H2 + O2 -> 2H2O}", + "spoken": "係数二、エイチの下付き数字二、プラス、オーの下付き数字二、右向き矢印、係数二、エイチの下付き数字二、オー", + "judge": { + "reading": "mixed", + "reason": "No part is spelled; the element symbols are read by letter, and the subscripts and reaction arrow are described in Japanese." + }, + "label": "spelled", + "note": "Reads every subscript and the arrow by name. It does not work as a reading of an equation." + }, + { + "id": "llm|misc-deriv|ja", + "object": "llm", + "caseId": "misc-deriv", + "language": "ja", + "reference": "\\frac{d}{dx}x^{n}=nx^{n-1}", + "spoken": "分子がディー、分母がディーエックスの分数に、エックスのエヌ乗。イコール、エヌかけるエックスの、肩にエヌひく一。", + "judge": { + "reading": "mixed", + "reason": "The phrase 「肩に」 spells out the exponent’s visual placement instead of reading it as a power." + }, + "label": "spelled", + "note": "Mostly right. Unsure whether 肩に is a usual Japanese reading." + }, + { + "id": "sre|root-quadratic|ja", + "object": "sre", + "caseId": "root-quadratic", + "language": "ja", + "reference": "x=\\frac{-b\\pm\\sqrt{b^2-4ac}}{2a}", + "spoken": "x equals the fraction with numerator negative b plus or minus the square root of b squared minus 4 a c and denominator 2 a", + "judge": { + "reading": "spelled", + "reason": "The reading is mostly in English rather than Japanese." + }, + "label": "spelled", + "note": "Reads in English, not Japanese. A reading in another language does not pass." + }, + { + "id": "sre|chem-h2o2|ja", + "object": "sre", + "caseId": "chem-h2o2", + "language": "ja", + "reference": "\\ce{2H2O2 -> 2H2O + O2}", + "spoken": "2 empty normal H of sub 2 normal O sub 2 right arrow 2 empty normal H of sub 2 normal O plus normal O sub 2", + "judge": { + "reading": "spelled", + "reason": "The reading spells the chemical symbols with markup words such as “empty” and “normal,” and says “right arrow” in English." + }, + "label": "spelled", + "note": "" + }, + { + "id": "rule|sum-geo|ja", + "object": "rule", + "caseId": "sum-geo", + "language": "ja", + "reference": "\\sum_{k=0}^{\\infty} \\frac{1}{2^{k}}=2", + "spoken": "ケーがゼロから無限大までのにのケー乗分のいちの和 イコール に", + "judge": { + "reading": "natural", + "reason": "No part is spelled; the sum, fraction, power, and equality are read in standard Japanese mathematical terms." + }, + "label": "natural", + "note": "" + }, + { + "id": "llm|case-sens-aA|ja", + "object": "llm", + "caseId": "case-sens-aA", + "language": "ja", + "reference": "a \\in A", + "spoken": "小文字のエーは、大文字のエーに属する。", + "judge": { + "reading": "natural", + "reason": "No part is spelled; the reading naturally distinguishes lowercase a from uppercase A." + }, + "label": "natural", + "note": "Shorter case words are enough, for example 小 a and 大 A." + }, + { + "id": "sre|frac-nested|ja", + "object": "sre", + "caseId": "frac-nested", + "language": "ja", + "reference": "\\frac{1}{1+\\frac{1}{x}}", + "spoken": "the fraction with numerator 1 and denominator 1 plus 1 over x", + "judge": { + "reading": "spelled", + "reason": "The reading is mostly in English rather than Japanese." + }, + "label": "spelled", + "note": "Reads in English, not Japanese. A reading in another language does not pass." + }, + { + "id": "llm|set-builder|ja", + "object": "llm", + "caseId": "set-builder", + "language": "ja", + "reference": "\\{x \\mid x > 0\\}", + "spoken": "左中かっこ、エックス、縦棒、エックス大なりゼロ、右中かっこ。", + "judge": { + "reading": "spelled", + "reason": "It spells out the set braces as “left brace” and “right brace,” and reads the set-builder bar as “vertical bar.”" + }, + "label": "spelled", + "note": "" + }, + { + "id": "rule|frac-6-pi2|zh", + "object": "rule", + "caseId": "frac-6-pi2", + "language": "zh", + "reference": "\\frac{6}{\\pi^2}", + "spoken": "派的平方分之6", + "judge": { + "reading": "natural", + "reason": "No part is spelled; this is a natural reading of the fraction." + }, + "label": "natural", + "note": "" + }, + { + "id": "llm|unit-g|zh", + "object": "llm", + "caseId": "unit-g", + "language": "zh", + "reference": "9.8\\,\\mathrm{m/s^2}", + "spoken": "九点八米每秒平方", + "judge": { + "reading": "natural", + "reason": "No part is spelled; the quantity is read naturally in Mandarin Chinese." + }, + "label": "natural", + "note": "" + }, + { + "id": "llm|rel-parallel|zh", + "object": "llm", + "caseId": "rel-parallel", + "language": "zh", + "reference": "l_1 \\parallel l_2", + "spoken": "小写字母 l，下标一，平行于小写字母 l，下标二。", + "judge": { + "reading": "mixed", + "reason": "The reading spells out “小写字母” even though the formula contains only one case of l." + }, + "label": "mixed", + "note": "The letter has one case in the formula, so the case is not needed. If it is, say 小 l." + }, + { + "id": "sre|frac-6-pi2|zh", + "object": "sre", + "caseId": "frac-6-pi2", + "language": "zh", + "reference": "\\frac{6}{\\pi^2}", + "spoken": "the fraction with numerator 6 and denominator pi squared", + "judge": { + "reading": "natural", + "reason": "The reading is in English rather than Mandarin Chinese." + }, + "label": "mixed", + "note": "Not Chinese. The fraction also reads shorter." + }, + { + "id": "rule|case-sens-aA|zh", + "object": "rule", + "caseId": "case-sens-aA", + "language": "zh", + "reference": "a \\in A", + "spoken": "小写a 属于 大写A", + "judge": { + "reading": "natural", + "reason": "No part is spelled." + }, + "label": "mixed", + "note": "Use the short form 小a and 大A. The case words can go unless a variant of the letter appears." + }, + { + "id": "llm|set-builder|zh", + "object": "llm", + "caseId": "set-builder", + "language": "zh", + "reference": "\\{x \\mid x > 0\\}", + "spoken": "左花括号，x，使得 x 大于零，右花括号。", + "judge": { + "reading": "spelled", + "reason": "The reading names the left and right braces around the set." + }, + "label": "spelled", + "note": "The braces of a set are clear from the meaning and need no reading." + }, + { + "id": "sre|misc-binom|zh", + "object": "sre", + "caseId": "misc-binom", + "language": "zh", + "reference": "(a+b)^2=a^2+2ab+b^2", + "spoken": "open paren a plus b close paren squared equals a squared plus 2 a b plus b squared", + "judge": { + "reading": "spelled", + "reason": "The reading is mostly in English rather than Mandarin Chinese, and it names the main parentheses as “open paren” and “close paren.”" + }, + "label": "spelled", + "note": "Not Chinese, and the parentheses are read. Better: a 与 b 的和的平方 等于 a 方加 2ab 加 b 方." + }, + { + "id": "rule|set-interval|zh", + "object": "rule", + "caseId": "set-interval", + "language": "zh", + "reference": "x \\in (0, b]", + "spoken": "x 属于 左开右闭区间0到b", + "judge": { + "reading": "natural", + "reason": "No part is spelled; “左开右闭区间” naturally describes the interval notation." + }, + "label": "natural", + "note": "" + }, + { + "id": "llm|root-sqrt2|zh", + "object": "llm", + "caseId": "root-sqrt2", + "language": "zh", + "reference": "\\sqrt{2}\\approx 1.414", + "spoken": "根号二约等于一点四一四。", + "judge": { + "reading": "natural", + "reason": "No part is spelled; the formula is read naturally in Mandarin Chinese." + }, + "label": "natural", + "note": "" + }, + { + "id": "sre|rel-leq|zh", + "object": "sre", + "caseId": "rel-leq", + "language": "zh", + "reference": "x \\leq 3", + "spoken": "x is less than or equal to 3", + "judge": { + "reading": "spelled", + "reason": "The comparison is read in English rather than Mandarin Chinese." + }, + "label": "spelled", + "note": "Reads in English, not Chinese. A reading in another language does not pass." + }, + { + "id": "llm|set-builder|en", + "object": "llm", + "caseId": "set-builder", + "language": "en", + "reference": "\\{x \\mid x > 0\\}", + "spoken": "Open brace, x such that x is greater than zero, close brace.", + "judge": { + "reading": "spelled", + "reason": "It spells out the opening and closing braces around the set." + }, + "label": "mixed", + "note": "The braces of a set need no reading." + }, + { + "id": "rule|mb-1836666|en", + "object": "rule", + "caseId": "mb-1836666", + "language": "en", + "reference": "\\lVert\\ \\ # 1\\ \\rVert { C^ { -1 } } \\leq c_0", + "spoken": "lVert # 1 rVert C^ -1 leq c_0", + "judge": { + "reading": "spelled", + "reason": "The reading leaves the LaTeX commands and symbols `lVert`, `rVert`, `^`, `leq`, and `_` spelled out rather than reading their mathematical meaning." + }, + "label": "spelled", + "note": "The reading is not understandable: it leaves command names such as lVert." + }, + { + "id": "llm|mb-1167649|en", + "object": "llm", + "caseId": "mb-1167649", + "language": "en", + "reference": "\\Sigma_ { B_T }", + "spoken": "Capital sigma, subscript, open brace, capital B, subscript capital T, close brace.", + "judge": { + "reading": "spelled", + "reason": "The reading spells the braces and subscript markup, and adds unnecessary case labels for sigma, B, and T." + }, + "label": "spelled", + "note": "" + }, + { + "id": "sre|mb-37396|en", + "object": "sre", + "caseId": "mb-37396", + "language": "en", + "reference": "\\qKoec_i ( x_1 ) \\neq +\\infty", + "spoken": "backslash qKoec sub i times open paren x sub 1 close paren is not equal to positive infinity", + "judge": { + "reading": "spelled", + "reason": "It says the LaTeX command name “backslash qKoec” and spells out the parentheses around the function argument." + }, + "label": "spelled", + "note": "" + }, + { + "id": "sre|unit-g|ja", + "object": "sre", + "caseId": "unit-g", + "language": "ja", + "reference": "9.8\\,\\mathrm{m/s^2}", + "spoken": "9.8 empty meters divided by seconds squared", + "judge": { + "reading": "spelled", + "reason": "“Empty” and “divided by” spell spacing and fraction notation rather than reading the unit naturally in Japanese." + }, + "label": "mixed", + "note": "\"empty\" is not a reading." + }, + { + "id": "llm|case-sens-nN|ja", + "object": "llm", + "caseId": "case-sens-nN", + "language": "ja", + "reference": "N=n+1", + "spoken": "大文字のエヌ、イコール、小文字のエヌ、プラス、一。", + "judge": { + "reading": "natural", + "reason": "The case words for N and n are natural because both cases appear in the formula; no part is spelled." + }, + "label": "mixed", + "note": "Shorter case words are enough, unless the formula has several variants of n." + }, + { + "id": "sre|chem-co2|ja", + "object": "sre", + "caseId": "chem-co2", + "language": "ja", + "reference": "\\ce{C + O2 -> CO2}", + "spoken": "Celsius plus normal O sub 2 right arrow Celsius normal O sub 2", + "judge": { + "reading": "spelled", + "reason": "The reading is mostly English, and “sub” and “right arrow” spell the notation rather than reading it in Japanese." + }, + "label": "spelled", + "note": "Reads C as Celsius, and the oxygen molecule is lost." + }, + { + "id": "sre|chem-co|ja", + "object": "sre", + "caseId": "chem-co", + "language": "ja", + "reference": "\\ce{2C + O2 -> 2CO}", + "spoken": "2 empty Celsius plus normal O sub 2 right arrow 2 empty Celsius normal O", + "judge": { + "reading": "spelled", + "reason": "The reading uses the English phrases “empty Celsius,” “plus,” “normal,” and “right arrow” instead of Japanese readings." + }, + "label": "spelled", + "note": "Reads the formula symbols, not the chemical formula." + }, + { + "id": "rule|lim-fx-inf|ja", + "object": "rule", + "caseId": "lim-fx-inf", + "language": "ja", + "reference": "f(x) \\to \\infty", + "spoken": "エフエックスを無限大に近づける", + "judge": { + "reading": "natural", + "reason": "No part is spelled; the reading uses natural Japanese mathematical phrasing." + }, + "label": "natural", + "note": "" + }, + { + "id": "rule|chem-hydrate|zh", + "object": "rule", + "caseId": "chem-hydrate", + "language": "zh", + "reference": "\\ce{CuSO4*5H2O}", + "spoken": "CuSO4 点 5 H2O", + "judge": { + "reading": "spelled", + "reason": "It leaves the chemical formulas “CuSO4” and “H2O” as source notation instead of reading the element symbols letter by letter." + }, + "label": "spelled", + "note": "" + }, + { + "id": "sre|unit-temp|zh", + "object": "sre", + "caseId": "unit-temp", + "language": "zh", + "reference": "25^\\circ\\mathrm{C}", + "spoken": "25 raised to the composed with power Celsius", + "judge": { + "reading": "spelled", + "reason": "The reading uses the English phrase “raised to the ... power” instead of Mandarin Chinese." + }, + "label": "spelled", + "note": "Should be 25 摄氏度." + }, + { + "id": "llm|chem-h2o2|zh", + "object": "llm", + "caseId": "chem-h2o2", + "language": "zh", + "reference": "\\ce{2H2O2 -> 2H2O + O2}", + "spoken": "系数二，过氧化氢分子，化学式中氢的下标是二，氧的下标是二，反应生成系数二的水分子，化学式中氢的下标是二，再加一个氧气分子，化学式中氧的下标是二。", + "judge": { + "reading": "natural", + "reason": "No part is spelled; the coefficients, compound names, and subscript readings use natural Mandarin terminology." + }, + "label": "mixed", + "note": "Too mechanical." + }, + { + "id": "sre|lim-recip|zh", + "object": "sre", + "caseId": "lim-recip", + "language": "zh", + "reference": "\\lim_{x \\to \\infty}\\frac{1}{x}=0", + "spoken": "lim over x right arrow infinity of 1 over x equals 0", + "judge": { + "reading": "spelled", + "reason": "The reading is mostly English, and “right arrow” spells the arrow symbol." + }, + "label": "mixed", + "note": "Unsure whether lim reads this way." + }, + { + "id": "sre|case-sens-xX|zh", + "object": "sre", + "caseId": "case-sens-xX", + "language": "zh", + "reference": "f: X \\to Y, x \\mapsto y", + "spoken": "f colon X right arrow Y comma x right arrow from bar y", + "judge": { + "reading": "spelled", + "reason": "The reading spells the symbols as “colon,” “right arrow,” “comma,” and “right arrow from bar” instead of reading their meaning in Mandarin." + }, + "label": "spelled", + "note": "" + }, + { + "id": "llm|frac-mb-sigma|ko", + "object": "llm", + "caseId": "frac-mb-sigma", + "language": "ko", + "reference": "\\sigma_{0}=\\frac{f-1}{2}", + "spoken": "시그마 아래첨자 영은 분수 시작, 분자 에프 빼기 일, 분모 이, 분수 끝과 같다.", + "judge": { + "reading": "mixed", + "reason": "The reading spells out fraction markup with “분수 시작” and “분수 끝.”" + }, + "label": null, + "note": "" + }, + { + "id": "llm|lim-recip|ko", + "object": "llm", + "caseId": "lim-recip", + "language": "ko", + "reference": "\\lim_{x \\to \\infty}\\frac{1}{x}=0", + "spoken": "리미트 엑스가 무한대로 갈 때, 분수의 분자는 일, 분모는 엑스, 그 값은 영입니다.", + "judge": { + "reading": "natural", + "reason": "No part is spelled; the limit and fraction are read in natural Korean mathematical language." + }, + "label": null, + "note": "" + }, + { + "id": "llm|root-sqrt2|ko", + "object": "llm", + "caseId": "root-sqrt2", + "language": "ko", + "reference": "\\sqrt{2}\\approx 1.414", + "spoken": "제곱근 이는 약 일 점 사 일 사입니다.", + "judge": { + "reading": "natural", + "reason": "No part is spelled; the formula is read naturally in Korean." + }, + "label": null, + "note": "" + }, + { + "id": "llm|set-builder|ko", + "object": "llm", + "caseId": "set-builder", + "language": "ko", + "reference": "\\{x \\mid x > 0\\}", + "spoken": "왼쪽 중괄호, 엑스, 세로줄, 엑스가 영보다 크다, 오른쪽 중괄호.", + "judge": { + "reading": "spelled", + "reason": "The reading spells out the set’s opening and closing braces and names the vertical bar instead of reading the set-builder structure naturally." + }, + "label": null, + "note": "" + }, + { + "id": "llm|set-cup|ko", + "object": "llm", + "caseId": "set-cup", + "language": "ko", + "reference": "A \\cup B=\\{1,2,3\\}", + "spoken": "에이는 비와의 합집합이며, 중괄호를 열고 일, 이, 삼을 차례로 적은 뒤 중괄호를 닫은 집합과 같다.", + "judge": { + "reading": "mixed", + "reason": "It spells the set braces by saying to open and close the curly braces." + }, + "label": null, + "note": "" + }, + { + "id": "rule|case-sens-aA|ko", + "object": "rule", + "caseId": "case-sens-aA", + "language": "ko", + "reference": "a \\in A", + "spoken": "소문자 에이는 대문자 에이의 원소", + "judge": { + "reading": "natural", + "reason": "No part is spelled; the reading naturally distinguishes lowercase a from uppercase A." + }, + "label": null, + "note": "" + }, + { + "id": "rule|lim-e|ko", + "object": "rule", + "caseId": "lim-e", + "language": "ko", + "reference": "\\lim_{n \\to \\infty}\\left(1+\\frac{1}{n}\\right)^{n}=e", + "spoken": "엔이 무한대로 갈 때 괄호 일 더하기 분자가 일, 분모가 엔인 분수 괄호 닫고의 엔 제곱의 극한은 e", + "judge": { + "reading": "mixed", + "reason": "The reading is natural; it uses the standard fraction wording and names the meaningful parentheses around the compound expression." + }, + "label": null, + "note": "" + }, + { + "id": "rule|prob-fm-loss|ko", + "object": "rule", + "caseId": "prob-fm-loss", + "language": "ko", + "reference": "\\mathcal{L}_{FM}(\\theta) = \\mathbb{E}_{t \\sim \\mathcal{U}[0,1],\\, x \\sim p_t}\\left[\\|v_\\theta(x,t) - u_t(x)\\|^2\\right]", + "spoken": "엘 아래첨자 에프 엠 세타는 티가 대문자 유 영부터 일까지의 닫힌구간을 따름, 엑스가 피 아래첨자 티를 따름에 대한 브이 아래첨자 세타 괄호 엑스, 티 괄호 닫고 빼기 소문자 유 아래첨자 티 엑스의 노름의 제곱의 기댓값", + "judge": { + "reading": "natural", + "reason": "No part is spelled; the subscripts, function arguments, interval, and norm are read naturally in Korean." + }, + "label": null, + "note": "" + }, + { + "id": "rule|set-builder|ja", + "object": "rule", + "caseId": "set-builder", + "language": "ja", + "reference": "\\{x \\mid x > 0\\}", + "spoken": "エックス 大なり ゼロようなエックスの集合", + "judge": { + "reading": "natural", + "reason": "No part is spelled; it naturally reads the set of x such that x is greater than zero." + }, + "label": null, + "note": "" + }, + { + "id": "rule|set-interval|ko", + "object": "rule", + "caseId": "set-interval", + "language": "ko", + "reference": "x \\in (0, b]", + "spoken": "엑스는 영 초과 비 이하인 구간의 원소", + "judge": { + "reading": "natural", + "reason": "No part is spelled; the interval and membership are read naturally in Korean." + }, + "label": null, + "note": "" + }, + { + "id": "rule|sum-gauss|ja", + "object": "rule", + "caseId": "sum-gauss", + "language": "ja", + "reference": "\\sum_{k=1}^{n} k=\\frac{n(n+1)}{2}", + "spoken": "ケーがいちからエヌまでのケーの和 イコール 分子がエヌかっこエヌプラスいちかっことじ、分母がにの分数", + "judge": { + "reading": "mixed", + "reason": "The spoken reading spells the parentheses in the denominator expression by naming them." + }, + "label": null, + "note": "" + }, + { + "id": "sre|case-sens-xX|ko", + "object": "sre", + "caseId": "case-sens-xX", + "language": "ko", + "reference": "f: X \\to Y, x \\mapsto y", + "spoken": "f 콜론 X 오른쪽 화살표 Y 콤마 x 세로선이 붙은 오른쪽 화살표 y", + "judge": { + "reading": "mixed", + "reason": "The reading spells the mapsto symbol as “세로선이 붙은 오른쪽 화살표” instead of reading its mapping meaning." + }, + "label": null, + "note": "" + }, + { + "id": "sre|chem-water|ko", + "object": "sre", + "caseId": "chem-water", + "language": "ko", + "reference": "\\ce{2H2 + O2 -> 2H2O}", + "spoken": "2 기본 H 의 아래첨자 2 더하기 기본 O 아래첨자 2 오른쪽 화살표 2 기본 H 의 아래첨자 2 기본 O", + "judge": { + "reading": "mixed", + "reason": "“기본” spells font formatting, and “오른쪽 화살표” names the arrow symbol rather than reading its meaning." + }, + "label": null, + "note": "" + }, + { + "id": "sre|lim-recip|ko", + "object": "sre", + "caseId": "lim-recip", + "language": "ko", + "reference": "\\lim_{x \\to \\infty}\\frac{1}{x}=0", + "spoken": "x 오른쪽 화살표 무한대 에 대한 리미트 의 x 분의 1 은 0", + "judge": { + "reading": "mixed", + "reason": "The reading names the arrow as a right arrow instead of reading the limit condition naturally." + }, + "label": null, + "note": "" + }, + { + "id": "sre|misc-binom|ko", + "object": "sre", + "caseId": "misc-binom", + "language": "ko", + "reference": "(a+b)^2=a^2+2ab+b^2", + "spoken": "괄호 열고 a 더하기 b 괄호 닫고 제곱 은 a 제곱 더하기 2 a b 더하기 b 제곱", + "judge": { + "reading": "spelled", + "reason": "The reading names the main parentheses around a plus b instead of using a natural phrase for the squared sum." + }, + "label": null, + "note": "" + }, + { + "id": "sre|sum-sq|ko", + "object": "sre", + "caseId": "sum-sq", + "language": "ko", + "reference": "\\sum_{i=1}^{n} i^{2}", + "spoken": "시그마 i 는 1 에서 n 까지 i 제곱", + "judge": { + "reading": "natural", + "reason": "No part is spelled; the summation notation is read in a conventional Korean shorthand." + }, + "label": null, + "note": "" + }, + { + "id": "sre|unit-temp|ko", + "object": "sre", + "caseId": "unit-temp", + "language": "ko", + "reference": "25^\\circ\\mathrm{C}", + "spoken": "25 의 합성 함수 제곱 섭씨", + "judge": { + "reading": "mixed", + "reason": "The reading spells the superscript as “제곱,” which changes the degree symbol into an exponent." + }, + "label": null, + "note": "" + } + ] +} diff --git a/evals/tts-speech-formula/eval/evaluators/judge/calibration/collect.ts b/evals/tts-speech-formula/eval/evaluators/judge/calibration/collect.ts new file mode 100644 index 0000000000..e579605dda --- /dev/null +++ b/evals/tts-speech-formula/eval/evaluators/judge/calibration/collect.ts @@ -0,0 +1,146 @@ +import type { CalibrationFile, CalibrationItem } from './types' + +import process from 'node:process' + +import { glob, readFile, rename, stat, writeFile } from 'node:fs/promises' + +import { formulaSpeechCases } from '../../../cases/formula-speech' +import { mathBridgeCases } from '../../../cases/math-bridge' + +/** + * Collects the readings that a person labels in the calibration tool, see ./README.md. + * + * It reads the newest `result.json` of each case from the reports. + * + * - Without `--pick`, it keeps the items of `calibration.json` and refreshes only their judge verdicts, so the set + * stays the same while the judge prompt changes. Agreement is then measured on the same readings every time. + * - With `--pick`, or when the file is empty, it picks {@link perLanguage} readings for each language. The picks spread + * over the test objects and over the verdicts of the judge. Labeled items always stay. + * + * Run: pnpm -F @proj-airi/evals-tts-speech-formula run judge:collect [-- --pick] + */ + +const reportsRoot = new URL('../../../../.vieval/reports/', import.meta.url) +const calibrationFile = new URL('./calibration.json', import.meta.url) +/** The test objects that write their own words. `raw` is left out: it only passes the source through. */ +const objects = ['rule', 'llm', 'sre'] as const +const verdicts = ['natural', 'mixed', 'spelled'] as const +const perLanguage = 15 +const seed = 20261020 +/** Cases of the current datasets. Reports keep results of cases that a newer sample dropped, and those are left out. */ +const currentCases = new Set([...formulaSpeechCases, ...mathBridgeCases].map(item => item.id)) + +interface ResultFile { + id: string + language: string + tts: string + reference: unknown + spoken: string + reading?: string + verdicts?: { reason: string, reading: string }[] +} + +/** FNV-1a over the seed and a key, so the pick order is random but repeats with the same seed. */ +function order(key: string): number { + let hash = 2166136261 ^ seed + for (const char of key) { + hash ^= char.codePointAt(0)! + hash = Math.imul(hash, 16777619) + } + return hash >>> 0 +} + +/** The newest judged reading of each object, case, and language, without TTS. */ +async function newestReadings(): Promise { + const newest = new Map() + for (const object of objects) { + for await (const path of glob(`${object}/*/tts-speech-formula/attempt-*/run-*/artifacts/*/*/result.json`, { cwd: reportsRoot.pathname })) { + const file = new URL(path, reportsRoot) + const result = JSON.parse(await readFile(file, 'utf8')) as ResultFile + if (!currentCases.has(result.id) || result.tts !== 'none' || result.reading === undefined || typeof result.reference !== 'string' || !result.spoken.trim()) + continue + const id = `${object}|${result.id}|${result.language}` + const time = (await stat(file)).mtimeMs + if ((newest.get(id)?.time ?? 0) >= time) + continue + newest.set(id, { + time, + item: { + id, + object, + caseId: result.id, + language: result.language, + reference: result.reference, + spoken: result.spoken, + judge: { reading: result.reading, reason: result.verdicts?.[0]?.reason ?? '' }, + label: null, + note: '', + }, + }) + } + } + return [...newest.values()].map(({ item }) => item) +} + +/** + * Picks readings of one language in rounds over each object and each verdict, so a verdict that the judge rarely gives + * still gets picks. A spoken text that is already picked is skipped, because two objects can say the same words. + */ +function pick(readings: readonly CalibrationItem[]): CalibrationItem[] { + const buckets = new Map() + for (const item of [...readings].sort((left, right) => order(left.id) - order(right.id))) { + const key = `${item.object}|${item.judge.reading}` + buckets.set(key, [...(buckets.get(key) ?? []), item]) + } + const picked: CalibrationItem[] = [] + const spoken = new Set() + while (picked.length < perLanguage && [...buckets.values()].some(bucket => bucket.length > 0)) { + for (const verdict of verdicts) { + for (const object of objects) { + const item = buckets.get(`${object}|${verdict}`)?.shift() + if (!item || spoken.has(item.spoken) || picked.length >= perLanguage) + continue + spoken.add(item.spoken) + picked.push(item) + } + } + } + return picked +} + +/** Written next to the file and renamed over it, as the labeling server does, so a reader never sees half a file. */ +async function write(items: CalibrationItem[]) { + const partial = new URL('./calibration.json.partial', import.meta.url) + await writeFile(partial, `${JSON.stringify({ seed, items } satisfies CalibrationFile, null, 2)}\n`) + await rename(partial, calibrationFile) +} + +async function main() { + const previous = await readFile(calibrationFile, 'utf8').then(text => JSON.parse(text) as CalibrationFile, () => ({ items: [] })) + const labeled = new Map(previous.items.filter(item => item.label !== null).map(item => [item.id, item])) + + const readings = await newestReadings() + if (previous.items.length > 0 && !process.argv.includes('--pick')) { + const newest = new Map(readings.map(item => [item.id, item])) + const items = previous.items.map(item => ({ ...item, judge: newest.get(item.id)?.judge ?? item.judge })) + await write(items) + process.stdout.write(`refreshed the judge verdicts of ${items.length} readings, ${labeled.size} labeled\n`) + return + } + + const languages = [...new Set(readings.map(item => item.language))].sort() + const items = languages.flatMap(language => pick(readings.filter(item => item.language === language))) + // A labeled item keeps its label and note. Its judge verdict is the newest one. + .map(item => ({ ...item, label: labeled.get(item.id)?.label ?? null, note: labeled.get(item.id)?.note ?? '' })) + // Labeled items that the new pick left out stay, so collecting again never loses work. + const kept = new Set(items.map(item => item.id)) + for (const item of labeled.values()) { + if (!kept.has(item.id)) + items.push(item) + } + + await write(items) + process.stdout.write(`collected ${items.length} readings from ${readings.length}, ${labeled.size} labeled\n`) +} + +void main() diff --git a/evals/tts-speech-formula/eval/evaluators/judge/calibration/index.html b/evals/tts-speech-formula/eval/evaluators/judge/calibration/index.html new file mode 100644 index 0000000000..ebd2226d35 --- /dev/null +++ b/evals/tts-speech-formula/eval/evaluators/judge/calibration/index.html @@ -0,0 +1,74 @@ + + + + + + Judge calibration + + + +
+
+
+

Judge calibration

+

Loading…

+ +
+
+ + +
+
+ +
+ + + +

+ 1 natural · 2 mixed · 3 spelled · press the same key again to clear · + the judge verdict shows after you label. +

+
+ +
+ + + + + + diff --git a/evals/tts-speech-formula/eval/evaluators/judge/calibration/main.ts b/evals/tts-speech-formula/eval/evaluators/judge/calibration/main.ts new file mode 100644 index 0000000000..1b36c0a5d4 --- /dev/null +++ b/evals/tts-speech-formula/eval/evaluators/judge/calibration/main.ts @@ -0,0 +1,266 @@ +/// + +import type { CalibrationFile, CalibrationItem, ReadingLabel } from './types' + +import katex from 'katex' +import rehypeStringify from 'rehype-stringify' +import remarkParse from 'remark-parse' +import remarkRehype from 'remark-rehype' + +import { unified } from 'unified' + +import 'katex/contrib/mhchem' +import 'katex/dist/katex.min.css' + +const labels: readonly ReadingLabel[] = ['natural', 'mixed', 'spelled'] +const languageNames: Readonly> = { zh: 'Mandarin Chinese', en: 'English', ja: 'Japanese', ko: 'Korean' } + +const stage = document.querySelector('#stage')! +const template = document.querySelector('#card-template')! +const stats = document.querySelector('#stats')! +const progress = document.querySelector('#progress')! +const position = document.querySelector('#position')! +const toast = document.querySelector('#toast')! +const languageSelect = document.querySelector('#language')! +const showSelect = document.querySelector('#show')! +const previousButton = document.querySelector('#previous')! +const nextButton = document.querySelector('#next')! + +/** + * Renders the markdown of a judge reason or a note. remark-rehype drops raw HTML, so the text cannot inject markup. + */ +const markdown = unified().use(remarkParse).use(remarkRehype).use(rehypeStringify) +function renderMarkdown(text: string): string { + return String(markdown.processSync(text)) +} + +let items: CalibrationItem[] = [] +/** + * The items in the order of the current filters. It is computed when the page loads or a filter changes, never when a + * label changes, so labeling an item never moves the card to another item. + */ +let order: CalibrationItem[] = [] +let index = 0 + +function computeOrder() { + const language = languageSelect.value + const show = showSelect.value + const filtered = items.filter(item => !language || item.language === language) + if (show === 'disagree') + order = filtered.filter(item => item.label !== null && item.label !== item.judge.reading) + else if (show === 'unlabeled') + order = [...filtered.filter(item => item.label === null), ...filtered.filter(item => item.label !== null)] + else + order = filtered + index = 0 +} + +let toastTimer: ReturnType | undefined +function showToast(message: string, error = false) { + toast.textContent = message + toast.classList.toggle('error', error) + toast.classList.add('visible') + clearTimeout(toastTimer) + toastTimer = setTimeout(() => toast.classList.remove('visible'), error ? 4000 : 1400) +} + +function renderStats() { + const labeled = items.filter(item => item.label !== null) + const agree = labeled.filter(item => item.label === item.judge.reading).length + const rate = labeled.length ? `${Math.round(100 * agree / labeled.length)}%` : '–' + stats.textContent = `${labeled.length} of ${items.length} labeled · the judge agrees on ${agree} of ${labeled.length} (${rate})` + progress.style.width = `${items.length ? 100 * labeled.length / items.length : 0}%` +} + +/** Renders the formula with KaTeX. A formula that KaTeX cannot render shows a short notice above its source. */ +function renderFormula(target: HTMLElement, tex: string) { + try { + target.innerHTML = katex.renderToString(tex, { displayMode: true, throwOnError: true }) + } + catch { + target.innerHTML = '

KaTeX cannot render this source.

' + } +} + +/** A prompt for any agent: translate the reading and say how a native speaker would hear it. */ +function translationPrompt(item: CalibrationItem): string { + return [ + `This is a ${languageNames[item.language] ?? item.language} reading of a math formula for a text-to-speech voice.`, + `LaTeX: ${item.reference}`, + `Reading: ${item.spoken}`, + 'Translate the reading word by word into Chinese. Then say whether a native speaker hears it as a teacher reading the meaning, or as someone spelling the markup, such as naming every bracket or saying command names.', + ].join('\n') +} + +/** The scripts that a reading in each language must use. A reading without them is in another language. */ +const languageScripts: Readonly> = { + zh: /\p{Script=Han}/u, + ja: /[\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Han}]/u, + ko: /\p{Script=Hangul}/u, +} + +function chip(text: string, strong = false): HTMLSpanElement { + const element = document.createElement('span') + element.className = strong ? 'chip strong' : 'chip' + element.textContent = text + return element +} + +/** + * Updates only what a label changes: the chosen choice, the verdict of the judge, and the stats. The note box stays, + * so its focus and cursor stay too. + */ +function renderLabel(card: HTMLElement, item: CalibrationItem) { + for (const choice of card.querySelectorAll('.choice')) + choice.setAttribute('aria-checked', String(choice.dataset.label === item.label)) + + const judge = card.querySelector('.judge')! + // The verdict shows only after the person labels, so it does not decide the label for them. + judge.hidden = item.label === null + if (item.label !== null) { + const agree = item.label === item.judge.reading + judge.className = `judge ${agree ? 'agree' : 'disagree'}` + judge.innerHTML = `
${agree ? '✓ The judge agrees' : '✗ The judge disagrees'}: ${item.judge.reading}
` + + `
${renderMarkdown(item.judge.reason)}
` + } + renderStats() +} + +function renderNotePreview(card: HTMLElement, item: CalibrationItem) { + const preview = card.querySelector('.note-preview')! + preview.hidden = !item.note.trim() + preview.innerHTML = renderMarkdown(item.note) +} + +/** Renders the card of the current item. `direction` slides it in from the side it comes from. */ +function renderCard(direction: 'next' | 'previous' | 'none' = 'none') { + index = Math.min(Math.max(index, 0), Math.max(order.length - 1, 0)) + const item = order[index] + previousButton.disabled = index <= 0 + nextButton.disabled = index >= order.length - 1 + position.textContent = order.length ? `${index + 1} / ${order.length}` : '' + renderStats() + if (!item) { + stage.innerHTML = '
No readings match the filters.
' + return + } + + const card = template.content.firstElementChild!.cloneNode(true) as HTMLElement + if (direction !== 'none') + card.classList.add(`enter-${direction}`) + card.querySelector('.meta')!.append( + chip(languageNames[item.language] ?? item.language, true), + chip(item.object), + chip(item.caseId), + ) + renderFormula(card.querySelector('.formula')!, item.reference) + card.querySelector('.source')!.textContent = item.reference + const spoken = card.querySelector('.spoken')! + spoken.textContent = item.spoken + spoken.lang = item.language + // A test object can answer in another language, for example sre speaks only English. The text is shown as it came, + // so the warning says that this is the reading itself and not a display error. + const script = languageScripts[item.language] + if (script && !script.test(item.spoken)) { + const warning = document.createElement('p') + warning.className = 'language-warning' + warning.textContent = `This reading is not in ${languageNames[item.language] ?? item.language}. The test object wrote it this way. A reading in another language does not pass.` + spoken.after(warning) + } + + for (const choice of card.querySelectorAll('.choice')) + choice.addEventListener('click', () => choose(card, item, choice.dataset.label as ReadingLabel)) + + const noteBox = card.querySelector('textarea')! + noteBox.value = item.note + // The note is kept on the item as the person types, so a label chosen before the note is saved carries the new note. + noteBox.addEventListener('input', () => { + item.note = noteBox.value + renderNotePreview(card, item) + }) + noteBox.addEventListener('change', () => void save(item, 'Note saved')) + + card.querySelector('[data-action="copy"]')!.addEventListener('click', () => { + navigator.clipboard.writeText(translationPrompt(item)) + .then(() => showToast('Translation prompt copied')) + .catch(() => showToast('The browser blocked the clipboard', true)) + }) + + stage.replaceChildren(card) + renderLabel(card, item) + renderNotePreview(card, item) +} + +/** Saves of the page, one at a time, so a later save never lands before an earlier one. */ +let saves: Promise = Promise.resolve() + +/** Saves the label and the note of an item to calibration.json through the dev server. */ +function save(item: CalibrationItem, message: string): Promise { + const body = JSON.stringify({ id: item.id, label: item.label, note: item.note }) + const run = saves.then(async () => { + const response = await fetch('/api/items', { method: 'POST', body }) + if (!response.ok) + throw new Error(`${response.status} ${await response.text()}`) + }) + saves = run.catch(() => undefined) + return run.then( + () => showToast(message), + (error: unknown) => showToast(`Not saved: ${String(error)}`, true), + ) +} + +/** Chooses a label, or clears it when it is the label the item already has. The card stays on the item. */ +function choose(card: HTMLElement, item: CalibrationItem, label: ReadingLabel) { + item.label = item.label === label ? null : label + const choice = card.querySelector(`.choice[data-label="${label}"]`)! + choice.classList.remove('pulse') + // Reading the layout restarts the animation when the same choice is pressed twice in a row. + void choice.offsetWidth + choice.classList.add('pulse') + renderLabel(card, item) + void save(item, item.label === null ? 'Label cleared' : `Saved as ${item.label}`) +} + +function move(step: number) { + const next = index + step + if (next < 0 || next >= order.length) + return + index = next + renderCard(step > 0 ? 'next' : 'previous') +} + +document.addEventListener('keydown', (event) => { + if (event.target instanceof HTMLTextAreaElement || event.target instanceof HTMLSelectElement || event.isComposing || event.metaKey || event.ctrlKey || event.altKey) + return + const card = stage.querySelector('.card') + const item = order[index] + const chosen = ['1', '2', '3'].indexOf(event.key) + if (card && item && chosen >= 0) + choose(card, item, labels[chosen]!) + else if (event.key === 'ArrowRight' || event.key === 'Enter') + move(1) + else if (event.key === 'ArrowLeft') + move(-1) + else + return + event.preventDefault() +}) +previousButton.addEventListener('click', () => move(-1)) +nextButton.addEventListener('click', () => move(1)) +for (const select of [languageSelect, showSelect]) { + select.addEventListener('change', () => { + computeOrder() + renderCard('none') + }) +} + +async function load() { + const file = await (await fetch('/api/items')).json() as CalibrationFile + items = file.items + for (const language of [...new Set(items.map(item => item.language))].sort()) + languageSelect.append(new Option(languageNames[language] ?? language, language)) + computeOrder() + renderCard() +} + +void load() diff --git a/evals/tts-speech-formula/eval/evaluators/judge/calibration/style.css b/evals/tts-speech-formula/eval/evaluators/judge/calibration/style.css new file mode 100644 index 0000000000..f3a3e623df --- /dev/null +++ b/evals/tts-speech-formula/eval/evaluators/judge/calibration/style.css @@ -0,0 +1,548 @@ +/* + * Neutral surfaces with the AIRI default hue (220.44) as the accent, see DESIGN.md. + * The label colors name a state only with the text and the check mark next to them. + */ +:root { + color-scheme: light; + --hue: 220.44; + --bg: oklch(98.5% 0.004 var(--hue)); + --surface: oklch(100% 0 0); + --surface-muted: oklch(96.5% 0.006 var(--hue)); + --border: oklch(91% 0.008 var(--hue)); + --border-strong: oklch(84% 0.012 var(--hue)); + --text: oklch(24% 0.015 var(--hue)); + --muted: oklch(52% 0.015 var(--hue)); + --primary: oklch(62% 0.14 var(--hue)); + --primary-soft: oklch(94% 0.04 var(--hue)); + --natural: oklch(62% 0.15 150); + --natural-soft: oklch(95% 0.05 150); + --mixed: oklch(70% 0.15 70); + --mixed-soft: oklch(96% 0.05 80); + --spelled: oklch(60% 0.18 25); + --spelled-soft: oklch(95% 0.04 25); + --shadow: 0 1px 2px oklch(20% 0.02 var(--hue) / 6%), 0 8px 24px oklch(20% 0.02 var(--hue) / 6%); + --radius: 14px; + --ease: cubic-bezier(0.2, 0.8, 0.2, 1); +} + +@media (prefers-color-scheme: dark) { + :root { + color-scheme: dark; + --bg: oklch(17% 0.01 var(--hue)); + --surface: oklch(21% 0.012 var(--hue)); + --surface-muted: oklch(25% 0.012 var(--hue)); + --border: oklch(30% 0.012 var(--hue)); + --border-strong: oklch(38% 0.015 var(--hue)); + --text: oklch(94% 0.006 var(--hue)); + --muted: oklch(70% 0.012 var(--hue)); + --primary: oklch(72% 0.13 var(--hue)); + --primary-soft: oklch(30% 0.05 var(--hue)); + --natural-soft: oklch(30% 0.06 150); + --mixed-soft: oklch(32% 0.06 80); + --spelled-soft: oklch(30% 0.06 25); + --shadow: 0 1px 2px oklch(0% 0 0 / 30%), 0 8px 24px oklch(0% 0 0 / 25%); + } +} + +* { + box-sizing: border-box; +} + +body { + margin: 0; + background: var(--bg); + color: var(--text); + font: 15px/1.6 ui-sans-serif, system-ui, -apple-system, 'Segoe UI', 'PingFang SC', 'Hiragino Sans', 'Apple SD Gothic Neo', sans-serif; + -webkit-font-smoothing: antialiased; +} + +main { + max-width: 880px; + margin: 0 auto; + padding: 28px 16px 72px; +} + +kbd { + display: inline-grid; + place-items: center; + min-width: 1.6em; + height: 1.6em; + padding: 0 0.4em; + border: 1px solid var(--border-strong); + border-bottom-width: 2px; + border-radius: 6px; + background: var(--surface); + color: var(--muted); + font: 600 11px/1 ui-monospace, SFMono-Regular, Menlo, monospace; +} + +/* Header */ + +.top { + display: flex; + flex-wrap: wrap; + gap: 16px; + align-items: flex-end; + justify-content: space-between; + margin-bottom: 20px; +} + +.title { + flex: 1 1 280px; +} + +h1 { + margin: 0; + font-size: 22px; + font-weight: 650; + letter-spacing: -0.01em; +} + +.stats { + margin: 2px 0 10px; + color: var(--muted); + font-size: 13px; +} + +.progress { + height: 6px; + border-radius: 999px; + background: var(--surface-muted); + overflow: hidden; +} + +.progress-bar { + width: 0; + height: 100%; + border-radius: inherit; + background: var(--primary); + transition: width 400ms var(--ease); +} + +.filters { + display: flex; + gap: 10px; +} + +.field { + display: grid; + gap: 4px; + font-size: 12px; + color: var(--muted); +} + +select { + appearance: none; + min-width: 150px; + padding: 8px 32px 8px 12px; + border: 1px solid var(--border); + border-radius: 10px; + background: var(--surface) url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='12' height='12' viewBox='0 0 24 24' fill='none' stroke='%23888' stroke-width='2.5' stroke-linecap='round' stroke-linejoin='round'%3E%3Cpath d='m6 9 6 6 6-6'/%3E%3C/svg%3E") no-repeat right 12px center; + color: var(--text); + font: inherit; + font-size: 14px; + cursor: pointer; + transition: border-color 150ms var(--ease), box-shadow 150ms var(--ease); +} + +select:hover { + border-color: var(--border-strong); +} + +/* Buttons */ + +.button, +.choice { + font: inherit; + color: inherit; + cursor: pointer; + user-select: none; + -webkit-tap-highlight-color: transparent; + transition: + background-color 150ms var(--ease), + border-color 150ms var(--ease), + color 150ms var(--ease), + box-shadow 150ms var(--ease), + transform 120ms var(--ease); +} + +.button { + display: inline-flex; + align-items: center; + gap: 8px; + padding: 8px 14px; + border: 1px solid var(--border); + border-radius: 10px; + background: var(--surface); + font-size: 14px; + font-weight: 500; +} + +.button:hover:not(:disabled) { + border-color: var(--border-strong); + background: var(--surface-muted); +} + +.button:active:not(:disabled), +.choice:active { + transform: scale(0.97); +} + +.button:disabled { + opacity: 0.4; + cursor: default; +} + +.button.ghost { + border-color: transparent; + background: transparent; + color: var(--muted); + padding-inline: 10px; +} + +.button.ghost:hover { + background: var(--surface-muted); + color: var(--text); +} + +:is(.button, .choice, select, textarea):focus-visible { + outline: none; + box-shadow: 0 0 0 3px oklch(from var(--primary) l c h / 35%); +} + +/* Card */ + +.stage { + position: relative; +} + +.card { + padding: 24px; + border: 1px solid var(--border); + border-radius: var(--radius); + background: var(--surface); + box-shadow: var(--shadow); +} + +.card.enter-next { + animation: enter-next 260ms var(--ease); +} + +.card.enter-previous { + animation: enter-previous 260ms var(--ease); +} + +@keyframes enter-next { + from { opacity: 0; transform: translateX(24px); } +} + +@keyframes enter-previous { + from { opacity: 0; transform: translateX(-24px); } +} + +.meta { + display: flex; + flex-wrap: wrap; + gap: 6px; + margin-bottom: 8px; +} + +.chip { + padding: 2px 10px; + border-radius: 999px; + background: var(--surface-muted); + color: var(--muted); + font-size: 12px; + font-weight: 500; +} + +.chip.strong { + background: var(--primary-soft); + color: var(--primary); +} + +.formula { + overflow-x: auto; + padding: 8px 0; + font-size: 1.15em; +} + +.formula .katex-display { + margin: 0.6em 0; +} + +.formula-error { + color: var(--muted); + font-size: 13px; +} + +.source { + display: block; + padding: 8px 12px; + border-radius: 8px; + background: var(--surface-muted); + color: var(--muted); + font: 12px/1.5 ui-monospace, SFMono-Regular, Menlo, monospace; + white-space: pre-wrap; + word-break: break-all; +} + +.spoken { + margin: 18px 0; + padding: 18px 20px; + border-left: 3px solid var(--primary); + border-radius: 4px 12px 12px 4px; + background: var(--primary-soft); + font-size: 21px; + line-height: 1.7; + word-break: break-word; +} + +.language-warning { + margin: -8px 0 16px; + padding: 8px 12px; + border-radius: 8px; + background: var(--spelled-soft); + color: var(--text); + font-size: 13px; +} + +/* Label choices */ + +.choices { + display: grid; + grid-template-columns: repeat(3, 1fr); + gap: 10px; +} + +.choice { + position: relative; + display: grid; + grid-template-columns: auto 1fr; + grid-template-rows: auto auto; + column-gap: 10px; + align-items: center; + padding: 12px 14px; + border: 1.5px solid var(--border); + border-radius: 12px; + background: var(--surface); + text-align: left; +} + +.choice kbd { + grid-row: span 2; +} + +.choice-name { + font-weight: 650; +} + +.choice-help { + color: var(--muted); + font-size: 12px; +} + +.choice:hover { + border-color: var(--border-strong); + background: var(--surface-muted); +} + +.choice[aria-checked='true']::after { + content: '✓'; + position: absolute; + top: 8px; + right: 12px; + font-weight: 700; +} + +.choice[data-label='natural'][aria-checked='true'] { + border-color: var(--natural); + background: var(--natural-soft); +} + +.choice[data-label='mixed'][aria-checked='true'] { + border-color: var(--mixed); + background: var(--mixed-soft); +} + +.choice[data-label='spelled'][aria-checked='true'] { + border-color: var(--spelled); + background: var(--spelled-soft); +} + +.choice[data-label='natural'][aria-checked='true']::after { color: var(--natural); } +.choice[data-label='mixed'][aria-checked='true']::after { color: var(--mixed); } +.choice[data-label='spelled'][aria-checked='true']::after { color: var(--spelled); } + +/* A short pulse confirms the choice where the eye already is. */ +.choice.pulse { + animation: pulse 320ms var(--ease); +} + +@keyframes pulse { + 40% { transform: scale(1.03); } +} + +/* Note and judge */ + +.note { + display: grid; + gap: 6px; + margin-top: 18px; + color: var(--muted); + font-size: 12px; + font-weight: 500; +} + +textarea { + width: 100%; + min-height: 64px; + padding: 10px 12px; + border: 1px solid var(--border); + border-radius: 10px; + background: var(--surface); + color: var(--text); + font: inherit; + font-size: 14px; + resize: vertical; + transition: border-color 150ms var(--ease), box-shadow 150ms var(--ease); +} + +textarea:hover { + border-color: var(--border-strong); +} + +.judge, +.note-preview { + margin-top: 14px; + padding: 12px 14px; + border-radius: 10px; + font-size: 14px; + animation: reveal 220ms var(--ease); +} + +.note-preview { + background: var(--surface-muted); +} + +.judge { + border: 1px solid var(--border); +} + +.judge.agree { + border-color: var(--natural); + background: var(--natural-soft); +} + +.judge.disagree { + border-color: var(--spelled); + background: var(--spelled-soft); +} + +.judge-head { + display: flex; + gap: 8px; + align-items: center; + margin-bottom: 4px; + font-weight: 600; +} + +@keyframes reveal { + from { opacity: 0; transform: translateY(-4px); } +} + +.markdown > :first-child { margin-top: 0; } +.markdown > :last-child { margin-bottom: 0; } +.markdown p { margin: 0.3em 0; } +.markdown code { + padding: 0.1em 0.35em; + border-radius: 5px; + background: oklch(from var(--text) l c h / 8%); + font: 0.92em ui-monospace, SFMono-Regular, Menlo, monospace; +} +.markdown ul, +.markdown ol { margin: 0.3em 0; padding-left: 1.4em; } + +.tools { + display: flex; + justify-content: flex-end; + margin-top: 10px; +} + +/* Pager and hint */ + +.pager { + display: flex; + gap: 12px; + align-items: center; + justify-content: space-between; + margin-top: 16px; +} + +.position { + color: var(--muted); + font-size: 13px; + font-variant-numeric: tabular-nums; +} + +.hint { + margin-top: 20px; + color: var(--muted); + font-size: 13px; + text-align: center; +} + +.empty { + padding: 48px 24px; + color: var(--muted); + text-align: center; +} + +/* Toast */ + +.toast { + position: fixed; + top: 20px; + right: 20px; + padding: 10px 16px; + border-radius: 999px; + background: var(--text); + color: var(--bg); + font-size: 13px; + box-shadow: var(--shadow); + opacity: 0; + pointer-events: none; + transform: translateY(-8px); + transition: opacity 200ms var(--ease), transform 200ms var(--ease); +} + +.toast.visible { + opacity: 1; + transform: translateY(0); +} + +.toast.error { + background: var(--spelled); + color: white; +} + +/* Small screens */ + +@media (max-width: 640px) { + .filters { width: 100%; } + .field { flex: 1; } + select { min-width: 0; width: 100%; } + .card { padding: 18px 16px; } + .spoken { font-size: 18px; } + .choices { grid-template-columns: 1fr; } + .pager .button { padding-inline: 10px; } +} + +@media (prefers-reduced-motion: reduce) { + *, + *::before, + *::after { + animation: none !important; + transition: none !important; + } +} diff --git a/evals/tts-speech-formula/eval/evaluators/judge/calibration/types.ts b/evals/tts-speech-formula/eval/evaluators/judge/calibration/types.ts new file mode 100644 index 0000000000..d573d7eca1 --- /dev/null +++ b/evals/tts-speech-formula/eval/evaluators/judge/calibration/types.ts @@ -0,0 +1,25 @@ +/** A reading style, the same values as the judge answers. */ +export type ReadingLabel = 'natural' | 'mixed' | 'spelled' + +/** One reading that a person labels, to measure how often the judge agrees with a person. */ +export interface CalibrationItem { + /** `||`. */ + id: string + object: string + caseId: string + language: string + /** The LaTeX source of the case. */ + reference: string + /** The spoken text that the test object wrote. */ + spoken: string + /** The median verdict of the judge when the item was collected, and the reason of its first vote. */ + judge: { reading: string, reason: string } + /** The label of a person, or null before labeling. */ + label: ReadingLabel | null + note: string +} + +export interface CalibrationFile { + seed?: number + items: CalibrationItem[] +} diff --git a/evals/tts-speech-formula/eval/evaluators/judge/calibration/vite.config.ts b/evals/tts-speech-formula/eval/evaluators/judge/calibration/vite.config.ts new file mode 100644 index 0000000000..77b7e7f766 --- /dev/null +++ b/evals/tts-speech-formula/eval/evaluators/judge/calibration/vite.config.ts @@ -0,0 +1,90 @@ +import type { Plugin } from 'vite' + +import type { CalibrationFile, ReadingLabel } from './types' + +import { readFile, rename, writeFile } from 'node:fs/promises' +import { env } from 'node:process' +import { fileURLToPath } from 'node:url' + +import { defineConfig } from 'vite' + +/** `CALIBRATION_FILE` points the tool at another file, for example a copy to test the tool without touching labels. */ +const calibrationFile = env.CALIBRATION_FILE ?? fileURLToPath(new URL('./calibration.json', import.meta.url)) +const labels = new Set(['natural', 'mixed', 'spelled']) + +/** + * Runs the updates of the file one at a time. + * + * Leaving a note box and clicking a label sends two saves at once. Two read-modify-write updates that overlap wrote + * over each other and left two JSON documents in one file, so every update waits for the one before it. The new + * file is written next to the old one and renamed over it, so a failed write never leaves half a file. + */ +let updates: Promise = Promise.resolve() +function update(change: (file: CalibrationFile) => boolean): Promise { + const run = updates.then(async () => { + const file = JSON.parse(await readFile(calibrationFile, 'utf8')) as CalibrationFile + if (!change(file)) + return false + const partial = `${calibrationFile}.partial` + await writeFile(partial, `${JSON.stringify(file, null, 2)}\n`) + await rename(partial, calibrationFile) + return true + }) + updates = run.catch(() => undefined) + return run +} + +/** + * Serves `calibration.json` to the page and writes each label back to it. + * + * - `GET /api/items` returns the file. + * - `POST /api/items` with `{ id, label, note }` sets the label of one item. `label: null` clears it. + * + * The file is the only state, so a label is saved as soon as the person chooses it. + */ +function calibrationApi(): Plugin { + return { + name: 'calibration-api', + configureServer(server) { + server.middlewares.use('/api/items', async (request, response) => { + if (request.method === 'GET') { + response.setHeader('content-type', 'application/json') + response.end(await readFile(calibrationFile, 'utf8')) + return + } + if (request.method !== 'POST') { + response.statusCode = 405 + response.end() + return + } + + let body = '' + for await (const chunk of request) + body += chunk + const { id, label, note } = JSON.parse(body) as { id: string, label: ReadingLabel | null, note: string } + if (label !== null && !labels.has(label)) { + response.statusCode = 400 + response.end('unknown label') + return + } + const found = await update((file) => { + const item = file.items.find(candidate => candidate.id === id) + if (!item) + return false + item.label = label + item.note = note + return true + }) + response.statusCode = found ? 200 : 404 + response.end(found ? 'ok' : 'unknown item') + }) + }, + } +} + +/** The labeling tool of the judge calibration set. Run: pnpm -F @proj-airi/evals-tts-speech-formula run judge:label */ +export default defineConfig({ + root: fileURLToPath(new URL('.', import.meta.url)), + plugins: [calibrationApi()], + server: { port: Number(env.CALIBRATION_PORT ?? 5299), open: !env.CALIBRATION_FILE }, +}) diff --git a/evals/tts-speech-formula/eval/evaluators/judge/index.test.ts b/evals/tts-speech-formula/eval/evaluators/judge/index.test.ts new file mode 100644 index 0000000000..364e51b997 --- /dev/null +++ b/evals/tts-speech-formula/eval/evaluators/judge/index.test.ts @@ -0,0 +1,32 @@ +import { describe, expect, it } from 'vitest' + +import { judgePrompt, medianReading } from '.' + +describe('medianReading', () => { + const votes = (...readings: ('natural' | 'mixed' | 'spelled')[]) => readings.map(reading => ({ reading, reason: '' })) + + it('takes the reading of two votes of three', () => { + expect(medianReading(votes('natural', 'spelled', 'natural'))).toBe('natural') + expect(medianReading(votes('spelled', 'natural', 'spelled'))).toBe('spelled') + }) + + it('selects mixed when three votes differ', () => { + expect(medianReading(votes('natural', 'mixed', 'spelled'))).toBe('mixed') + }) +}) + +describe('judgePrompt', () => { + // Velin escapes backslashes in plain text, so the LaTeX example must stay in a code span. + it('names the language and keeps the LaTeX example unchanged', async () => { + const prompt = await judgePrompt('ja') + expect(prompt).toContain('Japanese') + expect(prompt).toContain(`\`${String.raw`$\frac{1}{2}$`}\``) + expect(prompt).not.toContain(' { + const prompt = await judgePrompt('ja') + for (const reading of ['大なりイコール', '下付き', '下标', '아래첨자', 'dot dot dot']) + expect(prompt).toContain(reading) + }) +}) diff --git a/evals/tts-speech-formula/eval/evaluators/judge/index.ts b/evals/tts-speech-formula/eval/evaluators/judge/index.ts new file mode 100644 index 0000000000..88b6369b5a --- /dev/null +++ b/evals/tts-speech-formula/eval/evaluators/judge/index.ts @@ -0,0 +1,107 @@ +import type { TaskRunContext } from 'vieval/config' + +import type { Language } from '../../cases/formula-speech/languages' +import type { Endpoint } from '../listener' + +import { object, parse, picklist, string } from 'valibot' + +import { languageName } from '../../cases/formula-speech/languages' +import { cacheKey } from '../../shared/cache-key' +import { generateTextWithTelemetry } from '../../shared/telemetry' +import { renderVelinPrompt } from '../../shared/velin' + +/** + * How a reading says a formula. + * + * - `natural`: it says the meaning, as a teacher reads the formula to a class. + * - `mixed`: it is mostly natural, with one or two parts that spell markup. + * - `spelled`: it spells the markup, for example "open parenthesis x close parenthesis", or leaves the source. + */ +export type Reading = 'natural' | 'mixed' | 'spelled' + +/** + * The judge score of each reading, recorded as `formula.natural`. The headline score `formula.pass` needs a correct + * reading and at least the mixed score, so a reading that the listener can write down but that spells markup fails. + */ +export const readingScores: Readonly> = { natural: 1, mixed: 0.5, spelled: 0 } + +/** Independent judge answers per case. The case takes the median, so one unstable answer does not decide it. */ +export const judgeVotes = 3 + +/** One vote of the judge. */ +export interface Verdict { + reading: Reading + /** One short sentence that names the spelled parts. Reports show it, so a reader can check the judge. */ + reason: string +} + +/** The answer contract in `judge.velin.md`. The JSON Schema below sends the same shape to the provider. */ +const verdictSchema = object({ reason: string(), reading: picklist(['natural', 'mixed', 'spelled']) }) + +const verdictFormat = { + type: 'json_schema', + json_schema: { + name: 'judge_verdict', + strict: true, + schema: { + type: 'object', + additionalProperties: false, + // The reason comes first, so the model names the spelled parts before it chooses the reading. + required: ['reason', 'reading'], + properties: { + reason: { type: 'string' }, + reading: { type: 'string', enum: ['natural', 'mixed', 'spelled'] }, + }, + }, + }, +} + +/** The system prompt of the judge. It judges only the style of a reading. The listener and the grader judge correctness. */ +export function judgePrompt(language: Language): Promise { + return renderVelinPrompt(new URL('./judge.velin.md', import.meta.url), { language: languageName[language] }) +} + +/** + * The median reading of several votes. With three votes, two votes decide, and a tie of three different readings + * selects `mixed`. + * + * @example + * medianReading([{ reading: 'natural', reason: '' }, { reading: 'spelled', reason: '' }, { reading: 'natural', reason: '' }]) + * // => 'natural' + */ +export function medianReading(verdicts: readonly Verdict[]): Reading { + const sorted = verdicts.map(({ reading }) => reading).sort((left, right) => readingScores[left] - readingScores[right]) + return sorted[Math.floor((sorted.length - 1) / 2)] ?? 'spelled' +} + +/** + * Judges whether a reading says the meaning of a formula or spells its markup. + * + * It judges the spoken text that the test object sends to TTS, not the ASR transcript, because ASR errors are not part + * of the reading. An empty reading makes no call and is `spelled`: it says nothing of the formula. + * + * Each vote is cached by model, prompt, source, reading, and vote number, so an unchanged reading gets the same verdict + * in every run and under every TTS model. `context` receives the usage of each call that reaches the model. + */ +export async function judgeReading(spoken: string, source: string, language: Language, vote: number, endpoint: Endpoint, context: TaskRunContext, signal: AbortSignal): Promise { + if (!spoken.trim()) + return { reading: 'spelled', reason: 'The reading is empty.' } + + const system = await judgePrompt(language) + const file = context.cache.namespace('judge').file({ ext: 'json', key: [endpoint.model, cacheKey(system, source, spoken, String(vote))] }) + if (await file.exists()) + return file.readJson() + + const { text } = await generateTextWithTelemetry(context, 'judge', { + ...endpoint, + abortSignal: signal, + maxCompletionTokens: 400, + messages: [{ role: 'system', content: system }, { role: 'user', content: `LaTeX source: ${source}\nSpoken text: ${spoken}` }], + responseFormat: verdictFormat, + temperature: 0, + }) + // A malformed verdict throws. The case then fails as a judge error instead of scoring a wrong 0. + const verdict = parse(verdictSchema, JSON.parse(text ?? '')) + await file.writeJson(verdict) + return verdict +} diff --git a/evals/tts-speech-formula/eval/evaluators/judge/judge.velin.md b/evals/tts-speech-formula/eval/evaluators/judge/judge.velin.md new file mode 100644 index 0000000000..4ceb76b554 --- /dev/null +++ b/evals/tts-speech-formula/eval/evaluators/judge/judge.velin.md @@ -0,0 +1,57 @@ + + +## Role + +You judge how a math formula is read aloud in {{ props.language }}. A text-to-speech voice reads the text, and a listener only hears it. + +You get the LaTeX source and the spoken text. The source can also be a sentence with prices and math. Then judge only how the math and the money are read. + +## What to judge + +Judge only the style of the reading. Another check grades whether the reading is correct, so never judge correctness. Never judge length alone. + +A natural reading says the meaning with the words that a teacher uses to read the formula to a class, in {{ props.language }}, as short as the meaning allows. + +- It can add a structure word where the grouping would be lost, for example "the quantity p plus q, squared" or "end sub". +- It can use "the fraction with numerator ... and denominator ..." when the numerator or the denominator is compound or has a power. For `6/\pi^2` the long form is natural, because "six over pi squared" can be heard as `(6/\pi)^2`. For `1/2`, the long form is mixed. +- It can name brackets that carry meaning, for example the square brackets of a class, or the parentheses of a call with more than one argument. +- It says the case of a letter only when the formula has both cases of that letter, and then in the short form, for example 大A and 小a, or capital A. `a` and `A` are the two cases of one letter, so `b \in B` needs its case words, and so do `\psi` and `\Psi`, and `A` and `B` in `g: A \to B, a \mapsto b`. Saying the case of a letter that has one case in the formula, for example capital for a lone `\Sigma`, is markup. + +## Standard readings + +These are the words of textbooks and classrooms. They are natural, never spelled markup, even when they name a symbol or a script. The list shows examples, not every word: another common word with the same meaning, such as Japanese 添字 for a subscript, is natural too. + +- English: "f of x", "x squared", "x sub one", "x to the power of n", "one over two", "the sum from k equals one to n of", "the set of x such that", "and so on" or "up to" for an ellipsis, "x prime", "x bar", "x hat". +- Mandarin Chinese: 下标 for a subscript, 的平方 and 次方 for powers, 分之 for a fraction, 分子 and 分母 for a long fraction, p 与 q 的和的平方 for `(p+q)^2`, 依此类推 for an ellipsis, English loanwords for functions such as sine or log. +- Japanese: 下付き for a subscript, の二乗 and 乗 for powers, 分の for a fraction, 分子が and 分母が for a long fraction, イコール, ノットイコール, 大なり, 小なり, 大なりイコール, 小なりイコール, ニアリーイコール, かっこ and かっことじ around a compound part, 大文字の and 小文字の for letter case. +- Korean: 아래첨자 for a subscript, 제곱 for powers, 분의 for a fraction, 괄호 and 괄호 닫고 around a compound part, 대문자 and 소문자 for letter case. +- In Japanese and Korean, かっこ and かっことじ, or 괄호 and 괄호 닫고, around a compound part is how a teacher groups it, also around the main structure: `(p+q)^2` is かっこピープラスキューかっことじの二乗, and `\sqrt{p^2+q^2}` is ルートかっこ…かっことじ. These languages have no other phrase for the group, so these words are natural, not spelled. +- Chemistry in every language: the element symbols of a formula read letter by letter, for example "N A H C O three" for `NaHCO3`, and the names of compounds. + +## Spelled markup + +A spelled reading reads the markup instead of the meaning. + +- It names most parentheses, brackets, braces, or commas, for example "open parenthesis x close parenthesis" for `f(x)`. A bracket word around every simple item is markup, even in the languages above. +- It names where a script or an argument starts and ends when the grouping is already clear, for example "start exponent", "subscript beginning", or "subscript ending". +- It says LaTeX command names that are not words of the language, for example "mathbb", "frac", "varsigma", "overline", "backslash", "left", or "right". +- It leaves LaTeX source or symbols for the voice, for example `$\frac{1}{2}$`, `^`, `_`, or `\alpha`. +- It names the brackets of the main structure when the language has a natural phrase for it, for example "open paren p plus q close paren squared" for `(p+q)^2` in English, where "the square of the quantity p plus q" exists, or 左花括号 and 右花括号 around a set. This is spelled, not mixed. Japanese and Korean group words are not this case, see the standard readings. +- It says "dot dot dot" or 点点点 for an ellipsis. This makes the reading at least mixed. + +## Language + +The reading must be in {{ props.language }}, apart from letter names, loanwords, and the names of functions. A reading that is mostly in another language is `spelled`, even when its words would be natural in that other language: the listener asked for {{ props.language }} and cannot use it. + +## Answer + +First write `reason`: one short English sentence that names each spelled part, or says that no part is spelled. Then write `reading`, and make it agree with the reason. + +- `natural`: the reason names no spelled part. +- `mixed`: most of the reading is natural, and the reason names one or two spelled parts. +- `spelled`: most of the reading spells markup, it leaves the source, or it is mostly in another language. diff --git a/evals/tts-speech-formula/eval/evaluators/listener/index.test.ts b/evals/tts-speech-formula/eval/evaluators/listener/index.test.ts new file mode 100644 index 0000000000..98dc551b72 --- /dev/null +++ b/evals/tts-speech-formula/eval/evaluators/listener/index.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, it } from 'vitest' + +import { listenerPrompt } from '.' + +// Velin renders markdown and writes markdown back, which escapes backslashes, underscores, and braces in plain text. +// The prompts keep every LaTeX example in a code span, so the listener sees it unchanged. +describe('listenerPrompt', () => { + it.each(['formula', 'chem', 'sentence'] as const)('keeps every LaTeX example unchanged for the %s form', async (form) => { + const prompt = await listenerPrompt('zh', form) + for (const example of [String.raw`\frac{B}{A}`, String.raw`\{x \mid ...\}`, String.raw`\sum_{k=1}^{n}`, String.raw`\sigma_0`, String.raw`7\,\mathrm{kg}`, String.raw`^\circ\mathrm{C}`]) + expect(prompt).toContain(`\`${example}\``) + expect(prompt).not.toContain(' { + expect(await listenerPrompt('en', 'chem')).toContain(String.raw`\ce{...}`) + expect(await listenerPrompt('en', 'sentence')).toContain(String.raw`\$19.99`) + expect(await listenerPrompt('en', 'formula')).not.toContain(String.raw`\ce{...}`) + }) +}) diff --git a/evals/tts-speech-formula/eval/evaluators/listener/index.ts b/evals/tts-speech-formula/eval/evaluators/listener/index.ts new file mode 100644 index 0000000000..62771047f8 --- /dev/null +++ b/evals/tts-speech-formula/eval/evaluators/listener/index.ts @@ -0,0 +1,164 @@ +import type { TaskRunContext } from 'vieval/config' + +import type { GradingMode } from '../../cases/formula-speech/cases' +import type { Language } from '../../cases/formula-speech/languages' + +import { object, parse, picklist, string } from 'valibot' + +import { languageName } from '../../cases/formula-speech/languages' +import { cacheKey } from '../../shared/cache-key' +import { generateTextWithTelemetry } from '../../shared/telemetry' +import { renderVelinPrompt } from '../../shared/velin' + +/** An OpenAI-compatible endpoint and the model to call on it. */ +export interface Endpoint { + baseURL: string + apiKey: string + model: string +} + +/** What the listener is told about the form of the audio. It never sees the reference. */ +export type AnswerForm = 'formula' | 'chem' | 'sentence' + +/** + * The system prompt of the listener. It asks for a literal transcription, never a correction toward a known formula, + * so that a misread formula stays wrong and the deterministic grader can see it. + */ +export function listenerPrompt(language: Language, form: AnswerForm): Promise { + return renderVelinPrompt(new URL('./listener.velin.md', import.meta.url), { language: languageName[language], form }) +} + +/** + * Independent listener answers per case. The case passes with a strict majority of passing answers. + * Three votes hold an unstable answer of the listener model to one vote of three. + */ +export const listenerVotes = 3 + +export function answerFormOf(mode: GradingMode): AnswerForm { + if (mode === 'chem') + return 'chem' + if (mode === 'currency') + return 'sentence' + return 'formula' +} + +/** + * How well the listener could hear the transcript. + * + * - `ok`: it heard all of it. + * - `partial`: some parts were garbled or missing. The answer has the parts that it heard, and the grader grades it. + * - `unclear`: nothing was recoverable. The grader scores the vote 0 with the rule `unclear`, so a report can tell + * an unintelligible clip apart from a wrong reading. + */ +export type HearingStatus = 'ok' | 'partial' | 'unclear' + +/** One vote of the listener. */ +export interface ListenerAnswer { + /** The LaTeX that the listener wrote down. Empty when the status is `unclear` or the transcript was empty. */ + latex: string + status: HearingStatus +} + +/** The answer contract in `listener.velin.md`. The JSON Schema below sends the same shape to the provider. */ +const answerSchema = object({ latex: string(), status: picklist(['ok', 'partial', 'unclear']) }) + +/** Structured output, so the hearing status is a field and never mixed into the LaTeX. */ +const answerFormat = { + type: 'json_schema', + json_schema: { + name: 'listener_answer', + strict: true, + schema: { + type: 'object', + additionalProperties: false, + required: ['latex', 'status'], + properties: { + latex: { type: 'string' }, + status: { type: 'string', enum: ['ok', 'partial', 'unclear'] }, + }, + }, + }, +} + +/** JSON escapes that are also the first letter of a TeX command: `\f` of `\frac`, `\t` of `\times`, `\n` of `\neq`. */ +const swallowedCommandLetters: Readonly> = { '\f': 'f', '\t': 't', '\n': 'n', '\b': 'b', '\r': 'r' } + +/** + * Restores TeX commands that a JSON parser turned into control characters. + * + * A model can write `"\frac"` in its JSON answer without escaping the backslash. `JSON.parse` then reads `\f` as a form + * feed, and the answer becomes `rac{1}{x}`. A LaTeX answer has no control characters, so each one is a lost command. + * Cached answers from before this repair get it when they are read. + * + * @example + * // JSON.parse('{"latex": "x^{-1}=\frac{1}{x}"}').latex has a form feed in place of `\f`. + * restoreTexCommands('x^{-1}=\u000Crac{1}{x}') + * // => 'x^{-1}=\\frac{1}{x}' + */ +function restoreTexCommands(latex: string): string { + return latex.replace(/[\f\t\n\u0008\r]/g, control => `\\${swallowedCommandLetters[control]}`) +} + +/** True when a LaTeX answer still holds words: CJK text, or a Latin word of four or more letters outside commands. */ +function hasWords(answer: string): boolean { + const bare = answer.replace(/\\[A-Z]+/gi, ' ') + return /[\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Han}\p{Script=Hangul}]/u.test(bare) || /[A-Za-z][a-z]{3,}/.test(bare) +} + +/** The model sometimes answers with display math over several lines. All of it is kept, on one line. */ +function oneLine(answer: string): string { + return answer.trim().split('\n').map(line => line.trim()).filter(Boolean).join(' ') +} + +async function ask(endpoint: Endpoint, system: string, prompt: string, context: TaskRunContext, signal: AbortSignal): Promise { + const { text } = await generateTextWithTelemetry(context, 'listener', { + ...endpoint, + abortSignal: signal, + // WORKAROUND: + // A garbled transcript makes the listener model write without end, and the case never finishes. + // whisper-1 can fall into a repetition loop on a short clip, for example `L-R-R-R-...` for Korean `l 1은 l 2와 평행`. + // Source: the `rel-parallel` case in Korean on the official StepFun TTS, 2026-10-19. T-157 used the same cap. + // Remove when the listener model stops on its own for such input. The step timeout in tasks/read-aloud stays as a backstop. + maxCompletionTokens: 2000, + messages: [{ role: 'system', content: system }, { role: 'user', content: prompt }], + responseFormat: answerFormat, + temperature: 0, + }) + // A malformed answer throws. The case then fails as a listener error instead of scoring a wrong 0. + const answer = parse(answerSchema, JSON.parse(text ?? '')) + return { latex: oneLine(restoreTexCommands(answer.latex)), status: answer.status } +} + +/** + * Writes down, as LaTeX, what the listener heard, with how well it could hear it. + * An empty transcript makes no call and is `unclear`. + * + * Each vote is cached by model, prompt, transcript, and vote number. An unchanged transcript therefore gets the same answer + * in every run, and run-to-run noise comes only from the system under test. + * A formula answer that still holds words gets one format repair. The repair is kept only when it is clean. + * `context` is the case context: it holds the cache, and it receives the usage of each call that reaches the model. + */ +export async function writeDown(heard: string, language: Language, form: AnswerForm, vote: number, endpoint: Endpoint, context: TaskRunContext, signal: AbortSignal): Promise { + if (!heard.trim()) + return { latex: '', status: 'unclear' } + + const system = await listenerPrompt(language, form) + const file = context.cache.namespace('listener').file({ ext: 'json', key: [endpoint.model, cacheKey(system, heard, String(vote))] }) + if (await file.exists()) { + const cached = await file.readJson() + return { ...cached, latex: restoreTexCommands(cached.latex) } + } + + const prompt = `ASR transcript: ${heard}` + let answer = await ask(endpoint, system, prompt, context, signal) + if (form !== 'sentence' && answer.status !== 'unclear' && hasWords(answer.latex)) { + const repaired = await ask(endpoint, system, `${prompt}\nYour previous answer was: ${answer.latex}\nIt still contains words. Write it again as notation only, still exactly as heard.`, context, signal) + // A failed repair keeps the first answer. The grader then scores it as written. + .catch(() => answer) + if (!hasWords(repaired.latex)) + answer = repaired + } + + await file.writeJson(answer) + return answer +} diff --git a/evals/tts-speech-formula/eval/evaluators/listener/listener.velin.md b/evals/tts-speech-formula/eval/evaluators/listener/listener.velin.md new file mode 100644 index 0000000000..831d767f4c --- /dev/null +++ b/evals/tts-speech-formula/eval/evaluators/listener/listener.velin.md @@ -0,0 +1,73 @@ + + +## Role + +You are a listener. A text-to-speech voice read something in {{ props.language }}, and you get the ASR transcript of that audio. The transcript can contain homophone errors, for example 派, パイ, or 파이 for pi, and "E B" for "A B". + +## What the audio was + +
+ +The audio was one chemical equation or formula. Write it as `\ce{...}` with standard chemical capitalisation, for example `KNO3`, `SO3`, and `Mg(OH)2`. + +- Species said by name (hydrogen, 氢气, 水素, 수소, water, 水, 물) become their usual formula, for example `H2` and `H2O`. +- Keep the coefficients, the species, and the arrow that were said. +- Use `<=>` only if a reversible reaction was said. + +
+
+ +The audio was a sentence that may contain prices and math. + +- Keep the words. +- Write every amount of money as `\$N`, for example `\$19.99`. "two dollars" is `\$2`. +- Put any math between `$...$`. + +
+
+ +The audio was one math formula. Write it down as a single LaTeX formula with NO words of any language left in it. + +
+ +## Rules + +1. Write exactly what was said. + - Never correct, complete, or "fix" it toward a well-known formula. + - If a part is missing or garbled, leave it missing or write what was literally said. + - A wrong-sounding number or letter stays as heard: "n minus e" is `n-e`, not `n-1`. +2. Fractions follow the spoken order. + - Chinese "A分之B", Japanese "A分のB", and Korean "A 분의 B" all mean `\frac{B}{A}`. This also holds when A is 1: Korean "1분의 x" is `\frac{x}{1}`. + - English "A over B" means `\frac{A}{B}`. + - "numerator X, denominator Y" means `\frac{X}{Y}`. +3. Japanese 小なり and 大なり (also heard as ショーなり and だいなり) are `<` and `>`. With イコール, they are `\leq` and `\geq`. +4. Letter case: write a Latin or Greek letter in upper case only if the speech says so (capital, upper, cap, 大写, 大文字, 대문자). The capitalisation in the ASR text is a guess and does not count. Otherwise, write the letter in lower case. This rule does not apply to chemistry. +5. Structure words become notation. + - "the set of x such that ..." is `\{x \mid ...\}`. + - "sum from k=1 to n of" is `\sum_{k=1}^{n}`. + - "the quantity p plus q, squared" is `(p+q)^2`. + - A power applies only to the letter or the group right before it, as in written notation. "a b squared", Chinese "ab的平方" and "ab方", Japanese "エービーの二乗", and Korean "에이 비 제곱" are `ab^2`. Only a spoken group, as "the quantity a b, squared" or "a与b的积的平方", is `(ab)^2`. + - "all of that over 2" is `\frac{...}{2}`. + - "limit as n approaches infinity" is `\lim_{n \to \infty}`. + - "f x" or "f of x" is `f(x)`. + - "sigma sub 0" is the letter `\sigma_0`. Only "sum" or "the sum from ... to ..." is `\sum`. + - "line 1" is `l_1`. + - "the (set of) rational / real / integer / natural / complex numbers" is `\mathbb{Q}` / `\mathbb{R}` / `\mathbb{Z}` / `\mathbb{N}` / `\mathbb{C}`. +6. Units become SI symbols in `\mathrm{}`, for example `7\,\mathrm{kg}`, `\mathrm{m/s^2}`, and `^\circ\mathrm{C}`. + +## Output + +Answer with a JSON object that has two fields: + +- `latex`: what you wrote down, on one line, with no explanation. +- `status`: how well you could hear it. + - `ok`: you heard all of it. + - `partial`: some parts are garbled or missing. `latex` has the parts that you heard. + - `unclear`: the transcript says nothing recoverable. `latex` is empty. diff --git a/evals/tts-speech-formula/eval/evaluators/notation/index.test.ts b/evals/tts-speech-formula/eval/evaluators/notation/index.test.ts new file mode 100644 index 0000000000..e05b48d4d0 --- /dev/null +++ b/evals/tts-speech-formula/eval/evaluators/notation/index.test.ts @@ -0,0 +1,25 @@ +import { describe, expect, it } from 'vitest' + +import { leftoverNotation } from '.' + +describe('leftoverNotation', () => { + it('finds characters that a voice cannot read as words', () => { + expect(leftoverNotation('9.8 m/s^2', String.raw`$9.8\,\mathrm{m/s^2}$`)).toEqual(['/', '^']) + expect(leftoverNotation('りんごは $3、バナナは $5 です。', 'りんごは $3、バナナは $5 です。')).toEqual(['$', '$']) + }) + + it('finds a command name of the input that came through as a word, also next to CJK text', () => { + expect(leftoverNotation('f X 趋于 Y x mapsto y', String.raw`$f: X \to Y, x \mapsto y$`)).toEqual(['mapsto']) + expect(leftoverNotation('x mapsto是y', String.raw`$x \mapsto y$`)).toEqual(['mapsto']) + }) + + it('accepts command names that are also words of a reading', () => { + expect(leftoverNotation('3 times 10 to the power of 8', String.raw`$3 \times 10^{8}$`)).toEqual([]) + expect(leftoverNotation('omega is not equal to Omega', String.raw`$\Omega \neq \omega$`)).toEqual([]) + expect(leftoverNotation('l 1 is parallel to l 2', String.raw`$l_1 \parallel l_2$`)).toEqual([]) + }) + + it('accepts a command name that the input does not use', () => { + expect(leftoverNotation('the vector maps to its image', '$v$')).toEqual([]) + }) +}) diff --git a/evals/tts-speech-formula/eval/evaluators/notation/index.ts b/evals/tts-speech-formula/eval/evaluators/notation/index.ts new file mode 100644 index 0000000000..1311bc8b16 --- /dev/null +++ b/evals/tts-speech-formula/eval/evaluators/notation/index.ts @@ -0,0 +1,67 @@ +/** + * Characters that a TTS voice cannot read as words. Each one leaves the reading to the TTS provider: + * some read `$5` and `m/s` well, and others skip or misread them, mostly in Japanese and Korean. + */ +const notationCharacters = /[\\^_{}$|~/]/g + +/** + * LaTeX command names that are also words of a reading, so finding them in spoken text is not a leftover: + * Greek letter names, and English words such as `times` in "3 times 10" or `right` in "right arrow". + */ +const spokenCommandNames = new Set([ + 'alpha', + 'beta', + 'gamma', + 'delta', + 'epsilon', + 'zeta', + 'theta', + 'iota', + 'kappa', + 'lambda', + 'omicron', + 'sigma', + 'upsilon', + 'omega', + 'nabla', + 'angle', + 'left', + 'right', + 'over', + 'parallel', + 'partial', + 'prime', + 'star', + 'dagger', + 'subset', + 'supset', + 'tilde', + 'times', + 'triangle', +]) + +/** + * Finds the notation that a test object left in its spoken text, so a TTS voice gets symbols instead of words. + * + * It finds two kinds: a character that has no reading, such as `^`, `/`, or `$`, and a LaTeX command name of the input + * that came through as a word, such as `mapsto`. It runs on text only, so it shows these defects without TTS or ASR. + * Command names shorter than four letters are not checked, because many of them are also words, for example `in`. + * + * @example + * leftoverNotation('9.8 m/s^2', '$9.8\\,\\mathrm{m/s^2}$') + * // => ['/', '^'] + * leftoverNotation('f X approaches Y x mapsto y', '$f: X \\to Y, x \\mapsto y$') + * // => ['mapsto'] + */ +export function leftoverNotation(spoken: string, markdown: string): string[] { + const leftovers = Array.from(spoken.matchAll(notationCharacters), match => match[0]) + const commandNames = new Set(Array.from(markdown.matchAll(/\\([a-z]+)/gi), match => match[1]!)) + for (const name of commandNames) { + if (name.length < 4 || spokenCommandNames.has(name.toLowerCase())) + continue + // `\b` is an ASCII word boundary, so a name next to CJK text, for example `mapsto是`, still matches. + if (new RegExp(`\\b${name}\\b`, 'i').test(spoken)) + leftovers.push(name) + } + return leftovers +} diff --git a/evals/tts-speech-formula/eval/evaluators/speech/gemini-transcription.test.ts b/evals/tts-speech-formula/eval/evaluators/speech/gemini-transcription.test.ts new file mode 100644 index 0000000000..1ed0a36ee7 --- /dev/null +++ b/evals/tts-speech-formula/eval/evaluators/speech/gemini-transcription.test.ts @@ -0,0 +1,49 @@ +import { generateTranscription } from '@xsai/generate-transcription' +import { afterEach, describe, expect, it, vi } from 'vitest' + +import { createGeminiTranscription } from './gemini-transcription' + +const parameters = { apiKey: 'gemini-key', baseURL: 'https://gemini.example/v1beta' } + +function transcribeClip() { + const request = createGeminiTranscription().transcription({ language: 'ja', model: 'gemini-test', parameters }) + return generateTranscription({ ...request, file: new Blob([new Uint8Array([1, 2, 3])], { type: 'audio/mpeg' }), fileName: 'speech.mp3', language: 'ja' }) +} + +describe('createGeminiTranscription', () => { + afterEach(() => { + vi.restoreAllMocks() + }) + + it('sends the clip as inline audio to generateContent and answers with the transcript', async () => { + const upstream = vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response(JSON.stringify({ + candidates: [{ content: { parts: [{ text: 'thinking', thought: true }, { text: ' 2分の1 ' }] } }], + }))) + + const result = await transcribeClip() + + expect(result).toEqual({ text: '2分の1' }) + const [url, init] = upstream.mock.calls[0]! + expect(String(url)).toBe('https://gemini.example/v1beta/models/gemini-test:generateContent') + expect(init?.headers).toMatchObject({ 'x-goog-api-key': 'gemini-key' }) + const body = JSON.parse(String(init?.body)) + expect(body.contents[0].parts[0].text).toContain('"ja"') + expect(body.contents[0].parts[1].inline_data).toEqual({ data: 'AQID', mime_type: 'audio/mpeg' }) + }) + + it('answers an empty transcript when Gemini returns no candidate', async () => { + vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response(JSON.stringify({ candidates: [] }))) + + expect(await transcribeClip()).toEqual({ text: '' }) + }) + + it('passes a failed upstream status through, so xsai reports it and vieval can retry it', async () => { + vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response('rate limited', { status: 429 })) + + await expect(transcribeClip()).rejects.toThrow('429') + }) + + it('names the missing parameter of the provider', () => { + expect(() => createGeminiTranscription().transcription({ model: 'gemini-test', parameters: { baseURL: parameters.baseURL } })).toThrow('has no "apiKey" parameter') + }) +}) diff --git a/evals/tts-speech-formula/eval/evaluators/speech/gemini-transcription.ts b/evals/tts-speech-formula/eval/evaluators/speech/gemini-transcription.ts new file mode 100644 index 0000000000..fa2fd7dbbf --- /dev/null +++ b/evals/tts-speech-formula/eval/evaluators/speech/gemini-transcription.ts @@ -0,0 +1,102 @@ +import type { TranscriptionExecutor } from './speech-models' + +import { Buffer } from 'node:buffer' + +// NOTICE: +// This module has no runtime relative import, because `eval/tasks/read-aloud/config.ts` imports it through the package +// export `./gemini-transcription`. Vieval loads configs with Node type stripping, which cannot follow an extensionless +// relative import. Type imports are stripped, so they are safe. +// Source: the same constraint as `speech-models.ts`, see eval/index.ts. +// Remove this notice when vieval loads configs with an extension-resolving loader. + +/** The text part of a Gemini `generateContent` answer. Thought parts carry the reasoning, not the transcript. */ +interface GenerateContentResponse { + candidates?: Array<{ content?: { parts?: Array<{ text?: string, thought?: boolean }> } }> +} + +/** + * Tells the model to act as a speech recognizer. The listener turns the words into notation later, so this step + * must keep the spoken words and must not use what it knows about formulas to fix or complete them. + */ +function transcriptionPrompt(language: string | undefined): string { + const spoken = language ? `The speech is in the language with the BCP 47 code "${language}".` : '' + return [ + `Transcribe this audio verbatim, as a speech recognizer does. ${spoken}`, + 'Write the words that you hear. Do not convert spoken math into notation or LaTeX.', + 'Do not correct, complete, or explain anything. Output only the transcript.', + 'If you hear no intelligible speech, output nothing.', + ].join('\n') +} + +function requiredParameter(parameters: Readonly>, name: string, model: string): string { + const value = parameters[name] + if (typeof value !== 'string' || !value) + throw new Error(`Gemini transcription model "${model}" has no "${name}" parameter. Set it on its speech provider, for example through its env.`) + return value +} + +/** + * Creates a transcription executor for Gemini models, through the Gemini `generateContent` API. + * + * Use when: + * - the `asr` axis selects a Gemini model. Gemini has no `/audio/transcriptions` endpoint, so the executor's `fetch` + * turns the xsai transcription request into one `generateContent` call with the clip as inline audio. + * + * Expects: + * - the provider parameters `baseURL`, the Gemini API root such as `https://generativelanguage.googleapis.com/v1beta/`, + * and `apiKey` + * + * Returns: + * - an executor whose `fetch` answers `{ text }`, as an OpenAI-compatible transcription endpoint does. A failed upstream + * response is returned unchanged, so xsai reports its status and vieval can retry a rate limit. + */ +export function createGeminiTranscription(): TranscriptionExecutor { + return { + id: 'gemini', + transcription: ({ model, parameters }) => { + const apiKey = requiredParameter(parameters, 'apiKey', model) + const baseURL = requiredParameter(parameters, 'baseURL', model) + const root = baseURL.endsWith('/') ? baseURL : `${baseURL}/` + + return { + baseURL: root, + model, + fetch: async (_input: RequestInfo | URL, init?: RequestInit) => { + // xsai sends the clip and its options as multipart form data. See `generateTranscription` of @xsai/generate-transcription. + if (!(init?.body instanceof FormData)) + throw new TypeError('Gemini transcription expects the multipart request of xsai.') + const file = init.body.get('file') + if (!(file instanceof Blob)) + throw new TypeError('Gemini transcription request has no audio file.') + const language = init.body.get('language') + + const response = await fetch(new URL(`models/${model}:generateContent`, root), { + body: JSON.stringify({ + contents: [{ + parts: [ + { text: transcriptionPrompt(typeof language === 'string' ? language : undefined) }, + { inline_data: { data: Buffer.from(await file.arrayBuffer()).toString('base64'), mime_type: file.type || 'audio/mpeg' } }, + ], + }], + // A transcript of the same clip must not change between runs. + generationConfig: { temperature: 0 }, + }), + headers: { 'Content-Type': 'application/json', 'x-goog-api-key': apiKey }, + method: 'POST', + signal: init.signal, + }) + if (!response.ok) + return response + + const answer = await response.json() as GenerateContentResponse + const parts = answer.candidates?.[0]?.content?.parts + // No candidate means the model heard nothing to transcribe, so the transcript is empty and the listener marks the case `unclear`. + const text = parts == null + ? '' + : parts.filter(part => !part.thought && typeof part.text === 'string').map(part => part.text).join('').trim() + return new Response(JSON.stringify({ text }), { headers: { 'Content-Type': 'application/json' } }) + }, + } + }, + } +} diff --git a/evals/tts-speech-formula/eval/evaluators/speech/index.ts b/evals/tts-speech-formula/eval/evaluators/speech/index.ts new file mode 100644 index 0000000000..1b32605466 --- /dev/null +++ b/evals/tts-speech-formula/eval/evaluators/speech/index.ts @@ -0,0 +1,86 @@ +import type { TaskCacheRuntime } from 'vieval/core/runner' + +import type { Language } from '../../cases/formula-speech/languages' +import type { OpenAISpeechModelRuntimeConfig, TranscriptionRequestOptions } from './speech-models' + +import { Buffer } from 'node:buffer' + +import { generateSpeech } from '@xsai/generate-speech' +import { generateTranscription } from '@xsai/generate-transcription' + +import { voiceLanguage } from '../../cases/formula-speech/languages' +import { cacheKey } from '../../shared/cache-key' +import { voiceFor } from './speech-models' + +export * from './speech-models' + +/** One synthesized clip, kept in the vieval cache so that a rerun reuses it. */ +export interface SpeechClip { + /** Cache file of the mp3. Reports copy it as the case artifact. */ + path: string + bytes: Buffer + /** True when the clip came from the cache, so this run sent no TTS request for it. */ + cached: boolean +} + +/** One transcript of a clip. */ +export interface Transcript { + text: string + /** True when the transcript came from the cache, so this run sent no ASR request for it. */ + cached: boolean +} + +/** + * Synthesizes the spoken text with the TTS model that the `tts` axis selected. + * The clip is cached by endpoint, model, voice, and text. A model that takes the language also gets the BCP 47 tag of the + * case language, and the tag is part of its cache key, so a clip that was read without a language is not reused. + */ +export async function synthesize(speech: OpenAISpeechModelRuntimeConfig, text: string, language: Language, cache: TaskCacheRuntime, signal: AbortSignal): Promise { + const voice = voiceFor(speech, language) + if (!voice) + throw new Error(`TTS model ${speech.model} has no voice for ${language}.`) + + const tag = speech.sendsLanguage ? voiceLanguage[language] : undefined + const keyParts = tag ? [speech.baseURL, voice, text, tag] : [speech.baseURL, voice, text] + const file = cache.namespace('speech').file({ ext: 'mp3', key: [speech.model, cacheKey(...keyParts)] }) + if (await file.exists()) + return { path: file.path, bytes: await file.readBuffer(), cached: true } + + const audio = await generateSpeech({ + abortSignal: signal, + apiKey: speech.apiKey, + baseURL: speech.baseURL, + input: text, + model: speech.model, + responseFormat: 'mp3', + voice, + ...(tag ? { extraBody: { language: tag } } : {}), + }) + const bytes = Buffer.from(audio) + await file.writeBuffer(bytes) + return { path: file.path, bytes, cached: false } +} + +/** + * Transcribes a clip with the ASR model that the `asr` axis selected. + * `modelId` is the registered model id, because an executor model has no endpoint that tells two models apart. + * The transcript is cached by model id, language, and audio content. + */ +export async function transcribe(transcription: TranscriptionRequestOptions, modelId: string, clip: SpeechClip, language: Language, cache: TaskCacheRuntime, signal: AbortSignal): Promise { + const file = cache.namespace('transcription').file({ ext: 'json', key: [modelId, cacheKey(language, clip.bytes.toString('base64'))] }) + if (await file.exists()) + return { text: (await file.readJson<{ text: string }>()).text, cached: true } + + const { text } = await generateTranscription({ + ...transcription, + abortSignal: signal, + file: new Blob([new Uint8Array(clip.bytes)], { type: 'audio/mpeg' }), + fileName: 'speech.mp3', + language, + // A transcript of the same clip must not change between runs. + temperature: '0', + }) + const transcript = text.replace(/\s*\n\s*/g, ' ').trim() + await file.writeJson({ text: transcript }) + return { text: transcript, cached: false } +} diff --git a/evals/tts-speech-formula/eval/evaluators/speech/speech-models.test.ts b/evals/tts-speech-formula/eval/evaluators/speech/speech-models.test.ts new file mode 100644 index 0000000000..806ba414c8 --- /dev/null +++ b/evals/tts-speech-formula/eval/evaluators/speech/speech-models.test.ts @@ -0,0 +1,157 @@ +import { describe, expect, it } from 'vitest' + +import { openaiSpeechFromRunContext, openaiTranscriptionFromRunContext, speechModelFrom, speechModelFromRun, speechModelMatrix, SpeechModels, speechProviderFrom, SpeechProviders, transcriptionModelFrom, transcriptionModelFromEval, transcriptionRequestFromRunContext, voiceFor } from './speech-models' + +describe('speechProviders', () => { + it('resolves requiredEnv and optionalEnv provider parameters from config env', async () => { + const plugin = SpeechProviders({ + providers: [speechProviderFrom({ id: 'tts', inferenceExecutor: 'openai', optionalEnv: { apiKey: 'TTS_API_KEY' }, requiredEnv: { baseURL: 'TTS_BASE_URL' } })], + }) + + const result = await plugin.configVieval?.({ env: { TTS_BASE_URL: 'http://127.0.0.1:5933/v1/' }, models: [] }) + + expect(result?.speechProviders?.[0]?.parameters).toEqual({ baseURL: 'http://127.0.0.1:5933/v1/' }) + }) + + it('throws when a required env value is missing', () => { + const plugin = SpeechProviders({ + providers: [speechProviderFrom({ id: 'tts', inferenceExecutor: 'openai', requiredEnv: { baseURL: 'TTS_BASE_URL' } })], + }) + + expect(() => plugin.configVieval?.({ env: {}, models: [] })).toThrow('TTS_BASE_URL') + }) +}) + +describe('speechModels', () => { + const createMatrixContext = async () => { + const providers = SpeechProviders({ + providers: [speechProviderFrom({ id: 'local', inferenceExecutor: 'openai', parameters: { apiKey: 'provider-key', baseURL: 'http://127.0.0.1:5933/v1/' } })], + }) + const models = SpeechModels({ + models: [ + speechModelFrom({ id: 'azure', model: 'microsoft/v1', provider: 'local', voice: 'en-US-AriaNeural', voices: { zh: 'zh-CN-XiaoxiaoNeural' } }), + transcriptionModelFrom({ model: 'whisper-1', parameters: { apiKey: '' }, provider: 'local' }), + ], + }) + // A hook may return nothing to keep the config it got. + const base = { env: {}, models: [] } + const withProviders = await providers.configVieval!(base) ?? base + const config = await models.configVieval!(withProviders) ?? withProviders + + return { + cache: {} as never, + models: config.models ?? [], + task: { + entry: { description: 'desc', directory: 'd', filePath: '/tmp/a.eval.ts', id: 'entry', name: 'entry' }, + id: 'task-1', + inferenceExecutor: { id: 'default' }, + matrix: { + eval: { asr: 'whisper-1' }, + meta: { evalRowId: 'asr=whisper-1', runRowId: 'tts=azure' }, + run: { tts: 'azure' }, + }, + }, + } + } + + it('adds speech and transcription models to the shared model registry with their provider parameters', async () => { + const context = await createMatrixContext() + + expect(context.models.map(model => model.id)).toEqual(['azure', 'openai:whisper-1']) + expect(context.models[0]?.parameters).toMatchObject({ apiKey: 'provider-key', baseURL: 'http://127.0.0.1:5933/v1/', capability: 'speech' }) + }) + + it('resolves models from matrix axes and turns them into runtime config', async () => { + const context = await createMatrixContext() + + const speech = openaiSpeechFromRunContext(speechModelFromRun(context, { axis: 'tts' })) + expect(speech.model).toBe('microsoft/v1') + expect(voiceFor(speech, 'zh')).toBe('zh-CN-XiaoxiaoNeural') + expect(voiceFor(speech, 'ja')).toBe('en-US-AriaNeural') + + // A model parameter wins over the provider parameter, so a local ASR server can run without a key. + const transcription = openaiTranscriptionFromRunContext(transcriptionModelFromEval(context, { axis: 'asr' })) + expect(transcription).toEqual({ apiKey: '', baseURL: 'http://127.0.0.1:5933/v1/', inferenceExecutor: 'openai', model: 'whisper-1' }) + }) + + it('rejects a model with the wrong capability, an unknown model, and a missing axis', async () => { + const context = await createMatrixContext() + const swapped = { ...context, task: { ...context.task, matrix: { ...context.task.matrix, run: { tts: 'whisper-1' } } } } + + expect(() => speechModelFromRun(swapped, { axis: 'tts' })).toThrow('is not a speech model') + expect(() => speechModelFromRun(context, { axis: 'voice' })).toThrow('Missing run matrix axis "voice".') + expect(() => transcriptionModelFromEval({ ...context, models: [] }, { axis: 'asr' })).toThrow('Unknown configured speech model "whisper-1"') + }) + + it('registers a model without env, and fails only when a task resolves it without its base URL', async () => { + const providers = SpeechProviders({ + providers: [speechProviderFrom({ id: 'azure', inferenceExecutor: 'openai', optionalEnv: { baseURL: 'TTS_AZURE_BASE_URL' } })], + }) + const models = SpeechModels({ models: [speechModelFrom({ id: 'azure', model: 'microsoft/v1', provider: 'azure' })] }) + const base = { env: {}, models: [] } + const withProviders = await providers.configVieval!(base) ?? base + const config = await models.configVieval!(withProviders) ?? withProviders + const azure = config.models?.[0] + + expect(azure?.id).toBe('azure') + expect(() => openaiSpeechFromRunContext(azure!)).toThrow('Model "azure" has no "baseURL" parameter. Set it on speech provider "azure"') + }) + + it('resolves a transcription executor per call, with the language of the call and the resolved provider env', async () => { + const calls: Array<{ language?: string, model: string, parameters: Readonly> }> = [] + const executorFetch = async () => new Response(JSON.stringify({ text: 'ok' })) + const providers = SpeechProviders({ + providers: [speechProviderFrom({ + id: 'native', + inferenceExecutor: { + id: 'native-asr', + transcription: (options) => { + calls.push(options) + return { baseURL: 'native://asr/', fetch: executorFetch, model: options.model } + }, + }, + optionalEnv: { apiKey: 'NATIVE_API_KEY' }, + })], + }) + const models = SpeechModels({ models: [transcriptionModelFrom({ id: 'native', model: 'native-model', provider: 'native' })] }) + const base = { env: { NATIVE_API_KEY: 'native-key' }, models: [] } + const withProviders = await providers.configVieval!(base) ?? base + const config = await models.configVieval!(withProviders) ?? withProviders + const native = config.models![0]! + + expect(native.inferenceExecutorId).toBe('native-asr') + const request = transcriptionRequestFromRunContext(native, { language: 'ja' }) + expect(request).toMatchObject({ baseURL: 'native://asr/', model: 'native-model' }) + expect(request.fetch).toBe(executorFetch) + expect(calls).toEqual([{ language: 'ja', model: 'native-model', parameters: { apiKey: 'native-key', capability: 'transcription' } }]) + }) + + it('sends the language only for a model that takes it', async () => { + const providers = SpeechProviders({ + providers: [speechProviderFrom({ id: 'local', inferenceExecutor: 'openai', parameters: { baseURL: 'http://127.0.0.1:5933/v1/' } })], + }) + const models = SpeechModels({ + models: [ + speechModelFrom({ id: 'official', model: 'stepfun/stepaudio-2.5-tts', provider: 'local', sendsLanguage: true, voice: 'jilingshaonv' }), + speechModelFrom({ id: 'azure', model: 'microsoft/v1', provider: 'local', voice: 'en-US-AriaNeural' }), + ], + }) + const base = { env: {}, models: [] } + const withProviders = await providers.configVieval!(base) ?? base + const config = await models.configVieval!(withProviders) ?? withProviders + const [official, azure] = config.models! + + expect(openaiSpeechFromRunContext(official!).sendsLanguage).toBe(true) + expect(openaiSpeechFromRunContext(azure!).sendsLanguage).toBe(false) + }) + + it('throws when a model references an unknown provider', () => { + const plugin = SpeechModels({ models: [speechModelFrom({ model: 'tts-1', provider: 'missing' })] }) + + expect(() => plugin.configVieval?.({ models: [] })).toThrow('Unknown speech provider "missing"') + }) + + it('builds a matrix axis from model names', () => { + expect(speechModelMatrix('tts', 'azure', 'official', 'azure')).toEqual({ tts: ['azure', 'official'] }) + }) +}) diff --git a/evals/tts-speech-formula/eval/evaluators/speech/speech-models.ts b/evals/tts-speech-formula/eval/evaluators/speech/speech-models.ts new file mode 100644 index 0000000000..6e6e6180d0 --- /dev/null +++ b/evals/tts-speech-formula/eval/evaluators/speech/speech-models.ts @@ -0,0 +1,486 @@ +import type { generateTranscription } from '@xsai/generate-transcription' +import type { ConfigHookPlugin, MatrixDefinition, ModelDefinition, TaskRunContext } from 'vieval/config' + +import process from 'node:process' + +import { resolveModelByName } from 'vieval/config' +import { envFrom, requiredEnvFrom } from 'vieval/core/inference-executors' + +// WORKAROUND: +// This module is one file, unlike `chat-models`, which splits its runtime config into its own module. +// Configs import it, and vieval 0.4.0 loads configs with Node type stripping, which cannot follow an extensionless relative import. +// Source: see eval/index.ts. +// Remove when vieval loads configs with an extension-resolving loader, or when this module moves into vieval. +// +// This module mirrors `vieval/plugins/chat-models` for speech. It depends only on the public vieval API, +// so it can move into vieval as `vieval/plugins/speech-models`. + +/** + * What a speech model does. A `speech` model synthesizes speech from text (TTS). + * A `transcription` model turns speech into text (ASR). + */ +export type SpeechModelCapability = 'speech' | 'transcription' + +/** + * One provider definition that speech models reference by id. + * + * A provider is an OpenAI-compatible endpoint (`POST /audio/speech` and `POST /audio/transcriptions`), which most TTS and + * ASR services expose, for example unspeech and a whisper.cpp server. A transcription provider can instead be a + * {@link TranscriptionExecutor}, for example a native speech recognizer. + */ +export interface SpeechProviderDefinition { + /** + * Stable provider id referenced by `speechModelFrom({ provider })`. + */ + id: string + inferenceExecutor: 'openai' | TranscriptionExecutor + /** + * Optional provider parameters resolved from env. + * + * Expects: + * - map key is the provider parameter name, for example `apiKey` + * - map value is the env key name + */ + optionalEnv?: Record + /** + * Literal provider parameters. + */ + parameters?: Record + /** + * Required provider parameters resolved from env. A missing key throws when the config loads. + * + * Expects: + * - map key is the provider parameter name, for example `baseURL` + * - map value is the env key name + */ + requiredEnv?: Record +} + +/** + * xsai request options of one transcription call. A runtime without an HTTP endpoint provides its own `fetch`, + * as xsai providers do, so callers send every model the same `generateTranscription` request. + */ +export interface TranscriptionRequestOptions { + apiKey?: string + baseURL: string | URL + /** The `fetch` of an xsai provider, so the same type serves every provider. */ + fetch?: Parameters[0]['fetch'] + model: string +} + +/** + * A transcription runtime that is not an OpenAI-compatible endpoint, for example Apple Speech on macOS. + */ +export interface TranscriptionExecutor { + /** Stable id, used as the `inferenceExecutorId` of its models. */ + id: string + /** + * Builds the request options of one call. `language` is the language that the caller passes to `generateTranscription`, + * so a runtime that needs one recognizer per language, for example per locale, can select it. + * `parameters` are the provider and model parameters, with env already resolved. An executor that wraps an HTTP API + * that is not OpenAI-compatible reads its endpoint and key from them. + */ + transcription: (options: { language?: string, model: string, parameters: Readonly> }) => TranscriptionRequestOptions +} + +/** + * Speech-model specialization of the canonical `ModelDefinition`. + * The capability and the voices are in `parameters`, so the model stays in the shared `config.models` registry. + */ +export type SpeechModelDefinition = Omit & { + inferenceExecutor: 'openai' | TranscriptionExecutor + provider: string +} + +interface SpeechModelFromBaseOptions { + /** + * Alias names used by matrix lookup. + */ + aliases?: string[] + /** + * Stable model id. Matrix axes select the model by this id, its model name, or an alias. + * + * @default `openai:` + */ + id?: string + /** + * Concrete model name sent to the provider. + */ + model: string + /** + * Literal model parameters. They override provider parameters with the same name. + */ + parameters?: Record + /** + * Provider id from `SpeechProviders`. + */ + provider: string +} + +/** + * Builder input for `speechModelFrom`. + */ +export interface SpeechModelFromOptions extends SpeechModelFromBaseOptions { + /** + * The endpoint takes the language of the text as a BCP 47 tag in `extra_body.language`, as the AIRI speech API does. + * Without it, a provider guesses the language, and a voice can read a short text, such as a digit, in its own language. + * + * @default false + */ + sendsLanguage?: boolean + /** + * The voice for any language that `voices` does not name. + */ + voice?: string + /** + * The voice of each language, keyed by the language code that eval code passes to `voiceFor`. + */ + voices?: Record +} + +/** + * Builder input for `transcriptionModelFrom`. + */ +export interface TranscriptionModelFromOptions extends SpeechModelFromBaseOptions {} + +export interface SpeechModelsPluginOptions { + /** + * Speech and transcription model definitions to append to `config.models`. + */ + models: readonly SpeechModelDefinition[] +} + +export interface SpeechProvidersPluginOptions { + /** + * Env source for env-backed provider parameters. + * + * @default config.env, then process.env + */ + env?: NodeJS.ProcessEnv + providers: readonly SpeechProviderDefinition[] +} + +/** + * Partial config shape that the speech plugins read and write. + */ +export interface SpeechPluginConfig { + env?: NodeJS.ProcessEnv + models?: ModelDefinition[] + speechProviders?: SpeechProviderDefinition[] +} + +export type SpeechPlugin = ConfigHookPlugin + +export type MatrixModelScope = 'eval' | 'run' + +export interface SpeechModelFromMatrixOptions { + /** + * Matrix axis whose selected value is a model id, model name, or alias. + */ + axis: string +} + +type MatrixModelContext = Pick + +/** + * Builds one speech-provider definition. + * + * Use when: + * - several speech or transcription models share one endpoint and key + */ +export function speechProviderFrom(options: SpeechProviderDefinition): SpeechProviderDefinition { + return { ...options } +} + +/** + * Builds one TTS model definition. + * + * Use when: + * - a run or eval matrix axis selects the TTS model, for example to compare providers + */ +export function speechModelFrom(options: SpeechModelFromOptions): SpeechModelDefinition { + return createModelDefinition(options, { + capability: 'speech', + sendsLanguage: options.sendsLanguage ?? false, + voice: options.voice, + voices: options.voices ?? {}, + }) +} + +/** + * Builds one ASR model definition. + * + * Use when: + * - a matrix axis selects the speech recognition model, for example as part of the evaluator + */ +export function transcriptionModelFrom(options: TranscriptionModelFromOptions): SpeechModelDefinition { + return createModelDefinition(options, { capability: 'transcription' }) +} + +/** + * Creates a matrix axis from configured speech model names. + * + * Returns: + * - matrix axis object compatible with `runMatrix.extend/override` and `evalMatrix.extend/override` + */ +export function speechModelMatrix(axis: string, ...names: string[]): MatrixDefinition { + return { [axis]: Array.from(new Set(names)) } +} + +/** + * Contributes speech providers to config and resolves their env-backed parameters. + * Register it before `SpeechModels`, which reads the providers. + */ +export function SpeechProviders(options: SpeechProvidersPluginOptions): SpeechPlugin { + return { + configVieval(config) { + const env = config.env ?? options.env ?? process.env + return { + ...config, + speechProviders: [ + ...(config.speechProviders ?? []), + ...options.providers.map(provider => ({ ...provider, parameters: resolveProviderParameters(provider, env) })), + ], + } + }, + name: 'vieval:speech-providers', + } +} + +/** + * Contributes speech and transcription models to the shared `config.models` registry. + * Each model takes the parameters of its provider, and its own parameters win. + */ +export function SpeechModels(options: SpeechModelsPluginOptions): SpeechPlugin { + return { + configVieval(config) { + const providers = new Map((config.speechProviders ?? []).map(provider => [provider.id, provider])) + const models = options.models.map((model) => { + const provider = providers.get(model.provider) + if (provider == null) + throw new Error(`Unknown speech provider "${model.provider}" referenced by model "${model.id}".`) + const inferenceExecutor = provider.inferenceExecutor + return { + ...model, + inferenceExecutor, + inferenceExecutorId: typeof inferenceExecutor === 'string' ? inferenceExecutor : inferenceExecutor.id, + parameters: { ...provider.parameters, ...model.parameters }, + } + }) + + return { + ...config, + models: [...(config.models ?? []), ...models], + } + }, + name: 'vieval:speech-models', + } +} + +/** + * Resolves a configured speech or transcription model from one scoped matrix axis. + * + * Expects: + * - `options.axis` to exist in `context.task.matrix[scope]` + * - the selected model to have the expected capability + */ +export function speechModelFromMatrix( + context: MatrixModelContext, + scope: MatrixModelScope, + capability: SpeechModelCapability, + options: SpeechModelFromMatrixOptions, +): ModelDefinition { + const selected = context.task.matrix[scope][options.axis] + if (selected == null) + throw new Error(`Missing ${scope} matrix axis "${options.axis}".`) + + const model = resolveModelByName(context.models, String(selected)) + if (model == null) + throw new Error(`Unknown configured speech model "${String(selected)}" from ${scope} matrix axis "${options.axis}".`) + if (model.parameters?.capability !== capability) + throw new Error(`Model "${model.id}" from ${scope} matrix axis "${options.axis}" is not a ${capability} model.`) + + return model +} + +/** Resolves a TTS model from one run-matrix axis, for example when the TTS provider is part of the system under test. */ +export function speechModelFromRun(context: MatrixModelContext, options: SpeechModelFromMatrixOptions): ModelDefinition { + return speechModelFromMatrix(context, 'run', 'speech', options) +} + +/** Resolves a TTS model from one eval-matrix axis, for example when TTS only renders text for an evaluator. */ +export function speechModelFromEval(context: MatrixModelContext, options: SpeechModelFromMatrixOptions): ModelDefinition { + return speechModelFromMatrix(context, 'eval', 'speech', options) +} + +/** Resolves an ASR model from one run-matrix axis, for example when speech recognition is under test. */ +export function transcriptionModelFromRun(context: MatrixModelContext, options: SpeechModelFromMatrixOptions): ModelDefinition { + return speechModelFromMatrix(context, 'run', 'transcription', options) +} + +/** Resolves an ASR model from one eval-matrix axis, for example when speech recognition is part of the evaluator. */ +export function transcriptionModelFromEval(context: MatrixModelContext, options: SpeechModelFromMatrixOptions): ModelDefinition { + return speechModelFromMatrix(context, 'eval', 'transcription', options) +} + +function createModelDefinition(options: SpeechModelFromBaseOptions, parameters: Record): SpeechModelDefinition { + return { + aliases: options.aliases ?? [], + id: options.id ?? `openai:${options.model}`, + inferenceExecutor: 'openai', + inferenceExecutorId: 'openai', + model: options.model, + parameters: { ...options.parameters, ...parameters }, + provider: options.provider, + } +} + +function resolveProviderParameters(provider: SpeechProviderDefinition, env: NodeJS.ProcessEnv): Record { + const parameters: Record = { ...provider.parameters } + for (const [name, key] of Object.entries(provider.optionalEnv ?? {})) { + const value = envFrom(env, { name: key, type: 'string' }) + if (value != null) + parameters[name] = value + } + for (const [name, key] of Object.entries(provider.requiredEnv ?? {})) + parameters[name] = requiredEnvFrom(env, { name: key, type: 'string' }) + return parameters +} + +/** + * Runtime config of one OpenAI-compatible TTS model, ready for a `POST /audio/speech` request. + */ +export interface OpenAISpeechModelRuntimeConfig { + /** + * API key. A local server, for example unspeech without auth, needs none. + * + * @default '' + */ + apiKey: string + baseURL: string + inferenceExecutor: 'openai' + model: string + /** True when the endpoint takes the language of the text in `extra_body.language`. */ + sendsLanguage: boolean + /** The voice for any language that `voices` does not name. */ + voice?: string + /** The voice of each language. */ + voices: Readonly> +} + +/** + * Runtime config of one OpenAI-compatible ASR model, ready for a `POST /audio/transcriptions` request. + */ +export interface OpenAITranscriptionModelRuntimeConfig { + /** + * API key. A local server, for example whisper.cpp, needs none. + * + * @default '' + */ + apiKey: string + baseURL: string + inferenceExecutor: 'openai' + model: string +} + +/** + * Resolves the runtime config of a TTS model that a speech-model helper resolved. + * + * Expects: + * - `model` to be a `speech` model with a `baseURL` parameter + */ +export function openaiSpeechFromRunContext(model: ModelDefinition): OpenAISpeechModelRuntimeConfig { + const parameters = parametersOf(model, 'speech') + return { + apiKey: optionalString(parameters, 'apiKey', model) ?? '', + baseURL: requiredString(parameters, 'baseURL', model), + inferenceExecutor: 'openai', + model: model.model, + sendsLanguage: parameters.sendsLanguage === true, + voice: optionalString(parameters, 'voice', model), + voices: stringRecord(parameters.voices, model), + } +} + +/** + * Resolves the runtime config of an ASR model that a speech-model helper resolved. + * + * Expects: + * - `model` to be a `transcription` model with a `baseURL` parameter + */ +export function openaiTranscriptionFromRunContext(model: ModelDefinition): OpenAITranscriptionModelRuntimeConfig { + const parameters = parametersOf(model, 'transcription') + return { + apiKey: optionalString(parameters, 'apiKey', model) ?? '', + baseURL: requiredString(parameters, 'baseURL', model), + inferenceExecutor: 'openai', + model: model.model, + } +} + +/** + * Resolves the xsai request options of one transcription call, for an OpenAI-compatible model or a {@link TranscriptionExecutor}. + * + * Expects: + * - `model` to be a `transcription` model + * - `language` to be the language of the call. An executor can select its recognizer from it. + */ +export function transcriptionRequestFromRunContext(model: ModelDefinition, options: { language?: string } = {}): TranscriptionRequestOptions { + if (model.parameters?.capability !== 'transcription') + throw new Error(`Model "${model.id}" is not a transcription model.`) + if (isTranscriptionExecutor(model.inferenceExecutor)) + return model.inferenceExecutor.transcription({ language: options.language, model: model.model, parameters: model.parameters ?? {} }) + return openaiTranscriptionFromRunContext(model) +} + +function isTranscriptionExecutor(value: unknown): value is TranscriptionExecutor { + return typeof value === 'object' && value != null && 'transcription' in value && typeof value.transcription === 'function' +} + +/** + * Selects the voice of one language: the language voice, or the default voice. + * Returns `undefined` when the model has no voice for the language. + */ +export function voiceFor(config: Pick, language: string): string | undefined { + return config.voices[language] ?? config.voice +} + +function parametersOf(model: ModelDefinition, capability: string): Record { + const parameters = model.parameters ?? {} + if (model.inferenceExecutorId !== 'openai') + throw new Error(`Expected an openai model, got "${model.inferenceExecutorId}" for "${model.id}".`) + if (parameters.capability !== capability) + throw new Error(`Model "${model.id}" is not a ${capability} model.`) + return parameters +} + +function optionalString(parameters: Record, name: string, model: ModelDefinition): string | undefined { + const value = parameters[name] + if (value == null || value === '') + return undefined + if (typeof value !== 'string') + throw new Error(`Parameter "${name}" of model "${model.id}" must be a string.`) + return value +} + +function requiredString(parameters: Record, name: string, model: ModelDefinition): string { + const value = optionalString(parameters, name, model) + if (value != null) + return value + + // Providers resolve env lazily, so an unused model needs no env. A selected model reports what is missing here. + const provider = 'provider' in model && typeof model.provider === 'string' ? ` Set it on speech provider "${model.provider}", for example through its env.` : '' + throw new Error(`Model "${model.id}" has no "${name}" parameter.${provider}`) +} + +function isStringRecord(value: unknown): value is Record { + return typeof value === 'object' && value != null && Object.values(value).every(item => typeof item === 'string') +} + +function stringRecord(value: unknown, model: ModelDefinition): Record { + if (value == null) + return {} + if (!isStringRecord(value)) + throw new Error(`Parameter "voices" of model "${model.id}" must map languages to voice names.`) + return value +} diff --git a/evals/tts-speech-formula/eval/index.ts b/evals/tts-speech-formula/eval/index.ts new file mode 100644 index 0000000000..8945964aa7 --- /dev/null +++ b/evals/tts-speech-formula/eval/index.ts @@ -0,0 +1,16 @@ +// The package entry for test objects. +// +// WORKAROUND: +// `./config`, `./speech-models`, and `./gemini-transcription` are separate package exports, so that a vieval config can import them. +// vieval 0.4.0 loads configs with Node type stripping, which does not resolve a relative import without its extension, +// and this file re-exports such imports. The repository lint rule forbids the `.ts` extension in relative imports. +// Source: `Cannot find module .../src/method-config` when a test object config imported this module. +// Remove when vieval loads configs through a resolver that adds extensions, for example jiti or Vite. +export * from './cases/formula-speech' +export * from './evaluators/grader' +export * from './evaluators/listener' +export * from './evaluators/speech' +export * from './shared/cache-key' +export * from './shared/telemetry' +export * from './shared/velin' +export * from './tasks/read-aloud' diff --git a/evals/tts-speech-formula/eval/shared/cache-key.ts b/evals/tts-speech-formula/eval/shared/cache-key.ts new file mode 100644 index 0000000000..b913e7d40d --- /dev/null +++ b/evals/tts-speech-formula/eval/shared/cache-key.ts @@ -0,0 +1,12 @@ +import { createHash } from 'node:crypto' + +/** + * Builds one cache key segment from every input that changes a cached answer. + * + * @example + * cacheKey('zh', '$x$') + * // => 'b5f3…' (32 hex characters) + */ +export function cacheKey(...parts: string[]): string { + return createHash('sha256').update(JSON.stringify(parts)).digest('hex').slice(0, 32) +} diff --git a/evals/tts-speech-formula/eval/shared/telemetry.test.ts b/evals/tts-speech-formula/eval/shared/telemetry.test.ts new file mode 100644 index 0000000000..b4fc1d567d --- /dev/null +++ b/evals/tts-speech-formula/eval/shared/telemetry.test.ts @@ -0,0 +1,9 @@ +import { describe, expect, it } from 'vitest' + +import { usageRecord } from './telemetry' + +describe('usageRecord', () => { + it('uses the Chat Completions names that the vieval viewer reads', () => { + expect(usageRecord({ inputTokens: 120, outputTokens: 30, totalTokens: 150 })).toEqual({ completion_tokens: 30, prompt_tokens: 120, total_tokens: 150 }) + }) +}) diff --git a/evals/tts-speech-formula/eval/shared/telemetry.ts b/evals/tts-speech-formula/eval/shared/telemetry.ts new file mode 100644 index 0000000000..7a397581a9 --- /dev/null +++ b/evals/tts-speech-formula/eval/shared/telemetry.ts @@ -0,0 +1,44 @@ +import type { GenerateTextResult } from '@xsai/generate-text' +import type { TaskRunContext } from 'vieval/config' + +import { generateText } from '@xsai/generate-text' +import { emitChatModelErrorTelemetry, emitChatModelRequestTelemetry, emitChatModelResponseTelemetry } from 'vieval/plugins/chat-models' + +type GenerateTextOptions = Parameters[0] + +/** + * Converts xsai usage to the Chat Completions field names that the vieval viewer reads. + * xsai normalizes the provider usage to camel case, so the provider's own names are gone at this point. + * + * @example + * usageRecord({ inputTokens: 120, outputTokens: 30, totalTokens: 150 }) + * // => { prompt_tokens: 120, completion_tokens: 30, total_tokens: 150 } + */ +export function usageRecord(usage: GenerateTextResult['totalUsage']): Record { + return { completion_tokens: usage.outputTokens, prompt_tokens: usage.inputTokens, total_tokens: usage.totalTokens } +} + +/** + * Calls a chat model and reports the call to vieval: one `inference.requested` event, then `inference.responded` with the + * token usage and the latency, or `inference.failed`. The viewer sums the usage per case and per model. + * + * Only calls that reach the provider report usage. A cached answer makes no call, so a report shows what the run spent. + */ +export async function generateTextWithTelemetry(context: TaskRunContext, role: string, options: GenerateTextOptions): Promise { + const provider = { id: role, model: options.model } + emitChatModelRequestTelemetry(context, { data: { role }, provider }) + const startedAt = performance.now() + try { + const result = await generateText(options) + emitChatModelResponseTelemetry(context, { + latencyMs: Math.round(performance.now() - startedAt), + provider, + response: { usage: usageRecord(result.totalUsage) }, + }) + return result + } + catch (error) { + emitChatModelErrorTelemetry(context, { error, provider }) + throw error + } +} diff --git a/evals/tts-speech-formula/eval/shared/velin.ts b/evals/tts-speech-formula/eval/shared/velin.ts new file mode 100644 index 0000000000..9a8a6947ea --- /dev/null +++ b/evals/tts-speech-formula/eval/shared/velin.ts @@ -0,0 +1,27 @@ +import { readFile } from 'node:fs/promises' + +import { renderMarkdownString } from '@velin-dev/core-vue/render-node' + +/** + * Rendered prompts by file and props. A run asks for the same few prompts thousands of times, + * and each render compiles a Vue component, so every distinct prompt renders once per process. + */ +const rendered = new Map>() + +/** + * Renders a Velin markdown prompt. + * + * Velin escapes backslashes, underscores, and braces in plain text when it writes markdown back. + * Keep every LaTeX example of a prompt in a code span. + */ +export function renderVelinPrompt(file: URL, props: Readonly>): Promise { + const key = JSON.stringify([file.href, props]) + let prompt = rendered.get(key) + if (!prompt) { + prompt = readFile(file, 'utf8') + .then(source => renderMarkdownString(source, props)) + .then(result => result.rendered.trim()) + rendered.set(key, prompt) + } + return prompt +} diff --git a/evals/tts-speech-formula/eval/tasks/read-aloud/config.ts b/evals/tts-speech-formula/eval/tasks/read-aloud/config.ts new file mode 100644 index 0000000000..5b3afb656b --- /dev/null +++ b/evals/tts-speech-formula/eval/tasks/read-aloud/config.ts @@ -0,0 +1,214 @@ +import type { MatrixDefinition } from 'vieval/config' + +import { dirname, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' + +import { createGeminiTranscription } from '@proj-airi/evals-tts-speech-formula/gemini-transcription' +import { speechModelFrom, SpeechModels, speechProviderFrom, SpeechProviders, transcriptionModelFrom } from '@proj-airi/evals-tts-speech-formula/speech-models' +import { createAppleSpeechProvider } from '@xsai-apple-speech/transcription-native' +import { defineConfig, loadEnv } from 'vieval' +import { chatModelFrom, ChatModels, chatProviderFrom, ChatProviders } from 'vieval/plugins/chat-models' + +/** Each language is one task file. During development, `EVAL_LANGUAGES=zh,en` limits a run to some of them. */ +const languages = [ + 'zh', + 'en', + 'ja', + 'ko', +] + +/** + * The listener answers three votes per case, so 4 cases send up to 12 requests at once. + * `vieval run` runs each of the four test objects in its own process, and this pool is per process, + * so the listener and ASR proxies get up to 48 requests at once. With 8, they answered 429 `model_cooldown`. + */ +const caseConcurrency = 4 + +/** The Apple Speech locale of each task language. Every one must be installed in the macOS speech settings. */ +const appleSpeechLocales: Readonly> = { zh: 'zh-CN', en: 'en-US', ja: 'ja-JP', ko: 'ko-KR' } + +/** + * Apple Speech on macOS. It is a native recognizer, not an endpoint, so it is a transcription executor. + * Creating the provider loads nothing. The native addon loads on the first transcription. + */ +const appleSpeech = createAppleSpeechProvider() + +/** The package root. Every test object reads `.env` and `.env.test` from here, so one file configures all of them. */ +const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), '../../..') + +/** + * The model plugins. They register every model below, whether the matrix selects it or not, as `ChatModels` does. + * + * Endpoints and keys come from `.env` and are optional at load time. Thus, a provider that the matrix does not select + * needs no env. A selected model without its base URL fails when the task resolves it. + */ +const modelPlugins = [ + ChatProviders({ + providers: [chatProviderFrom({ + id: 'listener', + inferenceExecutor: 'openai', + optionalEnv: { baseURL: 'LISTENER_BASE_URL' }, + requiredEnv: { apiKey: 'LISTENER_API_KEY' }, + })], + }), + ChatModels({ + models: [chatModelFrom({ + model: 'gpt-6-luna', + provider: 'listener', + })], + }), + SpeechProviders({ + providers: [ + speechProviderFrom({ + id: 'official', + inferenceExecutor: 'openai', + optionalEnv: { apiKey: 'TTS_OFFICIAL_API_KEY', baseURL: 'TTS_OFFICIAL_BASE_URL' }, + }), + speechProviderFrom({ + id: 'azure', + inferenceExecutor: 'openai', + optionalEnv: { apiKey: 'TTS_AZURE_API_KEY', baseURL: 'TTS_AZURE_BASE_URL' }, + }), + speechProviderFrom({ + id: 'asr', + inferenceExecutor: 'openai', + optionalEnv: { apiKey: 'ASR_API_KEY', baseURL: 'ASR_BASE_URL' }, + }), + speechProviderFrom({ + id: 'apple-speech', + inferenceExecutor: { + id: 'apple-speech', + // One recognizer per locale. The `speech` transcriber is the model for long-form audio, not dictation. + transcription: ({ language }) => appleSpeech.transcription({ + locale: appleSpeechLocales[language ?? 'en'] ?? 'en-US', + transcriber: 'speech', + }), + }, + }), + speechProviderFrom({ + id: 'gemini', + // Gemini has no `/audio/transcriptions` endpoint, so the executor sends a `generateContent` request with the clip. + inferenceExecutor: createGeminiTranscription(), + optionalEnv: { apiKey: 'ASR_GEMINI_API_KEY', baseURL: 'ASR_GEMINI_BASE_URL' }, + }), + ], + }), + SpeechModels({ + models: [ + // The official AIRI provider and its default voice. + speechModelFrom({ + id: 'official', + model: 'stepfun/stepaudio-2.5-tts', + provider: 'official', + // The AIRI speech API passes the language to StepFun. Without it, this Chinese voice read English digits in Chinese. + sendsLanguage: true, + voice: 'jilingshaonv', + // NOTICE: + // `jilingshaonv` still read short English digits in Chinese in 3 of 9 clips with `language: en`. + // `lively-girl` read them in English in 18 of 18 clips. No voice read bare Japanese digits reliably, so Japanese keeps the default. + // Source: the voice A/B check of 2026-10-19 (official API, Gemini transcripts). + // Remove when `DEFAULT_TTS_VOICES` of the official API recommends an English voice for `en-US` and this suite reads it. + voices: { en: 'lively-girl' }, + }), + // Azure through unspeech. + speechModelFrom({ + id: 'azure', + model: 'microsoft/v1', + provider: 'azure', + voices: { zh: 'zh-CN-XiaoxiaoNeural', en: 'en-US-AriaNeural', ja: 'ja-JP-NanamiNeural', ko: 'ko-KR-SunHiNeural' }, + }), + transcriptionModelFrom({ + id: 'whisper-1', + model: 'whisper-1', + provider: 'asr', + }), + transcriptionModelFrom({ + id: 'apple-speech', + model: 'apple-speech', + provider: 'apple-speech', + }), + transcriptionModelFrom({ + id: 'gemini-3.8-flash', + model: 'gemini-3.8-flash', + provider: 'gemini', + }), + ], + }), +] + +/** Languages are tasks, not matrix values, so a development filter can drop some without changing what a task measures. */ +function selectedLanguages(env: NodeJS.ProcessEnv): string[] { + const selected = env.EVAL_LANGUAGES?.split(',').map(language => language.trim()).filter(Boolean) + return selected?.length ? languages.filter(language => selected.includes(language)) : languages +} + +/** A plugin that registers models: the chat or speech plugins of vieval. */ +type ModelPlugin + = | ReturnType + | ReturnType + | ReturnType + | ReturnType + +/** What a test object adds to the shared config, for example the model that it calls. */ +export interface FormulaSpeechConfigOptions { + /** Plugins that register the models of the test object. They run after the shared plugins. */ + plugins?: ModelPlugin[] + /** Run-matrix axes of the test object, for example the model that the `llm` object rewrites with. */ + runMatrix?: MatrixDefinition +} + +/** + * Builds the workspace config of one test object. + * + * A test object is one way to turn a reply into spoken text. Its project runs the read-aloud tasks under its own `tasks/` + * with the shared matrix, plus the axes of its own options. + */ +export function defineFormulaSpeechConfig(testObject: string, options: FormulaSpeechConfigOptions = {}) { + const env = loadEnv('test', packageRoot, '') + const selectedTts = env.EVAL_TTS?.split(',').map(id => id.trim()).filter(Boolean) + const selectedAsr = env.EVAL_ASR?.split(',').map(id => id.trim()).filter(Boolean) + + const matrix = { + /** `none` grades the spoken text without TTS: the floor that AIRI controls. Every other value is a TTS model. `EVAL_TTS=none` skips TTS. */ + tts: selectedTts?.length ? selectedTts : ['none', 'official'], + /** + * Transcribes the clip of every TTS model. Unused when `tts` is only `none`. Add `whisper-1` or `gemini-3.8-flash` to compare. + * `EVAL_ASR=gemini-3.8-flash` selects other ASR models for one run. + */ + asr: selectedAsr?.length ? selectedAsr : ['apple-speech'], + /** Writes down what it heard as LaTeX. */ + listener: ['gpt-6-luna'], + /** Judges whether the spoken text says the meaning or spells markup. It uses the endpoint of the listener. */ + judge: ['gpt-6-luna'], + } + + return defineConfig({ + env, + plugins: [ + ...modelPlugins, + ...(options.plugins ?? []), + ], + scheduler: { concurrency: { case: caseConcurrency } }, + projects: [ + { + name: `formula-speech-${testObject}`, + root: '.', + include: selectedLanguages(env).map(language => `tasks/read-aloud/formula-speech/${language}.eval.ts`), + runMatrix: { + override: { + tts: matrix.tts, + ...options.runMatrix, + }, + }, + evalMatrix: { + override: { + // The listener and the ASR model are part of the measurement, not of the test object. + listener: matrix.listener, + judge: matrix.judge, + ...(matrix.tts.some(id => id !== 'none') ? { asr: matrix.asr } : {}), + }, + }, + }, + ], + }) +} diff --git a/evals/tts-speech-formula/eval/tasks/read-aloud/index.ts b/evals/tts-speech-formula/eval/tasks/read-aloud/index.ts new file mode 100644 index 0000000000..fec45a6037 --- /dev/null +++ b/evals/tts-speech-formula/eval/tasks/read-aloud/index.ts @@ -0,0 +1,300 @@ +import type { TaskRunContext } from 'vieval/config' + +import type { Language } from '../../cases/formula-speech' +import type { Grade } from '../../evaluators/grader' +import type { Reading } from '../../evaluators/judge' +import type { HearingStatus } from '../../evaluators/listener' + +import { env } from 'node:process' + +import { errorMessageFrom } from '@moeru/std' +import { defineMetric, defineScore, describeTask } from 'vieval' +import { createRetryPolicy, isRetriableProviderError, readRetryAfter, runWithRetry } from 'vieval/core/inference-executors' +import { modelFromEval, openaiFromRunContext } from 'vieval/plugins/chat-models' + +import { formulaSpeechCases } from '../../cases/formula-speech' +import { mathBridgeCases } from '../../cases/math-bridge' +import { grade, majorityScore } from '../../evaluators/grader' +import { judgeReading, judgeVotes, medianReading, readingScores } from '../../evaluators/judge' +import { answerFormOf, listenerVotes, writeDown } from '../../evaluators/listener' +import { leftoverNotation } from '../../evaluators/notation' +import { openaiSpeechFromRunContext, speechModelFromRun, synthesize, transcribe, transcriptionModelFromEval, transcriptionRequestFromRunContext } from '../../evaluators/speech' + +/** + * `EVAL_DATASET` selects the cases: `formula-speech`, the stratified set, or `math-bridge`, a sample of arXiv formulas. + * `EVAL_CASES` limits the run to some case ids, for example while a new rule is in progress. + */ +const dataset = env.EVAL_DATASET === 'math-bridge' ? mathBridgeCases : formulaSpeechCases +const selectedIds = env.EVAL_CASES?.split(',').map(id => id.trim()).filter(Boolean) +const cases = selectedIds?.length ? dataset.filter(item => selectedIds.includes(item.id)) : dataset + +/** The listener wrote the formula back, so its reading is correct. A strict majority of the listener votes decides it. */ +const correct = defineScore('formula.correct', { + description: { 'en': 'A listener model wrote the formula back from what it heard, and the grader matched it with the reference. A strict majority of the votes decides it.', 'zh-Hans': '听写模型根据听到的内容写回公式，判分器与参考答案比对一致。多数票决定。' }, + grader: { id: 'grader', kind: 'rule', reducer: 'majority', votes: listenerVotes }, + label: { 'en': 'Written back correctly', 'zh-Hans': '听写正确' }, +}) +/** The judge saw a reading that says the meaning: 1 for natural, 0.5 for mixed, and 0 for spelled markup. */ +const natural = defineScore('formula.natural', { + description: { 'en': 'A judge model read the spoken text: 1 when it says the meaning, 0.5 when one or two parts spell markup, and 0 when it spells markup. It is the median of the votes.', 'zh-Hans': '评委模型阅读朗读文本：读出含义为 1，有一两处逐个念记号为 0.5，逐个念记号为 0。取投票中位数。' }, + grader: { id: 'judge', kind: 'model', reducer: 'median', votes: judgeVotes }, + label: { 'en': 'Natural reading', 'zh-Hans': '读法自然' }, +}) +/** + * The headline score: the reading is correct and a listener wants to hear it. A mixed reading passes, because one word + * for a bracket that carries meaning is not a fault. It replaces the mean of the two scores in vieval 0.9. + */ +const pass = defineScore('formula.pass', { + combine: { all: [correct, { atLeast: readingScores.mixed, score: natural }] }, + description: { 'en': 'The reading is correct, and it is natural or mixed.', 'zh-Hans': '读法正确，并且自然或基本自然。' }, + label: { 'en': 'Correct and natural', 'zh-Hans': '正确且自然' }, + primary: true, +}) + +const strata = defineMetric('tts_speech_formula.stratum', { + description: { 'en': 'The group of notation that the case covers.', 'zh-Hans': '用例覆盖的记法类别。' }, + label: { 'en': 'Stratum', 'zh-Hans': '类别' }, +}) +const gradingRule = defineMetric('tts_speech_formula.rule', { + description: { 'en': 'The grader check that decided the score.', 'zh-Hans': '决定分数的判分规则。' }, + label: { 'en': 'Grading rule', 'zh-Hans': '判分规则' }, +}) +const leftoverSymbols = defineMetric('tts_speech_formula.leftover_notation', { + better: 'lower', + description: { 'en': 'Symbols and LaTeX command names that the test object left in the spoken text, so TTS gets notation instead of words. It needs no TTS.', 'zh-Hans': '测试对象留在朗读文本里的符号和 LaTeX 命令名。TTS 拿到的是符号而不是词。不需要 TTS。' }, + label: { 'en': 'Leftover notation', 'zh-Hans': '残留记号' }, + unit: 'symbols', +}) +const ttsCharacters = defineMetric('tts_speech_formula.tts.characters_sent', { + better: 'lower', + description: { 'en': 'Characters that this run sent to the TTS provider. A cached clip sends none. Most TTS providers bill by characters.', 'zh-Hans': '本次运行发给 TTS 服务的字符数。命中缓存时为 0。多数 TTS 服务按字符计费。' }, + label: { 'en': 'TTS characters sent', 'zh-Hans': 'TTS 发送字符数' }, + unit: 'characters', +}) +const asrRequests = defineMetric('tts_speech_formula.asr.requests', { + better: 'lower', + description: { 'en': 'ASR requests that this run sent. A cached transcript sends none.', 'zh-Hans': '本次运行发出的 ASR 请求数。命中缓存时为 0。' }, + label: { 'en': 'ASR requests', 'zh-Hans': 'ASR 请求数' }, + unit: 'requests', +}) +const hearing = defineMetric('tts_speech_formula.hearing', { + description: { 'en': 'How well the listener could hear the case: ok, partial, or unclear. It is the status of most votes, and the worse one on a tie.', 'zh-Hans': '听写者能否听清：ok、partial 或 unclear。取多数票的状态，平票时取更差的。' }, + label: { 'en': 'Hearing', 'zh-Hans': '可听清程度' }, +}) +const readingMetric = defineMetric('tts_speech_formula.reading', { + description: { 'en': 'How the judge saw the reading: natural says the meaning, spelled reads the markup, and mixed is between. It is the median of the judge votes.', 'zh-Hans': '评委对朗读方式的判断：natural 读出含义，spelled 逐个念记号，mixed 介于两者之间。取评委投票的中位数。' }, + label: { 'en': 'Reading', 'zh-Hans': '朗读方式' }, +}) +const votes = defineMetric('tts_speech_formula.votes', { + description: { 'en': 'The grade of each listener answer. The case passes with a strict majority.', 'zh-Hans': '每次听写的得分。多数票通过才算通过。' }, + label: { 'en': 'Listener votes', 'zh-Hans': '听写投票' }, +}) + +/** Worst first, so a tie between two statuses selects the worse one. */ +const hearingOrder: readonly HearingStatus[] = ['unclear', 'partial', 'ok'] + +/** + * The hearing status of a case: the status of most votes. On a tie, the worse status wins, so one clear vote + * does not hide that the other votes could not hear the clip. + * + * @example + * majorityHearing(['ok', 'unclear', 'unclear']) + * // => 'unclear' + * majorityHearing(['ok', 'partial', 'unclear']) + * // => 'unclear' + */ +function majorityHearing(statuses: readonly HearingStatus[]): HearingStatus { + let selected = hearingOrder[0]! + let selectedCount = -1 + for (const status of hearingOrder) { + const count = statuses.filter(item => item === status).length + if (count > selectedCount) { + selected = status + selectedCount = count + } + } + return selected +} + +/** + * What a read-aloud adapter gets besides the reply: the case context of vieval. + * An adapter that calls a model resolves it from `models` and `task.matrix`, caches its answers in `cache`, + * and reports its usage to `events`, for example with `generateTextWithTelemetry`. + */ +export type ReadAloudContext = TaskRunContext & { signal: AbortSignal } + +/** + * The adapter of a test object: it turns one reply into the text that a TTS provider reads. + * Each test object under `eval/test-objects/` implements one. + */ +export type ReadAloud = (markdown: string, language: Language, context: ReadAloudContext) => Promise + +// WORKAROUND: +// Some calls never answer, and one case then holds the whole run for many minutes. +// The official StepFun TTS sometimes holds a request open for more than 5 minutes, and a model can write without end. +// Source: T-152 §5.1 (StepFun hangs over 5 minutes), and the Korean `rel-parallel` case on 2026-10-19. +// Remove when every provider answers or fails within this time. The timeout fails the try, and `step` sends the call again. +const stepTimeoutMs = 120_000 + +/** Total tries of one step, including the first. The waits between them add up to about 70 seconds. */ +const stepMaxAttempts = 4 + +// WORKAROUND: +// The listener and rewrite proxies answer 429 `model_cooldown` for some seconds when four workspaces call them at once, +// and the ASR proxy sometimes drops the TLS connection. The case retry of vieval waited only 0.5, 1, and 2 seconds, +// so all retries of a case failed inside one cooldown. +// Source: the comparison run attempt-2026-10-19T06-36-24-868Z, where 93 `llm` cases and 33 `rule` cases failed this way. +// Remove when vieval limits and cools down calls per provider across workspaces. +/** + * Wait before retry `attempt` of a step: the provider's `Retry-After` when it sent a longer one, else 10, 20, then + * 40 seconds, plus up to a quarter more. + * The default backoff of vieval starts at 0.5 seconds, which is far shorter than one cooldown of the proxies. + * The random part spreads the cases that one cooldown stopped together, so they do not call again at the same moment. + */ +function stepRetryDelayMs(attempt: number, error: unknown): number { + const backoffMs = 10_000 * 2 ** (attempt - 1) + return Math.max(readRetryAfter(error) ?? 0, backoffMs + Math.random() * backoffMs / 4) +} + +/** + * Retries the calls that a rate limit, a server error, a dropped connection, or a timeout stopped + * (`isRetriableProviderError` of vieval). `runWithRetry` stops waiting and retrying once the run is interrupted. + */ +const stepRetryPolicy = createRetryPolicy({ delayMs: stepRetryDelayMs, maxAttempts: stepMaxAttempts }) + +/** The provider error that `step` wrapped, or the error itself when no step wrapped it. */ +function causeOf(error: unknown): unknown { + return error instanceof Error && error.cause != null ? error.cause : error +} + +/** + * Runs one external call with a timeout and names it in the error. + * A retriable failure tries the call again after a wait (`stepRetryPolicy`). Other errors and an aborted run fail + * the step at once. + * Node reports a network failure as `fetch failed` and keeps the reason, for example `ECONNRESET`, in `cause`. + * The report then shows the step, the message, and the reason together. The thrown error keeps the provider error + * in `cause`, so the case retry below can still classify it. + */ +async function step(name: string, signal: AbortSignal, run: (signal: AbortSignal) => Promise): Promise { + try { + return await runWithRetry(() => run(AbortSignal.any([signal, AbortSignal.timeout(stepTimeoutMs)])), stepRetryPolicy, { signal }) + } + catch (error) { + const cause = error instanceof Error && error.cause != null ? ` (${errorMessageFrom(error.cause) ?? 'no reason'})` : '' + throw new Error(`${name}: ${errorMessageFrom(error) ?? 'unknown error'}${cause}`, { cause: error }) + } +} + +/** + * Registers the read-aloud task of one language for one test object. + * + * WORKAROUND: + * Each language has its own eval file, because two tasks of one file overwrite each other's cases. + * In vieval 0.4.0, the task id comes from the eval file, not from the `describeTask` name, so both tasks get one id. + * Source: the first comparison run, where the `zh` cases of a file disappeared behind the `en` cases. + * Remove when vieval derives the task id from the task name. Then one eval file can register all four languages. + * + * Call stack: + * + * eval/test-objects//tasks/read-aloud/formula-speech/.eval.ts + * -> {@link describeFormulaSpeechTask} + * -> {@link ReadAloud} of the test object, then synthesize and transcribe (../../evaluators/speech) unless `tts` is `none` + * -> writeDown (./listener), one call per vote + * -> grade (./score) + */ +export function describeFormulaSpeechTask(language: Language, readAloud: ReadAloud) { + describeTask(language, ({ casesFromInputs }) => { + casesFromInputs('case', cases, async (context) => { + const item = context.matrix.inputs + // `none` sends the spoken text straight to the listener. It measures what AIRI hands to TTS, independent of any provider. + const tts = String(context.task.matrix.run.tts) + const { cache, signal } = context + + // Reports of different test objects match one sample by this id, so it leaves the test object out. + context.recordMetric('benchmark.case.id', `${item.id}|${language}|${tts}`) + context.recordMetric(strata, item.stratum) + + const markdown = typeof item.input === 'string' ? item.input : item.input[language] + const spoken = await step('read aloud', signal, stepSignal => readAloud(markdown, language, { ...context, signal: stepSignal })) + const leftover = leftoverNotation(spoken, markdown) + context.recordMetric(leftoverSymbols, leftover.length) + await context.recordArtifact('spoken.txt', { body: `${spoken}\n`, contentType: 'text/plain', label: { 'en': 'Spoken text', 'zh-Hans': '朗读文本' }, purpose: 'tts_speech_formula.spoken_text' }) + + let heard = spoken + if (tts !== 'none' && spoken.trim()) { + const speech = openaiSpeechFromRunContext(speechModelFromRun(context, { axis: 'tts' })) + const clip = await step(`TTS ${speech.model}`, signal, stepSignal => synthesize(speech, spoken, language, cache, stepSignal)) + await context.recordArtifact('speech.mp3', { contentType: 'audio/mpeg', label: { 'en': 'Speech', 'zh-Hans': '语音' }, path: clip.path, purpose: 'tts_speech_formula.speech' }) + const asrModel = transcriptionModelFromEval(context, { axis: 'asr' }) + const transcription = transcriptionRequestFromRunContext(asrModel, { language }) + const transcript = await step(`ASR ${asrModel.id}`, signal, stepSignal => transcribe(transcription, asrModel.id, clip, language, cache, stepSignal)) + heard = transcript.text + // Chat calls report token usage through vieval inference events. TTS and ASR calls are not chat calls, + // and the vieval event schema accepts only chat, so their billing basis goes into metrics. + context.recordMetric(ttsCharacters, clip.cached ? 0 : spoken.length) + context.recordMetric(asrRequests, transcript.cached ? 0 : 1) + } + + const listener = openaiFromRunContext(modelFromEval(context, { axis: 'listener' })) + // An OpenAI-compatible provider without a base URL is OpenAI itself. + const endpoint = { apiKey: listener.apiKey, baseURL: listener.baseURL || 'https://api.openai.com/v1/', model: listener.model } + let answers: Awaited>[] = [] + let grades: Grade[] = [] + let score: 0 | 1 | undefined + // An empty transcript of a spoken clip says nothing about the reading: the Gemini ASR returned nothing for some + // long Chinese, Japanese, and Korean clips in every run. The case keeps its judge score, and its correctness is unscored. + if (tts !== 'none' && spoken.trim() && !heard.trim()) { + context.recordUnscored(correct, 'asr_empty', { rationale: 'The ASR returned an empty transcript.' }) + } + else { + const form = answerFormOf(item.mode) + answers = await step(`listener ${endpoint.model}`, signal, stepSignal => Promise.all(Array.from({ length: listenerVotes }, (_, vote) => writeDown(heard, language, form, vote, endpoint, context, stepSignal)))) + // An unintelligible vote scores 0 under its own rule, so a report tells an unclear clip apart from a wrong reading. + grades = answers.map((answer): Grade => answer.status === 'unclear' ? { score: 0, rule: 'unclear' } : grade(item, answer.latex)) + score = majorityScore(grades.map(({ score: vote }) => vote)) + context.recordMetric(hearing, majorityHearing(answers.map(({ status }) => status))) + + // A score of 0 is a result, not a failure. The case fails only when a provider, a model, or the listener cannot be reached. + const rule = grades.find(({ score: vote }) => vote === score)?.rule ?? grades[0]!.rule + context.recordScore(correct, score, { metadata: { listener: endpoint.model }, rationale: `${grades.map(({ score: vote }) => vote).join(', ')} by ${rule}` }) + context.recordMetric(votes, grades.map(({ score: vote }) => vote)) + context.recordMetric(gradingRule, rule) + } + + // The listener checks that the reading is correct. The judge checks that it says the meaning instead of spelling + // markup, because a listener model can write down "open parenthesis x close parenthesis" as well as "of x". + // The judge reads the spoken text, not the transcript, so it judges only the test object. + const judge = openaiFromRunContext(modelFromEval(context, { axis: 'judge' })) + const judgeEndpoint = { apiKey: judge.apiKey, baseURL: judge.baseURL || 'https://api.openai.com/v1/', model: judge.model } + const verdicts = await step(`judge ${judgeEndpoint.model}`, signal, stepSignal => Promise.all(Array.from({ length: judgeVotes }, (_, vote) => judgeReading(spoken, markdown, language, vote, judgeEndpoint, context, stepSignal)))) + const reading: Reading = medianReading(verdicts) + context.recordScore(natural, readingScores[reading], { + grader: { id: 'judge', kind: 'model', model: judgeEndpoint.model, reducer: 'median', votes: judgeVotes }, + rationale: verdicts.find(verdict => verdict.reading === reading)?.reason, + }) + context.recordMetric(readingMetric, reading) + await context.recordArtifact('result.json', { + body: `${JSON.stringify({ id: item.id, language, tts, markdown, spoken, leftover, heard, reference: item.reference, answers, grades, score, verdicts, reading }, null, 2)}\n`, + contentType: 'application/json', + label: { 'en': 'Result', 'zh-Hans': '结果' }, + purpose: 'tts_speech_formula.result', + }) + }) + }, { + // `pass` is derived from the two recorded scores, so the task announces it before the first case. + scores: [correct, natural, pass], + // WORKAROUND: + // Providers sometimes drop a connection, and the case fails with `fetch failed` for a reason outside the test object. + // The ASR proxy and the listener proxy dropped some requests during the first full runs. + // Source: the `ja` and `ko` cases of the official TTS in the smoke run on 2026-10-19, which passed when run again. + // Remove when the providers are stable. A retry replays the case, but every finished step returns from the cache, + // so only the failed call runs again. A score of 0 is not a failure, so retries never hide a wrong reading. + // `step` already retries the calls that a rate limit or a dropped connection stops, so this retry is the last + // resort when a provider stays unavailable for all tries of a step. It waits 60 seconds, longer than one cooldown + // of the proxies, or the provider's `Retry-After` when that is longer. + // `step` wraps the provider error in `cause`. A bug or a grader error is not a provider error, so it fails at once. + autoRetry: 1, + autoRetryDelay: (_retryIndex, error) => Math.max(readRetryAfter(causeOf(error)) ?? 0, 60_000), + autoRetryIf: error => isRetriableProviderError(causeOf(error)), + }) +} diff --git a/evals/tts-speech-formula/eval/test-objects/llm/src/read-aloud.test.ts b/evals/tts-speech-formula/eval/test-objects/llm/src/read-aloud.test.ts new file mode 100644 index 0000000000..2a9c4d6e53 --- /dev/null +++ b/evals/tts-speech-formula/eval/test-objects/llm/src/read-aloud.test.ts @@ -0,0 +1,9 @@ +import { describe, expect, it } from 'vitest' + +import { rewritePrompt } from './read-aloud' + +describe('rewritePrompt', () => { + it('names the language of the voice', async () => { + expect(await rewritePrompt('ko')).toContain('natural Korean words') + }) +}) diff --git a/evals/tts-speech-formula/eval/test-objects/llm/src/read-aloud.ts b/evals/tts-speech-formula/eval/test-objects/llm/src/read-aloud.ts new file mode 100644 index 0000000000..b9a70616d3 --- /dev/null +++ b/evals/tts-speech-formula/eval/test-objects/llm/src/read-aloud.ts @@ -0,0 +1,40 @@ +import type { Language, ReadAloud } from '@proj-airi/evals-tts-speech-formula' + +import { cacheKey, generateTextWithTelemetry, languageName, renderVelinPrompt } from '@proj-airi/evals-tts-speech-formula' +import { modelFromRun, openaiFromRunContext } from 'vieval/plugins/chat-models' + +export function rewritePrompt(language: Language): Promise { + return renderVelinPrompt(new URL('./rewrite.velin.md', import.meta.url), { language: languageName[language] }) +} + +/** + * A model rewrites the reply for speech. It is the upper reference, but too slow for the live path. + * The answer is cached by model, prompt, and input, so a rerun makes no new call. + */ +export const readAloud: ReadAloud = async (markdown, language, context) => { + const { cache, signal } = context + // The `rewrite` run axis selects the model. `vieval.config.ts` of this test object registers it. + const rewrite = openaiFromRunContext(modelFromRun(context, { axis: 'rewrite' })) + // An OpenAI-compatible provider without a base URL is OpenAI itself. + const endpoint = { apiKey: rewrite.apiKey, baseURL: rewrite.baseURL || 'https://api.openai.com/v1/', model: rewrite.model } + const system = await rewritePrompt(language) + const file = cache.namespace('rewrite').file({ ext: 'json', key: [endpoint.model, cacheKey(system, markdown)] }) + if (await file.exists()) + return (await file.readJson<{ text: string }>()).text + + const { text } = await generateTextWithTelemetry(context, 'rewrite', { + ...endpoint, + abortSignal: signal, + // WORKAROUND: + // A rewrite model can write without end, and the case never finishes. + // xsai sends no output limit unless the caller sets one, and a spoken reply needs far fewer tokens. + // Source: the same failure of the listener model, see eval/evaluators/listener. + // Remove when the rewrite model always stops on its own. + maxCompletionTokens: 2000, + messages: [{ role: 'system', content: system }, { role: 'user', content: markdown }], + temperature: 0, + }) + const spoken = text?.trim() ?? '' + await file.writeJson({ text: spoken }) + return spoken +} diff --git a/evals/tts-speech-formula/eval/test-objects/llm/src/rewrite.velin.md b/evals/tts-speech-formula/eval/test-objects/llm/src/rewrite.velin.md new file mode 100644 index 0000000000..05d50b0e11 --- /dev/null +++ b/evals/tts-speech-formula/eval/test-objects/llm/src/rewrite.velin.md @@ -0,0 +1,24 @@ + + +## Role + +You prepare text for a text-to-speech voice that speaks {{ props.language }}. + +## Task + +Rewrite the message so that a listener who only hears it can write down every formula exactly. + +- Read math, chemistry, units, and money as natural {{ props.language }} words. +- Read the structure too: which part is the numerator, where a group starts and ends, and the upper and lower limits. +- Keep all other words. + +## Output + +- Do not explain or add content. +- Do not use LaTeX, markdown, or math symbols. +- Output only the text to speak. diff --git a/evals/tts-speech-formula/eval/test-objects/llm/tasks/read-aloud/formula-speech/en.eval.ts b/evals/tts-speech-formula/eval/test-objects/llm/tasks/read-aloud/formula-speech/en.eval.ts new file mode 100644 index 0000000000..fb474090bb --- /dev/null +++ b/evals/tts-speech-formula/eval/test-objects/llm/tasks/read-aloud/formula-speech/en.eval.ts @@ -0,0 +1,5 @@ +import { describeFormulaSpeechTask } from '@proj-airi/evals-tts-speech-formula' + +import { readAloud } from '../../../src/read-aloud' + +describeFormulaSpeechTask('en', readAloud) diff --git a/evals/tts-speech-formula/eval/test-objects/llm/tasks/read-aloud/formula-speech/ja.eval.ts b/evals/tts-speech-formula/eval/test-objects/llm/tasks/read-aloud/formula-speech/ja.eval.ts new file mode 100644 index 0000000000..8a686cf651 --- /dev/null +++ b/evals/tts-speech-formula/eval/test-objects/llm/tasks/read-aloud/formula-speech/ja.eval.ts @@ -0,0 +1,5 @@ +import { describeFormulaSpeechTask } from '@proj-airi/evals-tts-speech-formula' + +import { readAloud } from '../../../src/read-aloud' + +describeFormulaSpeechTask('ja', readAloud) diff --git a/evals/tts-speech-formula/eval/test-objects/llm/tasks/read-aloud/formula-speech/ko.eval.ts b/evals/tts-speech-formula/eval/test-objects/llm/tasks/read-aloud/formula-speech/ko.eval.ts new file mode 100644 index 0000000000..711a60ce25 --- /dev/null +++ b/evals/tts-speech-formula/eval/test-objects/llm/tasks/read-aloud/formula-speech/ko.eval.ts @@ -0,0 +1,5 @@ +import { describeFormulaSpeechTask } from '@proj-airi/evals-tts-speech-formula' + +import { readAloud } from '../../../src/read-aloud' + +describeFormulaSpeechTask('ko', readAloud) diff --git a/evals/tts-speech-formula/eval/test-objects/llm/tasks/read-aloud/formula-speech/zh.eval.ts b/evals/tts-speech-formula/eval/test-objects/llm/tasks/read-aloud/formula-speech/zh.eval.ts new file mode 100644 index 0000000000..54e6e9b874 --- /dev/null +++ b/evals/tts-speech-formula/eval/test-objects/llm/tasks/read-aloud/formula-speech/zh.eval.ts @@ -0,0 +1,5 @@ +import { describeFormulaSpeechTask } from '@proj-airi/evals-tts-speech-formula' + +import { readAloud } from '../../../src/read-aloud' + +describeFormulaSpeechTask('zh', readAloud) diff --git a/evals/tts-speech-formula/eval/test-objects/llm/vieval.config.ts b/evals/tts-speech-formula/eval/test-objects/llm/vieval.config.ts new file mode 100644 index 0000000000..ba34975cda --- /dev/null +++ b/evals/tts-speech-formula/eval/test-objects/llm/vieval.config.ts @@ -0,0 +1,30 @@ +import { defineFormulaSpeechConfig } from '@proj-airi/evals-tts-speech-formula/config' +import { chatModelFrom, ChatModels, chatProviderFrom, ChatProviders } from 'vieval/plugins/chat-models' + +/** The rewrite model. It is a run axis of this test object, so a report names the model that wrote the spoken text. */ +const rewriteModels = ['gpt-6-luna'] + +export default defineFormulaSpeechConfig('llm', { + plugins: [ + ChatProviders({ + providers: [ + chatProviderFrom({ + id: 'rewrite', + inferenceExecutor: 'openai', + optionalEnv: { baseURL: 'REWRITE_BASE_URL' }, + requiredEnv: { apiKey: 'REWRITE_API_KEY' }, + }), + ], + }), + ChatModels({ + models: rewriteModels.map((model) => { + return chatModelFrom({ id: `rewrite:${model}`, model, provider: 'rewrite' }) + }), + }), + ], + runMatrix: { + rewrite: rewriteModels.map((model) => { + return `rewrite:${model}` + }), + }, +}) diff --git a/evals/tts-speech-formula/eval/test-objects/raw/src/read-aloud.ts b/evals/tts-speech-formula/eval/test-objects/raw/src/read-aloud.ts new file mode 100644 index 0000000000..785b35a4af --- /dev/null +++ b/evals/tts-speech-formula/eval/test-objects/raw/src/read-aloud.ts @@ -0,0 +1,4 @@ +import type { ReadAloud } from '@proj-airi/evals-tts-speech-formula' + +/** The markdown as the model wrote it. This is AIRI without formula speech, the lower reference. */ +export const readAloud: ReadAloud = async markdown => markdown diff --git a/evals/tts-speech-formula/eval/test-objects/raw/tasks/read-aloud/formula-speech/en.eval.ts b/evals/tts-speech-formula/eval/test-objects/raw/tasks/read-aloud/formula-speech/en.eval.ts new file mode 100644 index 0000000000..fb474090bb --- /dev/null +++ b/evals/tts-speech-formula/eval/test-objects/raw/tasks/read-aloud/formula-speech/en.eval.ts @@ -0,0 +1,5 @@ +import { describeFormulaSpeechTask } from '@proj-airi/evals-tts-speech-formula' + +import { readAloud } from '../../../src/read-aloud' + +describeFormulaSpeechTask('en', readAloud) diff --git a/evals/tts-speech-formula/eval/test-objects/raw/tasks/read-aloud/formula-speech/ja.eval.ts b/evals/tts-speech-formula/eval/test-objects/raw/tasks/read-aloud/formula-speech/ja.eval.ts new file mode 100644 index 0000000000..8a686cf651 --- /dev/null +++ b/evals/tts-speech-formula/eval/test-objects/raw/tasks/read-aloud/formula-speech/ja.eval.ts @@ -0,0 +1,5 @@ +import { describeFormulaSpeechTask } from '@proj-airi/evals-tts-speech-formula' + +import { readAloud } from '../../../src/read-aloud' + +describeFormulaSpeechTask('ja', readAloud) diff --git a/evals/tts-speech-formula/eval/test-objects/raw/tasks/read-aloud/formula-speech/ko.eval.ts b/evals/tts-speech-formula/eval/test-objects/raw/tasks/read-aloud/formula-speech/ko.eval.ts new file mode 100644 index 0000000000..711a60ce25 --- /dev/null +++ b/evals/tts-speech-formula/eval/test-objects/raw/tasks/read-aloud/formula-speech/ko.eval.ts @@ -0,0 +1,5 @@ +import { describeFormulaSpeechTask } from '@proj-airi/evals-tts-speech-formula' + +import { readAloud } from '../../../src/read-aloud' + +describeFormulaSpeechTask('ko', readAloud) diff --git a/evals/tts-speech-formula/eval/test-objects/raw/tasks/read-aloud/formula-speech/zh.eval.ts b/evals/tts-speech-formula/eval/test-objects/raw/tasks/read-aloud/formula-speech/zh.eval.ts new file mode 100644 index 0000000000..54e6e9b874 --- /dev/null +++ b/evals/tts-speech-formula/eval/test-objects/raw/tasks/read-aloud/formula-speech/zh.eval.ts @@ -0,0 +1,5 @@ +import { describeFormulaSpeechTask } from '@proj-airi/evals-tts-speech-formula' + +import { readAloud } from '../../../src/read-aloud' + +describeFormulaSpeechTask('zh', readAloud) diff --git a/evals/tts-speech-formula/eval/test-objects/raw/vieval.config.ts b/evals/tts-speech-formula/eval/test-objects/raw/vieval.config.ts new file mode 100644 index 0000000000..a17363b1ca --- /dev/null +++ b/evals/tts-speech-formula/eval/test-objects/raw/vieval.config.ts @@ -0,0 +1,3 @@ +import { defineFormulaSpeechConfig } from '@proj-airi/evals-tts-speech-formula/config' + +export default defineFormulaSpeechConfig('raw') diff --git a/evals/tts-speech-formula/eval/test-objects/rule/src/read-aloud.test.ts b/evals/tts-speech-formula/eval/test-objects/rule/src/read-aloud.test.ts new file mode 100644 index 0000000000..e4850b0f26 --- /dev/null +++ b/evals/tts-speech-formula/eval/test-objects/rule/src/read-aloud.test.ts @@ -0,0 +1,23 @@ +import { toSpeechText } from '@proj-airi/core-agent' +import { describe, expect, it } from 'vitest' + +import { formulaSpeechCases, languages, voiceLanguage } from '../../../cases/formula-speech' +import { leftoverNotation } from '../../../evaluators/notation' + +describe('rule read-aloud', () => { + // Runs the rule engine on every case, without TTS or ASR, so it costs nothing. + // The snapshot lists the notation that the rule engine still hands to TTS. Fix the rule engine to shrink it. + it('lists the notation that the rule engine leaves in the spoken text', () => { + const leftovers: Record = {} + for (const item of formulaSpeechCases) { + for (const language of languages) { + const markdown = typeof item.input === 'string' ? item.input : item.input[language] + const found = leftoverNotation(toSpeechText(markdown, { language: voiceLanguage[language] }), markdown) + if (found.length > 0) + leftovers[`${item.id}|${language}`] = found.join(' ') + } + } + + expect(leftovers).toMatchInlineSnapshot(`{}`) + }) +}) diff --git a/evals/tts-speech-formula/eval/test-objects/rule/src/read-aloud.ts b/evals/tts-speech-formula/eval/test-objects/rule/src/read-aloud.ts new file mode 100644 index 0000000000..f40cb93abc --- /dev/null +++ b/evals/tts-speech-formula/eval/test-objects/rule/src/read-aloud.ts @@ -0,0 +1,7 @@ +import type { ReadAloud } from '@proj-airi/evals-tts-speech-formula' + +import { toSpeechText } from '@proj-airi/core-agent' +import { voiceLanguage } from '@proj-airi/evals-tts-speech-formula' + +/** `toSpeechText` of `@proj-airi/core-agent`, the rule engine that AIRI ships. The voice language selects the formula language. */ +export const readAloud: ReadAloud = async (markdown, language) => toSpeechText(markdown, { language: voiceLanguage[language] }) diff --git a/evals/tts-speech-formula/eval/test-objects/rule/src/structure.test.ts b/evals/tts-speech-formula/eval/test-objects/rule/src/structure.test.ts new file mode 100644 index 0000000000..fe10f3ae08 --- /dev/null +++ b/evals/tts-speech-formula/eval/test-objects/rule/src/structure.test.ts @@ -0,0 +1,123 @@ +import type { MathJson } from '@proj-airi/mathjson' + +import { ComputeEngine } from '@cortex-js/compute-engine' +import { MathSpeech } from '@proj-airi/speech-transformer-math' +import { describe, expect, it } from 'vitest' + +import { formulaSpeechCases } from '../../../cases/formula-speech' + +// The built-in packs. The chemistry cases are skipped below, so the reader has no extension. +const reader = new MathSpeech() + +/** + * The standard MathJSON heads that both trees use with one meaning. Other heads differ by design, see + * docs/ai/adr/2026-10-20-speech-text-preprocessor.md: Compute Engine reads `AB` as a product and folds `50\%`, and the + * speech tree keeps names, percents, units, and groups. + */ +const sharedHeads = new Set([ + 'Equal', + 'NotEqual', + 'Less', + 'Greater', + 'LessEqual', + 'GreaterEqual', + 'Element', + 'NotElement', + 'SubsetEqual', + 'Subset', + 'Perpendicular', + 'Parallel', + 'Similar', + 'Congruent', + 'To', + 'Divide', + 'Power', + 'Sqrt', + 'Root', + 'Sum', + 'Product', + 'Integrate', + 'Limit', + 'D', + 'Interval', + 'Set', + 'Abs', + 'Union', + 'Intersection', + 'Add', + 'Subtract', + 'Negate', + 'PlusMinus', + 'Factorial', + 'Binomial', +]) + +/** Compute Engine names of shared meanings that differ from the speech tree. */ +const computeEngineNames: Readonly> = { Tilde: 'Similar', TildeFullEqual: 'Congruent' } +/** The signature `f: X \to Y` holds its arrow in `Mapping`. */ +const speechNames: Readonly> = { Mapping: 'To' } +/** + * Heads that chain. Compute Engine writes `a + b + c` as one `Add`, and the speech tree nests `Add` in pairs, so a + * nested head of the same name counts once. + */ +const chained = new Set(['Add', 'Multiply', 'Equal', 'Less', 'Greater', 'LessEqual', 'GreaterEqual']) +/** Compute Engine units are built from `Divide` and `Power`. The speech tree has `Unit`, so unit subtrees are skipped. */ +const unitHeads = new Set(['__unit__', 'Unit']) + +/** Counts the shared heads of a MathJSON tree, for example `{ Divide: 2, Power: 1 }`. */ +function headCounts(expression: unknown, rename: Readonly> = {}): Map { + const counts = new Map() + const visit = (value: unknown, parent?: string) => { + if (!Array.isArray(value)) + return + const [head, ...args] = value as [unknown, ...unknown[]] + if (typeof head !== 'string' || unitHeads.has(head)) + return + const name = rename[head] ?? head + if (sharedHeads.has(name) && (!chained.has(name) || parent !== name)) + counts.set(name, (counts.get(name) ?? 0) + 1) + args.forEach(argument => visit(argument, name)) + } + visit(expression) + return counts +} + +/** Lists the heads that one tree has more often than the other, for example `Power -1` when the speech tree lacks one. */ +function difference(speech: Map, reference: Map): string { + const heads = [...new Set([...speech.keys(), ...reference.keys()])].sort() + return heads + .map(head => [head, (speech.get(head) ?? 0) - (reference.get(head) ?? 0)] as const) + .filter(([, delta]) => delta !== 0) + .map(([head, delta]) => `${head} ${delta > 0 ? '+' : ''}${delta}`) + .join(', ') +} + +describe('speech MathJSON against Compute Engine', () => { + // Runs on every formula case without a model call. The snapshot lists where the shared heads differ, so a change in + // lowering shows up here before an eval run. A `+` is a head that only the speech tree has, and a `-` one that it lacks. + it('finds the same standard structures as Compute Engine', () => { + const engine = new ComputeEngine() + const differences: Record = {} + for (const item of formulaSpeechCases) { + // Currency cases are sentences, and Compute Engine has no chemistry. + if (typeof item.input !== 'string' || item.input.includes('\\ce')) + continue + const tex = item.input.replace(/^\$|\$$/g, '') + const speech: MathJson = reader.toMathJson(tex) + const reference = engine.parse(tex, { form: 'raw' }).json + const found = difference(headCounts(speech, speechNames), headCounts(reference, computeEngineNames)) + if (found) + differences[item.id] = found + } + + // Known differences that the speech tree keeps by design: + // - root-quadratic: Compute Engine reads `-b \pm \sqrt{...}` as a `Measurement` with an uncertainty. + // - unit-c: Compute Engine folds `3\times10^{8}` into one number, so it has no `Power`. + expect(differences).toMatchInlineSnapshot(` + { + "root-quadratic": "PlusMinus +1", + "unit-c": "Power +1", + } + `) + }) +}) diff --git a/evals/tts-speech-formula/eval/test-objects/rule/tasks/read-aloud/formula-speech/en.eval.ts b/evals/tts-speech-formula/eval/test-objects/rule/tasks/read-aloud/formula-speech/en.eval.ts new file mode 100644 index 0000000000..fb474090bb --- /dev/null +++ b/evals/tts-speech-formula/eval/test-objects/rule/tasks/read-aloud/formula-speech/en.eval.ts @@ -0,0 +1,5 @@ +import { describeFormulaSpeechTask } from '@proj-airi/evals-tts-speech-formula' + +import { readAloud } from '../../../src/read-aloud' + +describeFormulaSpeechTask('en', readAloud) diff --git a/evals/tts-speech-formula/eval/test-objects/rule/tasks/read-aloud/formula-speech/ja.eval.ts b/evals/tts-speech-formula/eval/test-objects/rule/tasks/read-aloud/formula-speech/ja.eval.ts new file mode 100644 index 0000000000..8a686cf651 --- /dev/null +++ b/evals/tts-speech-formula/eval/test-objects/rule/tasks/read-aloud/formula-speech/ja.eval.ts @@ -0,0 +1,5 @@ +import { describeFormulaSpeechTask } from '@proj-airi/evals-tts-speech-formula' + +import { readAloud } from '../../../src/read-aloud' + +describeFormulaSpeechTask('ja', readAloud) diff --git a/evals/tts-speech-formula/eval/test-objects/rule/tasks/read-aloud/formula-speech/ko.eval.ts b/evals/tts-speech-formula/eval/test-objects/rule/tasks/read-aloud/formula-speech/ko.eval.ts new file mode 100644 index 0000000000..711a60ce25 --- /dev/null +++ b/evals/tts-speech-formula/eval/test-objects/rule/tasks/read-aloud/formula-speech/ko.eval.ts @@ -0,0 +1,5 @@ +import { describeFormulaSpeechTask } from '@proj-airi/evals-tts-speech-formula' + +import { readAloud } from '../../../src/read-aloud' + +describeFormulaSpeechTask('ko', readAloud) diff --git a/evals/tts-speech-formula/eval/test-objects/rule/tasks/read-aloud/formula-speech/zh.eval.ts b/evals/tts-speech-formula/eval/test-objects/rule/tasks/read-aloud/formula-speech/zh.eval.ts new file mode 100644 index 0000000000..54e6e9b874 --- /dev/null +++ b/evals/tts-speech-formula/eval/test-objects/rule/tasks/read-aloud/formula-speech/zh.eval.ts @@ -0,0 +1,5 @@ +import { describeFormulaSpeechTask } from '@proj-airi/evals-tts-speech-formula' + +import { readAloud } from '../../../src/read-aloud' + +describeFormulaSpeechTask('zh', readAloud) diff --git a/evals/tts-speech-formula/eval/test-objects/rule/vieval.config.ts b/evals/tts-speech-formula/eval/test-objects/rule/vieval.config.ts new file mode 100644 index 0000000000..e2aed2b773 --- /dev/null +++ b/evals/tts-speech-formula/eval/test-objects/rule/vieval.config.ts @@ -0,0 +1,3 @@ +import { defineFormulaSpeechConfig } from '@proj-airi/evals-tts-speech-formula/config' + +export default defineFormulaSpeechConfig('rule') diff --git a/evals/tts-speech-formula/eval/test-objects/sre/src/read-aloud.ts b/evals/tts-speech-formula/eval/test-objects/sre/src/read-aloud.ts new file mode 100644 index 0000000000..734ee2432f --- /dev/null +++ b/evals/tts-speech-formula/eval/test-objects/sre/src/read-aloud.ts @@ -0,0 +1,55 @@ +import type { Language, ReadAloud } from '@proj-airi/evals-tts-speech-formula' + +import { createRequire } from 'node:module' + +import katex from 'katex' + +import 'katex/contrib/mhchem' + +/** + * Reads formulas with speech-rule-engine. + * + * The engine keeps one global locale. Calls are queued, so concurrent cases cannot switch the locale in the middle of another read. + */ +class SpeechRuleEngineReader { + // WORKAROUND: + // The ESM entry `speech-rule-engine/js/index.js` loads its CLI, which requires `commander`, and it cannot find the locale files. + // The CommonJS bundle that `main` names works, but the package ships types only for the ESM sources. + // Source: node_modules/speech-rule-engine/package.json (no `exports` or `types`) and js/common/cli.js. + // Remove when speech-rule-engine ships typed exports that load locales. + private readonly engine: typeof import('speech-rule-engine/js/common/system.js') = createRequire(import.meta.url)('speech-rule-engine') + private queue: Promise = Promise.resolve() + private locale: string | undefined + + read(markdown: string, language: Language): Promise { + const next = this.queue.then(() => this.readNow(markdown, language)) + this.queue = next.catch(() => undefined) + return next + } + + private async readNow(markdown: string, language: Language): Promise { + // Only English and Korean have locales. Chinese and Japanese get the English reading, as an SRE integration in AIRI would. + const locale = language === 'ko' ? 'ko' : 'en' + if (this.locale !== locale) { + await this.engine.setupEngine({ locale, domain: 'clearspeak', modality: 'speech', style: 'default' }) + await this.engine.engineReady() + this.locale = locale + } + return markdown + .replace(/\$\$([\s\S]+?)\$\$|\$([^$]+)\$/g, (_, block: string | undefined, inline: string | undefined) => ` ${this.engine.toSpeech(mathml(block ?? inline ?? ''))} `) + .replace(/\s+/g, ' ') + .trim() + } +} + +/** KaTeX MathML without the outer span. The span makes SRE also read the TeX annotation. */ +function mathml(tex: string): string { + return katex.renderToString(tex, { output: 'mathml', throwOnError: false, strict: 'ignore' }) + .replace(/^/, '') + .replace(/<\/span>$/, '') +} + +const speechRuleEngine = new SpeechRuleEngineReader() + +/** speech-rule-engine ClearSpeak on each formula. It has no Chinese or Japanese locale, so those languages get the English reading. */ +export const readAloud: ReadAloud = async (markdown, language) => speechRuleEngine.read(markdown, language) diff --git a/evals/tts-speech-formula/eval/test-objects/sre/tasks/read-aloud/formula-speech/en.eval.ts b/evals/tts-speech-formula/eval/test-objects/sre/tasks/read-aloud/formula-speech/en.eval.ts new file mode 100644 index 0000000000..fb474090bb --- /dev/null +++ b/evals/tts-speech-formula/eval/test-objects/sre/tasks/read-aloud/formula-speech/en.eval.ts @@ -0,0 +1,5 @@ +import { describeFormulaSpeechTask } from '@proj-airi/evals-tts-speech-formula' + +import { readAloud } from '../../../src/read-aloud' + +describeFormulaSpeechTask('en', readAloud) diff --git a/evals/tts-speech-formula/eval/test-objects/sre/tasks/read-aloud/formula-speech/ja.eval.ts b/evals/tts-speech-formula/eval/test-objects/sre/tasks/read-aloud/formula-speech/ja.eval.ts new file mode 100644 index 0000000000..8a686cf651 --- /dev/null +++ b/evals/tts-speech-formula/eval/test-objects/sre/tasks/read-aloud/formula-speech/ja.eval.ts @@ -0,0 +1,5 @@ +import { describeFormulaSpeechTask } from '@proj-airi/evals-tts-speech-formula' + +import { readAloud } from '../../../src/read-aloud' + +describeFormulaSpeechTask('ja', readAloud) diff --git a/evals/tts-speech-formula/eval/test-objects/sre/tasks/read-aloud/formula-speech/ko.eval.ts b/evals/tts-speech-formula/eval/test-objects/sre/tasks/read-aloud/formula-speech/ko.eval.ts new file mode 100644 index 0000000000..711a60ce25 --- /dev/null +++ b/evals/tts-speech-formula/eval/test-objects/sre/tasks/read-aloud/formula-speech/ko.eval.ts @@ -0,0 +1,5 @@ +import { describeFormulaSpeechTask } from '@proj-airi/evals-tts-speech-formula' + +import { readAloud } from '../../../src/read-aloud' + +describeFormulaSpeechTask('ko', readAloud) diff --git a/evals/tts-speech-formula/eval/test-objects/sre/tasks/read-aloud/formula-speech/zh.eval.ts b/evals/tts-speech-formula/eval/test-objects/sre/tasks/read-aloud/formula-speech/zh.eval.ts new file mode 100644 index 0000000000..54e6e9b874 --- /dev/null +++ b/evals/tts-speech-formula/eval/test-objects/sre/tasks/read-aloud/formula-speech/zh.eval.ts @@ -0,0 +1,5 @@ +import { describeFormulaSpeechTask } from '@proj-airi/evals-tts-speech-formula' + +import { readAloud } from '../../../src/read-aloud' + +describeFormulaSpeechTask('zh', readAloud) diff --git a/evals/tts-speech-formula/eval/test-objects/sre/vieval.config.ts b/evals/tts-speech-formula/eval/test-objects/sre/vieval.config.ts new file mode 100644 index 0000000000..68392ba0c1 --- /dev/null +++ b/evals/tts-speech-formula/eval/test-objects/sre/vieval.config.ts @@ -0,0 +1,3 @@ +import { defineFormulaSpeechConfig } from '@proj-airi/evals-tts-speech-formula/config' + +export default defineFormulaSpeechConfig('sre') diff --git a/evals/tts-speech-formula/package.json b/evals/tts-speech-formula/package.json new file mode 100644 index 0000000000..2d32fc9456 --- /dev/null +++ b/evals/tts-speech-formula/package.json @@ -0,0 +1,63 @@ +{ + "name": "@proj-airi/evals-tts-speech-formula", + "type": "module", + "private": true, + "description": "Vieval suite for how well listeners understand formulas that AIRI reads aloud", + "author": { + "name": "Moeru AI Project AIRI Team", + "email": "airi@moeru.ai", + "url": "https://github.com/moeru-ai" + }, + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/moeru-ai/airi.git", + "directory": "evals/tts-speech-formula" + }, + "exports": { + ".": "./eval/index.ts", + "./config": "./eval/tasks/read-aloud/config.ts", + "./gemini-transcription": "./eval/evaluators/speech/gemini-transcription.ts", + "./speech-models": "./eval/evaluators/speech/speech-models.ts" + }, + "scripts": { + "eval:compare": "pnpm -F @proj-airi/core-agent build && vieval run --config ./vieval.config.ts --with-viewer", + "typecheck": "tsc --noEmit", + "cases:math-bridge": "tsx eval/cases/math-bridge/fetch.ts", + "judge:collect": "tsx eval/evaluators/judge/calibration/collect.ts", + "judge:label": "vite --config eval/evaluators/judge/calibration/vite.config.ts" + }, + "dependencies": { + "@cortex-js/compute-engine": "catalog:", + "@duckdb/duckdb-wasm": "catalog:", + "@moeru/std": "catalog:", + "@proj-airi/core-agent": "workspace:^", + "@proj-airi/duckdb-wasm": "catalog:", + "@proj-airi/mathjson": "workspace:^", + "@proj-airi/speech-transformer-math": "workspace:^", + "@velin-dev/core-vue": "catalog:", + "@vieval/ui": "catalog:", + "@xsai-apple-speech/transcription-native": "catalog:", + "@xsai/generate-speech": "catalog:", + "@xsai/generate-text": "catalog:", + "@xsai/generate-transcription": "catalog:", + "katex": "catalog:", + "speech-rule-engine": "catalog:", + "valibot": "catalog:", + "web-worker": "catalog:" + }, + "optionalDependencies": { + "@xsai-apple-speech/transcription-native-darwin-arm64": "catalog:", + "@xsai-apple-speech/transcription-native-darwin-x64": "catalog:" + }, + "devDependencies": { + "@types/node": "catalog:", + "rehype-stringify": "catalog:", + "remark-parse": "catalog:", + "remark-rehype": "catalog:", + "unified": "catalog:", + "vieval": "catalog:", + "vite": "catalog:", + "vitest": "catalog:vitest" + } +} diff --git a/evals/tts-speech-formula/tsconfig.json b/evals/tts-speech-formula/tsconfig.json new file mode 100644 index 0000000000..b7a7e09d48 --- /dev/null +++ b/evals/tts-speech-formula/tsconfig.json @@ -0,0 +1,24 @@ +{ + "compilerOptions": { + "target": "ESNext", + "lib": [ + "ESNext", + "DOM" + ], + "module": "ESNext", + "moduleResolution": "bundler", + "types": [ + "node" + ], + "strict": true, + "noEmit": true, + "isolatedModules": true, + "verbatimModuleSyntax": true, + "skipLibCheck": true + }, + "include": [ + "eval/**/*.ts", + "vieval.config.ts", + "vitest.config.ts" + ] +} diff --git a/evals/tts-speech-formula/vieval.config.ts b/evals/tts-speech-formula/vieval.config.ts new file mode 100644 index 0000000000..13432a9b21 --- /dev/null +++ b/evals/tts-speech-formula/vieval.config.ts @@ -0,0 +1,14 @@ +import { defineConfig } from 'vieval' + +/** + * Compares the test objects on the formula-speech dataset. + * + * Each directory under `eval/test-objects/` is one workspace: one way to turn a reply into spoken text. + * `vieval run` runs every workspace in its own process, and `benchmark.case.id` aligns their cases, which leaves the test object out. + * The listener, ASR, and TTS answers are cached once for all test objects under `.vieval/cache` next to this file. + */ +export default defineConfig({ + id: 'tts-speech-formula', + cache: { namespace: 'formula-speech-v1' }, + workspaces: ['eval/test-objects/*'], +}) diff --git a/evals/tts-speech-formula/vitest.config.ts b/evals/tts-speech-formula/vitest.config.ts new file mode 100644 index 0000000000..21baad8ab3 --- /dev/null +++ b/evals/tts-speech-formula/vitest.config.ts @@ -0,0 +1,8 @@ +import { defineConfig } from 'vitest/config' + +export default defineConfig({ + test: { + // Unit tests cover the evaluators and the adapters. `*.eval.ts` files call paid models and run through vieval. + include: ['eval/**/*.test.ts'], + }, +}) diff --git a/packages/core-agent/README.md b/packages/core-agent/README.md index a57abc10a7..102d72e3ab 100644 --- a/packages/core-agent/README.md +++ b/packages/core-agent/README.md @@ -262,8 +262,31 @@ await response.finish() ``` Producers can synthesize concurrently. Playback follows their reservation order. -A producer holds its text until `end()`. Then `toSpeechText` removes markdown that has no sound, for example emphasis delimiters, link URLs, and code blocks. +A producer streams its text to TTS by segments. `MarkdownSegmenter` ends a segment after a sentence or a line, only where no formula, code, link, emphasis, or table is open. Then `toSpeechText` reads the segment and removes markdown that has no sound, for example emphasis delimiters, link URLs, and code blocks. +So an answer starts to speak after its first sentence, and a formula that later writes complete is still read as one formula. Symbols that markdown keeps as text, for example `2 * 3` and `x ~= 3`, stay in the speech text. +TeX formulas, `\ce{}` chemistry, units, and dollar prices are read as words of the sentence language: Chinese, English, Japanese, or Korean. +The rules are in [`@proj-airi/speech-transformer-math`](../speech-transformer-math) and [`@proj-airi/speech-transformer-chem`](../speech-transformer-chem), see [the ADR](../../docs/ai/adr/2026-10-20-speech-text-preprocessor.md). +`mathSpeech` is the instance that AIRI uses: the built-in locale packs, with chemistry. + +`toMathJson(tex)` returns the MathJSON of a formula without reading it, for tools that inspect formulas. +`options.language` is the BCP 47 code of the voice. The words around a formula or a price select its language first. + +`toSpeechText` is a host of preprocessor plugins. The default plugins are `moneySpeechPreprocessor` and `mathSpeechPreprocessor`. +A plugin can have three hooks: `prepare` rewrites the markdown, `formula` reads one formula, and `prose` rewrites a run of text. +To add a reading, pass `preprocessors: [myPlugin, ...defaultSpeechPreprocessors()]`. + +```ts +const spoken = toSpeechText(markdown, { + preprocessors: [ + { name: 'brand', prose: text => text.replaceAll('AIRI', 'ai ree') }, + ...defaultSpeechPreprocessors(), + ], +}) +``` + +The words of each language are in a locale pack of `@proj-airi/speech-transformer-math`. +To add a language or change one, call `mathSpeech.register(pack)`. A language without a pack reads in English. Special tokens and flushes keep their position relative to the text. Cancelling an acknowledgment releases its place without reporting a user interruption. Whole-turn interruption aborts generation, synthesis, queued audio, and playback for that response. diff --git a/packages/core-agent/package.json b/packages/core-agent/package.json index 67f966b82b..f83700f7f9 100644 --- a/packages/core-agent/package.json +++ b/packages/core-agent/package.json @@ -44,11 +44,14 @@ "dependencies": { "@moeru/eventa": "catalog:", "@moeru/std": "catalog:", + "@proj-airi/mathjson": "workspace:^", "@proj-airi/pipelines-audio": "workspace:^", "@proj-airi/plugin-protocol": "workspace:^", "@proj-airi/provider-inference": "workspace:^", "@proj-airi/server-sdk": "workspace:^", "@proj-airi/server-shared": "workspace:^", + "@proj-airi/speech-transformer-chem": "workspace:^", + "@proj-airi/speech-transformer-math": "workspace:^", "@proj-airi/stream-kit": "workspace:^", "@xsai-ext/providers": "catalog:", "@xsai-ext/responses": "catalog:", @@ -62,6 +65,7 @@ "nanoid": "catalog:", "rehype-parse": "catalog:", "rehype-stringify": "catalog:", + "remark-math": "catalog:", "remark-parse": "catalog:", "unified": "catalog:", "unist-util-visit": "catalog:", diff --git a/packages/core-agent/src/voice/index.ts b/packages/core-agent/src/voice/index.ts index fb617d65c0..7dc18f01bb 100644 --- a/packages/core-agent/src/voice/index.ts +++ b/packages/core-agent/src/voice/index.ts @@ -26,6 +26,10 @@ export type { export type { Interruption, VoiceInterruptionEvent } from './interruption' export { SpeechStream, VoiceResponse } from './output/response' export type { SpeechAudio, SpeechClip, SpeechClipEnd, SpeechOutput } from './output/response' +export { defaultSpeechPreprocessors, toSpeechText } from './output/speech-transformation' +export type { SpeechTextOptions } from './output/speech-transformation' +export { mathSpeech, mathSpeechPreprocessor, moneySpeechPreprocessor, speakMath, toMathJson } from './output/speech-transformation/preprocessor' +export type { SpeechFormulaContext, SpeechPreprocessor, SpeechProseContext } from './output/speech-transformation/preprocessor' export type { AudioPluginTask, FinalizeContext, diff --git a/packages/core-agent/src/voice/output/response.test.ts b/packages/core-agent/src/voice/output/response.test.ts index 5365144a88..1732128420 100644 --- a/packages/core-agent/src/voice/output/response.test.ts +++ b/packages/core-agent/src/voice/output/response.test.ts @@ -155,6 +155,47 @@ describe('voiceController output', () => { await controller.close() }) + it('reads a formula that two writes split as one formula in the voice language', async () => { + const requested: string[] = [] + const playback = new Playback({ nowMs: () => 0, play: () => ({ done: Promise.resolve({ throughMs: 1 }), stop: async () => ({ throughMs: 0 }) }) }) + const controller = new VoiceController({ speech: () => ({ playback, language: 'zh-CN', synthesize: async (request) => { + requested.push(request.text) + return new Blob([request.text]) + } }) }) + const response = controller.openResponse({ sessionId: 'alice', turnId: 'formula' }) + const speech = response.openSpeech({ purpose: 'answer' }) + // A model streams tokens, so the closing `$` and the `\\to` arrive in later writes. + speech.write('结果是 $\\lim_{n \\t') + speech.write('o \\infty} \\frac{1}{n} = 0$。\n\n$$\n') + speech.write('x \\in A\n$$') + speech.end() + + expect(await response.finish()).toBe('finished') + expect(requested).toEqual(['结果是 当n趋于无穷大时 n分之1的极限 等于 0。', 'x 属于 A']) + await controller.close() + }) + + it('synthesizes the first sentence before the model ends, and holds an open formula', async () => { + const requested: string[] = [] + const playback = new Playback({ nowMs: () => 0, play: () => ({ done: Promise.resolve({ throughMs: 1 }), stop: async () => ({ throughMs: 0 }) }) }) + const controller = new VoiceController({ speech: () => ({ playback, language: 'en-US', synthesize: async (request) => { + requested.push(request.text) + return new Blob([request.text]) + } }) }) + const response = controller.openResponse({ sessionId: 'alice', turnId: 'stream' }) + const speech = response.openSpeech({ purpose: 'answer' }) + speech.write('The answer is ready. The area is $x^') + await vi.waitFor(() => expect(requested).toEqual(['The answer is ready.'])) + + // The formula is still open, so its source does not reach TTS. + speech.write('2$. Done') + speech.end() + + expect(await response.finish()).toBe('finished') + expect(requested).toEqual(['The answer is ready.', 'The area is x squared.', 'Done']) + await controller.close() + }) + it('retains speech-onset samples while waiting for playback silence', async () => { const source = createPushStream() const audio = new AudioInput(pcmSource(source.stream), { historyMs: 4 }) diff --git a/packages/core-agent/src/voice/output/response.ts b/packages/core-agent/src/voice/output/response.ts index 52a04f590b..1cc919bc1a 100644 --- a/packages/core-agent/src/voice/output/response.ts +++ b/packages/core-agent/src/voice/output/response.ts @@ -1,5 +1,4 @@ import type { AudioPlayback, IntentHandle, PcmBlock, PlaybackGroup, PlaybackItem, PlaybackReceipt, SpeechPipelineEvents, SpeechPipelineOptions } from '@proj-airi/pipelines-audio' - import type { Outcome } from '@proj-airi/stream-kit' import type { TurnRef } from '../turn' @@ -9,7 +8,8 @@ import { createPushStream } from '@proj-airi/stream-kit' import { nanoid } from 'nanoid/non-secure' import { errorFromCause, errorMessageFromValue } from '../../utils/error' -import { toSpeechText } from './speech-text' +import { toSpeechText } from './speech-transformation' +import { MarkdownSegmenter } from './speech-transformation/segmenter' /** Each synthesized part carries optional display text. Audio ownership transfers to playback. */ export interface SpeechAudio { @@ -43,6 +43,8 @@ export type SpeechOutput = { readonly onPlaybackStart?: (clip: SpeechClip) => void /** Runs once for each started clip after playback ends, stops, or fails. */ readonly onPlaybackEnd?: (clip: SpeechClip, end: SpeechClipEnd) => void + /** BCP 47 code of the voice. The words around a formula or a price select its language first. */ + readonly language?: string } & ({ readonly synthesize: SpeechPipelineOptions['tts'] } | { @@ -51,7 +53,7 @@ export type SpeechOutput = { }) /** - * Marks where a special token or a flush occurs in the buffered text, so that markdown conversion keeps its position. + * Marks where a special token or a flush occurs in the streamed text, so that markdown conversion keeps its position. * It is a Unicode private use character. Model text does not contain it, and producer text is stripped of it. */ const controlMarker = '\uE000' @@ -75,9 +77,12 @@ export class SpeechStream { private settled = false private timer: ReturnType | undefined - /** Markdown that `end()` converts to speech text. A control marker holds the place of each entry in `controls`. */ - private markdown = '' + /** Holds the text after the last safe end of the streamed markdown. A control marker holds the place of a control. */ + private readonly segmenter = new MarkdownSegmenter() + /** Controls whose markers are still in the segmenter, in order. */ private readonly controls: SpeechControl[] = [] + /** The spoken text of the last segment, for the language of a formula that starts the next one. */ + private spoken = '' readonly signal = this.abort.signal readonly done = this.completion.promise @@ -133,38 +138,39 @@ export class SpeechStream { const clip = { id: item.segmentId, text: item.text } let started = false void Promise.race([previous, this.done]) - .then(async () => { - if (this.signal.aborted || response.signal.aborted) - return 'stopped' as const + .then(async () => { + if (this.signal.aborted || response.signal.aborted) + return 'stopped' as const - return response.playback.enqueue({ - id: item.id, - audio: item.audio, - signal: this.signal, - onStart: () => { - started = true - listeners.start.forEach(listener => listener({ item, startedAt: Date.now() })) - output.onPlaybackStart?.(clip) - } }) - }) - .then((result) => { - if (started) - output.onPlaybackEnd?.(clip, this.clipEnd(result)) + return response.playback.enqueue({ + id: item.id, + audio: item.audio, + signal: this.signal, + onStart: () => { + started = true + listeners.start.forEach(listener => listener({ item, startedAt: Date.now() })) + output.onPlaybackStart?.(clip) + }, + }) + }) + .then((result) => { + if (started) + output.onPlaybackEnd?.(clip, this.clipEnd(result)) - if (!pending.delete(item)) - return + if (!pending.delete(item)) + return - if (result === 'ended') - listeners.end.forEach(listener => listener({ item, endedAt: Date.now() })) - else - listeners.interrupt.forEach(listener => listener({ item, interruptedAt: Date.now(), reason: result })) + if (result === 'ended') + listeners.end.forEach(listener => listener({ item, endedAt: Date.now() })) + else + listeners.interrupt.forEach(listener => listener({ item, interruptedAt: Date.now(), reason: result })) - // REVIEW: can we have more manage-able error handling - // across our pipelines? - if (result === 'failed') - this.fail(new Error('Speech playback failed')) + // REVIEW: can we have more manage-able error handling + // across our pipelines? + if (result === 'failed') + this.fail(new Error('Speech playback failed')) // REVIEW: must we do this here? - }, cause => this.fail(errorFromCause(cause, 'Speech playback failed'))) + }, cause => this.fail(errorFromCause(cause, 'Speech playback failed'))) }, stopByIntent: (_id, reason) => { for (const item of pending) @@ -249,33 +255,32 @@ export class SpeechStream { return signal.aborted ? { status, reason: errorMessageFromValue(signal.reason) } : { status } } - // REVIEW: - // Every purpose holds its text until `end()`, so markdown is converted with the full context. - // read-aloud already has the full text, but answer and notification start to speak only after the model ends. - // answer needs a streaming commit at clause boundaries. See the chunker punctuation in pipelines-audio. + // Text streams to TTS by segments, so an answer starts to speak after its first sentence. The segmenter ends a segment + // only where markdown parses alone: a formula, a code block, or a link that is still open waits for its end. - /** Special tokens keep their position relative to the text. They are delivered when the producer ends. */ + /** Special tokens keep their position relative to the text. */ special(value: string) { if (this.sealed || this.response.signal.aborted) return - this.markdown += controlMarker this.controls.push({ type: 'special', value }) + this.deliver(this.segmenter.push(controlMarker)) } + /** Sends the text so far, unless a formula or another construct is still open. Then the flush follows its end. */ flush() { if (this.sealed) return - this.markdown += controlMarker this.controls.push({ type: 'flush' }) + this.deliver([...this.segmenter.push(controlMarker), ...this.segmenter.flush()]) } write(text: string): { status: 'accepted' | 'closed' } { if (this.sealed || this.response.signal.aborted) return { status: 'closed' } - this.markdown += text.replaceAll(controlMarker, '') + this.deliver(this.segmenter.push(text.replaceAll(controlMarker, ''))) return { status: 'accepted' } } @@ -284,24 +289,46 @@ export class SpeechStream { return if (!this.response.signal.aborted) - this.deliver() + this.deliver(this.segmenter.end()) this.sealed = true this.text?.close() this.intent?.end() } - /** Sends the speech text and the controls in source order. Controls that conversion lost are sent last, in order. */ - private deliver() { - const parts = toSpeechText(this.markdown, { marker: controlMarker }).split(controlMarker) - parts.forEach((text, index) => { - if (text) - this.deliverText(text) + /** + * Sends each segment as speech text, with its controls in source order. Controls that conversion lost are sent after + * the text of their segment, in order. + */ + private deliver(segments: readonly string[]) { + if (this.response.signal.aborted) + return - const control = this.controls[index] - if (control && index < parts.length - 1) - this.deliverControl(control) - }) - this.controls.slice(parts.length - 1).forEach(control => this.deliverControl(control)) + for (const segment of segments) { + const count = segment.split(controlMarker).length - 1 + const controls = this.controls.splice(0, count) + const parts = toSpeechText(segment, { marker: controlMarker, language: this.output.language, before: this.spoken }).split(controlMarker) + // Segments end inside a paragraph or between blocks. The space or the line break between them stays for TTS. + const separator = /\n\s*$/.test(segment) ? '\n' : /[ \t]$/.test(segment) ? ' ' : '' + parts.forEach((text, index) => { + const last = index === parts.length - 1 + const spoken = last && text ? text + separator : text + if (spoken) + this.deliverText(spoken) + if (text.trim()) + this.spoken = text + + const control = controls[index] + if (control && !last) + this.deliverControl(control) + }) + controls.slice(parts.length - 1).forEach(control => this.deliverControl(control)) + // NOTICE: + // The TTS chunker reads one grapheme past a sentence end, to tell `3.14` and `...` from an end. Without more text, + // the last sentence of a segment waits for the next segment, which can be a long formula or code block. + // Source: chunkTtsInput in packages/pipelines-audio/src/processors/tts-chunker.ts. + // Remove when the chunker ends a chunk at the end of its input without lookahead. + this.intent?.writeFlush() + } } private deliverText(text: string) { diff --git a/packages/core-agent/src/voice/output/speech-text.test.ts b/packages/core-agent/src/voice/output/speech-text.test.ts deleted file mode 100644 index 7c83cc7a18..0000000000 --- a/packages/core-agent/src/voice/output/speech-text.test.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { describe, expect, it } from 'vitest' - -import { toSpeechText } from './speech-text' - -describe('toSpeechText', () => { - it('removes markup that has no sound', () => { - expect(toSpeechText('This is **important** and *soft*.')).toBe('This is important and soft.') - expect(toSpeechText('这是**重要**的事情。')).toBe('这是重要的事情。') - expect(toSpeechText('Read [the guide](https://example.com/guide) first.')).toBe('Read the guide first.') - expect(toSpeechText('Run `pnpm test` now.')).toBe('Run pnpm test now.') - expect(toSpeechText('A ![cat](cat.png) sleeps.')).toBe('A cat sleeps.') - }) - - it('puts a line break between blocks so that each block ends a TTS chunk', () => { - expect(toSpeechText('# Plan\n\n- First step\n- Second step\n\n> Done')).toBe('Plan\nFirst step\nSecond step\nDone') - }) - - it('skips code blocks, HTML, and thematic breaks', () => { - expect(toSpeechText('Before.\n\n```ts\nconst a = 1\n```\n\n---\n\n
hidden
\n\nAfter.')).toBe('Before.\nAfter.') - }) - - it('keeps symbols that the writer meant as operators', () => { - expect(toSpeechText('2 * 3 = 6')).toBe('2 * 3 = 6') - expect(toSpeechText('2*3 = 6')).toBe('2*3 = 6') - expect(toSpeechText('2**10 = 1024')).toBe('2**10 = 1024') - expect(toSpeechText('x ~= 3.14, ~3 hours')).toBe('x ~= 3.14, ~3 hours') - }) - - it('keeps delimiters that CommonMark reads as emphasis between ASCII operands', () => { - expect(toSpeechText('x**2 + y**2 = r**2')).toBe('x**2 + y**2 = r**2') - expect(toSpeechText('a*b*c')).toBe('a*b*c') - }) - - it('keeps each marker, also inside unspoken nodes, in source order', () => { - const marker = '' - - expect(toSpeechText(`**${marker}Hi** there.`, { marker })).toBe(`${marker}Hi there.`) - expect(toSpeechText(`See [docs](https://e.com/${marker}).`, { marker })).toBe(`See docs${marker}.`) - expect(toSpeechText(`One.\n\n\`\`\`\n${marker}\n\`\`\`\n\nTwo.${marker}`, { marker })).toBe(`One.\n${marker}\nTwo.${marker}`) - }) -}) diff --git a/packages/core-agent/src/voice/output/speech-text.ts b/packages/core-agent/src/voice/output/speech-text.ts deleted file mode 100644 index 38da96e496..0000000000 --- a/packages/core-agent/src/voice/output/speech-text.ts +++ /dev/null @@ -1,113 +0,0 @@ -import type { Nodes, PhrasingContent, Root, RootContent } from 'mdast' -import type { Plugin } from 'unified' - -import remarkParse from 'remark-parse' - -import { unified } from 'unified' - -export interface SpeechTextOptions { - /** - * One character that the output keeps as often as the source has it, also inside unspoken nodes. - * The speech stream puts it where special tokens and flushes occur, then splits the output on it. - */ - marker?: string -} - -/** A delimiter between two ASCII operands is an operator, for example `x**2` or `a*b*c`, not emphasis. */ -const asciiOperand = /^[A-Z0-9]$/i - -/** - * Compiles markdown to the text that a TTS provider reads. - * - * Markup that has no sound is removed: emphasis delimiters, heading and list markers, link URLs, code - * blocks, and HTML. Blocks are joined with a line break, so the TTS chunker pauses between them. - * Symbols that the parser keeps as text, for example `2 * 3`, `2**10`, or `x ~= 3`, stay unchanged. - */ -export const remarkSpeech: Plugin<[SpeechTextOptions?], Root, string> = function (options) { - this.compiler = (tree, file) => compile(tree as Root, String(file), options?.marker) -} - -export function toSpeechText(markdown: string, options?: SpeechTextOptions): string { - return String(unified().use(remarkParse).use(remarkSpeech, options).processSync(markdown)) -} - -function compile(tree: Root, source: string, marker: string | undefined): string { - // TODO: Read literal symbols as words, for example `*` as "times" and `~=` as "approximately equals". - // This needs a spoken lexicon for each language. Its place, packages/i18n or a separate table, is not decided. - // TODO: Select the spoken language for each span. Replies often switch languages in one sentence, - // so the voice language alone cannot select the words. - - function count(text: string): number { - return marker ? text.split(marker).length - 1 : 0 - } - - function range(node: Nodes): string { - const start = node.position?.start.offset - const end = node.position?.end.offset - return start === undefined || end === undefined ? '' : source.slice(start, end) - } - - /** Adds the markers that the source of a node has but its spoken text lost, for example in a link URL. */ - function keepMarkers(node: Nodes, spoken: string): string { - const missing = count(range(node)) - count(spoken) - return marker && missing > 0 ? spoken + marker.repeat(missing) : spoken - } - - function isOperator(node: Nodes): boolean { - const start = node.position?.start.offset - const end = node.position?.end.offset - if (start === undefined || end === undefined) - return false - - return asciiOperand.test(source[start - 1] ?? '') && asciiOperand.test(source[end] ?? '') - } - - function blocks(nodes: readonly RootContent[]): string { - return nodes.map(block).filter(Boolean).join('\n') - } - - function block(node: RootContent): string { - switch (node.type) { - case 'paragraph': - case 'heading': - return phrasing(node.children) - case 'blockquote': - case 'list': - case 'listItem': - return blocks(node.children) - // TODO: Tell the listener that the reply has code, instead of a skip without a sound. - default: - return keepMarkers(node, '') - } - } - - function phrasing(nodes: readonly PhrasingContent[]): string { - return nodes.map(inline).join('') - } - - function inline(node: PhrasingContent): string { - switch (node.type) { - case 'text': - case 'inlineCode': - return node.value - case 'emphasis': - case 'strong': - case 'delete': - // CommonMark reads `x**2 + y**2` as strong text, but the writer meant two powers. - // The source text, delimiters included, keeps the operators. - return isOperator(node) ? range(node) : phrasing(node.children) - case 'link': - case 'linkReference': - return keepMarkers(node, phrasing(node.children)) - case 'image': - case 'imageReference': - return keepMarkers(node, node.alt ?? '') - case 'break': - return '\n' - default: - return keepMarkers(node, '') - } - } - - return blocks(tree.children) -} diff --git a/packages/core-agent/src/voice/output/speech-transformation/index.test.ts b/packages/core-agent/src/voice/output/speech-transformation/index.test.ts new file mode 100644 index 0000000000..a23944b948 --- /dev/null +++ b/packages/core-agent/src/voice/output/speech-transformation/index.test.ts @@ -0,0 +1,143 @@ +import { describe, expect, it } from 'vitest' + +import { toSpeechText } from '.' + +describe('toSpeechText', () => { + it('removes markup that has no sound', () => { + expect(toSpeechText('This is **important** and *soft*.')).toBe('This is important and soft.') + expect(toSpeechText('这是**重要**的事情。')).toBe('这是重要的事情。') + expect(toSpeechText('Read [the guide](https://example.com/guide) first.')).toBe('Read the guide first.') + expect(toSpeechText('Run `pnpm test` now.')).toBe('Run pnpm test now.') + expect(toSpeechText('A ![cat](cat.png) sleeps.')).toBe('A cat sleeps.') + }) + + it('puts a line break between blocks so that each block ends a TTS chunk', () => { + expect(toSpeechText('# Plan\n\n- First step\n- Second step\n\n> Done')).toBe('Plan\nFirst step\nSecond step\nDone') + }) + + it('skips code blocks, HTML, and thematic breaks', () => { + expect(toSpeechText('Before.\n\n```ts\nconst a = 1\n```\n\n---\n\n
hidden
\n\nAfter.')).toBe('Before.\nAfter.') + }) + + it('keeps symbols that the writer meant as operators', () => { + expect(toSpeechText('2 * 3 = 6')).toBe('2 * 3 = 6') + expect(toSpeechText('2*3 = 6')).toBe('2*3 = 6') + expect(toSpeechText('2**10 = 1024')).toBe('2**10 = 1024') + expect(toSpeechText('x ~= 3.14, ~3 hours')).toBe('x ~= 3.14, ~3 hours') + }) + + it('keeps delimiters that CommonMark reads as emphasis between ASCII operands', () => { + expect(toSpeechText('x**2 + y**2 = r**2')).toBe('x**2 + y**2 = r**2') + expect(toSpeechText('a*b*c')).toBe('a*b*c') + }) + + it('keeps each marker, also inside unspoken nodes, in source order', () => { + const marker = '' + + expect(toSpeechText(`**${marker}Hi** there.`, { marker })).toBe(`${marker}Hi there.`) + expect(toSpeechText(`See [docs](https://e.com/${marker}).`, { marker })).toBe(`See docs${marker}.`) + expect(toSpeechText(`One.\n\n\`\`\`\n${marker}\n\`\`\`\n\nTwo.${marker}`, { marker })).toBe(`One.\n${marker}\nTwo.${marker}`) + }) + + // The cases of the T-151 baseline recording, in Chinese, English, Japanese, and Korean sentences. + // Before this change each formula reached the TTS provider as TeX source, with `$` and backslashes. + it.each([ + ['a-inline', 'zh', '巴塞尔问题的结论是 $\\lim_{n \\to \\infty}\\sum_{i=1}^{n}\\frac{1}{i^2}=\\frac{\\pi^2}{6}$，这个结果由欧拉证明。', '巴塞尔问题的结论是 当n趋于无穷大时 i从1到n对i的平方分之1求和的极限 等于 分子是派的平方 分母是6的分数，这个结果由欧拉证明。'], + ['a-inline', 'en', 'The Basel problem states that $\\lim_{n \\to \\infty}\\sum_{i=1}^{n}\\frac{1}{i^2}=\\frac{\\pi^2}{6}$, which Euler proved.', 'The Basel problem states that the limit as n approaches infinity of the sum from i equals one to n of the fraction with numerator one and denominator i squared equals pi squared over six, which Euler proved.'], + ['a-inline', 'ja', 'バーゼル問題の結論は $\\lim_{n \\to \\infty}\\sum_{i=1}^{n}\\frac{1}{i^2}=\\frac{\\pi^2}{6}$ で、オイラーが証明しました。', 'バーゼル問題の結論は エヌを無限大に近づけたときのアイがいちからエヌまでのアイの二乗分のいちの和の極限 イコール 分子がパイの二乗、分母がろくの分数 で、オイラーが証明しました。'], + ['a-inline', 'ko', '바젤 문제의 결론은 $\\lim_{n \\to \\infty}\\sum_{i=1}^{n}\\frac{1}{i^2}=\\frac{\\pi^2}{6}$ 이며, 오일러가 증명했습니다.', '바젤 문제의 결론은 엔이 무한대로 갈 때 아이가 일부터 엔까지 아이 제곱 분의 일의 합의 극한은 분자가 파이 제곱, 분모가 육인 분수 이며, 오일러가 증명했습니다.'], + ['a-block', 'zh', '巴塞尔问题的结论是：\n\n$$\n\\lim_{n \\to \\infty}\\sum_{i=1}^{n}\\frac{1}{i^2}=\\frac{\\pi^2}{6}\n$$\n\n这个结果由欧拉证明。', '巴塞尔问题的结论是：\n当n趋于无穷大时 i从1到n对i的平方分之1求和的极限 等于 分子是派的平方 分母是6的分数\n这个结果由欧拉证明。'], + ['a-block', 'en', 'The Basel problem states that:\n\n$$\n\\lim_{n \\to \\infty}\\sum_{i=1}^{n}\\frac{1}{i^2}=\\frac{\\pi^2}{6}\n$$\n\nEuler proved this result.', 'The Basel problem states that:\nthe limit as n approaches infinity of the sum from i equals one to n of the fraction with numerator one and denominator i squared equals pi squared over six\nEuler proved this result.'], + ['a-block', 'ja', 'バーゼル問題の結論は次のとおりです。\n\n$$\n\\lim_{n \\to \\infty}\\sum_{i=1}^{n}\\frac{1}{i^2}=\\frac{\\pi^2}{6}\n$$\n\nオイラーがこれを証明しました。', 'バーゼル問題の結論は次のとおりです。\nエヌを無限大に近づけたときのアイがいちからエヌまでのアイの二乗分のいちの和の極限 イコール 分子がパイの二乗、分母がろくの分数\nオイラーがこれを証明しました。'], + ['a-block', 'ko', '바젤 문제의 결론은 다음과 같습니다.\n\n$$\n\\lim_{n \\to \\infty}\\sum_{i=1}^{n}\\frac{1}{i^2}=\\frac{\\pi^2}{6}\n$$\n\n오일러가 이를 증명했습니다.', '바젤 문제의 결론은 다음과 같습니다.\n엔이 무한대로 갈 때 아이가 일부터 엔까지 아이 제곱 분의 일의 합의 극한은 분자가 파이 제곱, 분모가 육인 분수\n오일러가 이를 증명했습니다.'], + ['b1-water', 'zh', '氢气在氧气中燃烧的化学方程式是 $\\ce{2H2 + O2 -> 2H2O}$。', '氢气在氧气中燃烧的化学方程式是 氢气 和 氧气 反应生成 水。'], + ['b1-water', 'en', 'When hydrogen burns in oxygen, the equation is $\\ce{2H2 + O2 -> 2H2O}$.', 'When hydrogen burns in oxygen, the equation is two H two plus O two yields two H two O.'], + ['b1-water', 'ja', '水素が酸素中で燃える反応式は $\\ce{2H2 + O2 -> 2H2O}$ です。', '水素が酸素中で燃える反応式は 水素と酸素が反応して 水ができる です。'], + ['b1-water', 'ko', '수소가 산소 속에서 타는 반응식은 $\\ce{2H2 + O2 -> 2H2O}$ 입니다.', '수소가 산소 속에서 타는 반응식은 수소와 산소가 반응하여 물이 생성된다 입니다.'], + ['b2-h2so4', 'zh', '硫酸的化学式是 $H_2SO_4$，它是一种强酸。', '硫酸的化学式是 H2 S O4，它是一种强酸。'], + ['b2-h2so4', 'en', 'The chemical formula of sulfuric acid is $H_2SO_4$, a strong acid.', 'The chemical formula of sulfuric acid is H two S O four, a strong acid.'], + ['b2-h2so4', 'ja', '硫酸の化学式は $H_2SO_4$ で、強い酸です。', '硫酸の化学式は エイチにエスオーよん で、強い酸です。'], + ['b2-h2so4', 'ko', '황산의 화학식은 $H_2SO_4$ 이며, 강산입니다.', '황산의 화학식은 에이치 아래첨자 이 에스 오 아래첨자 사 이며, 강산입니다.'], + ['b3-fe3', 'zh', '三价铁离子写作 $\\ce{Fe^{3+}}$，它的溶液呈黄色。', '三价铁离子写作 Fe 3价正离子，它的溶液呈黄色。'], + ['b3-fe3', 'en', 'The iron three ion is written as $\\ce{Fe^{3+}}$ in chemistry.', 'The iron three ion is written as F E three plus in chemistry.'], + ['b3-fe3', 'ja', '鉄(III)イオンは $\\ce{Fe^{3+}}$ と書き、その水溶液は黄色です。', '鉄(III)イオンは エフ イー さん価の陽イオン と書き、その水溶液は黄色です。'], + ['b3-fe3', 'ko', '철 3가 이온은 $\\ce{Fe^{3+}}$ 로 쓰며, 그 용액은 노란색입니다.', '철 3가 이온은 에프 E 삼가 양이온 로 쓰며, 그 용액은 노란색입니다.'], + ['b4-nh3', 'zh', '合成氨是一个可逆反应：$\\ce{N2 + 3H2 <=> 2NH3}$。', '合成氨是一个可逆反应：氮气 和 氢气 可逆反应生成 氨气。'], + ['b4-nh3', 'en', 'The Haber process is a reversible reaction: $\\ce{N2 + 3H2 <=> 2NH3}$.', 'The Haber process is a reversible reaction: N two plus three H two is in equilibrium with two N H three.'], + ['b4-nh3', 'ja', 'アンモニアの合成は可逆反応です：$\\ce{N2 + 3H2 <=> 2NH3}$。', 'アンモニアの合成は可逆反応です：窒素と水素が可逆的に反応して アンモニアができる。'], + ['b4-nh3', 'ko', '암모니아 합성은 가역 반응입니다: $\\ce{N2 + 3H2 <=> 2NH3}$.', '암모니아 합성은 가역 반응입니다: 질소와 수소가 가역적으로 반응하여 암모니아가 생성된다.'], + ['c1-perp', 'zh', '已知 $AB \\perp CD$，垂足为 O。', '已知 A B 垂直于 C D，垂足为 O。'], + ['c1-perp', 'en', 'Given that $AB \\perp CD$, the foot of the perpendicular is O.', 'Given that A B is perpendicular to C D, the foot of the perpendicular is O.'], + ['c1-perp', 'ja', '$AB \\perp CD$ のとき、垂線の足を O とします。', 'エービーとシーディーは垂直 のとき、垂線の足を O とします。'], + ['c1-perp', 'ko', '$AB \\perp CD$ 일 때, 수선의 발은 O 입니다.', '에이 비는 씨 디에 수직 일 때, 수선의 발은 O 입니다.'], + ['c2-parallel', 'zh', '如果 $l_1 \\parallel l_2$，那么同位角相等。', '如果 l1 平行于 l2，那么同位角相等。'], + ['c2-parallel', 'en', 'If $l_1 \\parallel l_2$, then the corresponding angles are equal.', 'If l one is parallel to l two, then the corresponding angles are equal.'], + ['c2-parallel', 'ja', '$l_1 \\parallel l_2$ ならば、同位角は等しい。', 'エルいちとエルには平行 ならば、同位角は等しい。'], + ['c2-parallel', 'ko', '$l_1 \\parallel l_2$ 이면 동위각은 같습니다.', '엘 아래첨자 일은 엘 아래첨자 이와 평행 이면 동위각은 같습니다.'], + ['c3-leq', 'zh', '题目给出的条件是 $a \\leq b$。', '题目给出的条件是 a 小于等于 b。'], + ['c3-leq', 'en', 'The problem gives the condition $a \\leq b$.', 'The problem gives the condition a is less than or equal to b.'], + ['c3-leq', 'ja', '問題の条件は $a \\leq b$ です。', '問題の条件は エー 小なりイコール ビー です。'], + ['c3-leq', 'ko', '문제의 조건은 $a \\leq b$ 입니다.', '문제의 조건은 에이는 비보다 작거나 같음 입니다.'], + ['c4-in', 'zh', '设元素 $x \\in A$，则 x 满足该条件。', '设元素 x 属于 A，则 x 满足该条件。'], + ['c4-in', 'en', 'Let the element $x \\in A$, so x satisfies the condition.', 'Let the element x is an element of A, so x satisfies the condition.'], + ['c4-in', 'ja', '要素 $x \\in A$ とすると、x はこの条件を満たします。', '要素 エックスはエーに属する とすると、x はこの条件を満たします。'], + ['c4-in', 'ko', '원소 $x \\in A$ 라고 하면, x는 이 조건을 만족합니다.', '원소 엑스는 에이의 원소 라고 하면, x는 이 조건을 만족합니다.'], + ['c5-subseteq', 'zh', '这两个集合满足 $A \\subseteq B$。', '这两个集合满足 A 包含于 B。'], + ['c5-subseteq', 'en', 'These two sets satisfy $A \\subseteq B$.', 'These two sets satisfy A is a subset of or equal to B.'], + ['c5-subseteq', 'ja', 'この二つの集合は $A \\subseteq B$ を満たします。', 'この二つの集合は エーはビーに含まれる を満たします。'], + ['c5-subseteq', 'ko', '두 집합은 $A \\subseteq B$ 를 만족합니다.', '두 집합은 에이는 비의 부분집합 를 만족합니다.'], + ['c6-angle', 'zh', '在这个直角三角形中，$\\angle ABC = 90^\\circ$。', '在这个直角三角形中，角A B C 等于 90度。'], + ['c6-angle', 'en', 'In this right triangle, $\\angle ABC = 90^\\circ$.', 'In this right triangle, angle A B C equals ninety degrees.'], + ['c6-angle', 'ja', 'この直角三角形では、$\\angle ABC = 90^\\circ$ です。', 'この直角三角形では、角エービーシー イコール きゅうじゅう度 です。'], + ['c6-angle', 'ko', '이 직각삼각형에서 $\\angle ABC = 90^\\circ$ 입니다.', '이 직각삼각형에서 각 에이 비 씨는 구십도 입니다.'], + ['c7-similar', 'zh', '由此可得 $\\triangle ABC \\sim \\triangle DEF$。', '由此可得 三角形A B C 相似于 三角形D E F。'], + ['c7-similar', 'en', 'It follows that $\\triangle ABC \\sim \\triangle DEF$.', 'It follows that triangle A B C is similar to triangle D E F.'], + ['c7-similar', 'ja', 'したがって、$\\triangle ABC \\sim \\triangle DEF$ です。', 'したがって、三角形エービーシーと三角形ディーイーエフは相似 です。'], + ['c7-similar', 'ko', '따라서 $\\triangle ABC \\sim \\triangle DEF$ 입니다.', '따라서 삼각형 에이 비 씨와 삼각형 디 E 에프는 닮음 입니다.'], + ])('reads the baseline formula %s in %s', (_, __, markdown, speech) => { + expect(toSpeechText(markdown)).toBe(speech) + }) + + it('reads a formula in the language of its sentence, and in the voice language without words around it', () => { + expect(toSpeechText('$x \\in A$')).toBe('x is an element of A') + expect(toSpeechText('$x \\in A$', { language: 'ja-JP' })).toBe('エックスはエーに属する') + expect(toSpeechText('$x \\in A$', { language: 'ko-KR' })).toBe('엑스는 에이의 원소') + expect(toSpeechText('$x \\in A$', { language: 'zh-CN' })).toBe('x 属于 A') + // Thai has no lexicon yet. English is used. + expect(toSpeechText('$x \\in A$', { language: 'th-TH' })).toBe('x is an element of A') + // Han text without kana is Japanese only when the voice is Japanese. + expect(toSpeechText('元素 $x \\in A$。', { language: 'ja-JP' })).toBe('元素 エックスはエーに属する。') + expect(toSpeechText('Then 设 $x \\in A$。')).toBe('Then 设 x 属于 A。') + expect(toSpeechText('First, $a \\leq b$. 然后 $x \\in A$。')).toBe('First, a is less than or equal to b. 然后 x 属于 A。') + }) + + it('reads the other TeX delimiters and TeX without delimiters', () => { + expect(toSpeechText('Inline \\(a_1 \\times b_1\\) here.')).toBe('Inline a one times b one here.') + expect(toSpeechText('结果：\\[\\frac{1}{2}\\]')).toBe('结果：2分之1') + expect(toSpeechText('集合 $A=\\{x \\mid x \\in \\mathbb{R}\\}$。')).toBe('集合 A 等于 满足x 属于 实数集的x的集合。') + expect(toSpeechText('设 \\alpha 为锐角，且 a \\leq b。')).toBe('设 阿尔法 为锐角，且 a 小于等于 b。') + expect(toSpeechText('Water is \\ce{H2O}, and E = mc^2 holds.')).toBe('Water is H two O, and E equals m c squared holds.') + expect(toSpeechText('Use `\\frac{1}{2}` here.')).toBe('Use one over two here.') + }) + + it('reads a script without braces in TeX without delimiters as the whole number', () => { + // TeX applies `^` to one character, but a reply that writes `2^10` without `$` means 2 to the 10th power. + expect(toSpeechText('2 的 10 次方也写成 2^10。')).toBe('2 的 10 次方也写成 2的10次方。') + expect(toSpeechText('也写成 $2^10$。')).toBe('也写成 2的1次方0。') + }) + + it('reads prices as words, and keeps paths and code that only look like TeX', () => { + expect(toSpeechText('It costs $5 and $10 today.')).toBe('It costs five dollars and ten dollars today.') + expect(toSpeechText('价格$5，另一个$10。')).toBe('价格5美元，另一个10美元。') + expect(toSpeechText('A ticket is $5, and the area is $x^2$.')).toBe('A ticket is five dollars, and the area is x squared.') + expect(toSpeechText('チケットは $5、面積は $x^2$ です。')).toBe('チケットは ごドル、面積は エックスの二乗 です。') + expect(toSpeechText('Open C:\\Users\\neko now.')).toBe('Open C:\\Users\\neko now.') + expect(toSpeechText('Before.\n\n```tex\n\\(x\\)\n```\n\nAfter.')).toBe('Before.\nAfter.') + }) + + it('keeps a marker inside a formula', () => { + const marker = '\uE000' + + expect(toSpeechText(`$a ${marker}\\leq b$ ok.`, { marker })).toBe(`a is less than or equal to b${marker} ok.`) + }) +}) diff --git a/packages/core-agent/src/voice/output/speech-transformation/index.ts b/packages/core-agent/src/voice/output/speech-transformation/index.ts new file mode 100644 index 0000000000..901197d659 --- /dev/null +++ b/packages/core-agent/src/voice/output/speech-transformation/index.ts @@ -0,0 +1,223 @@ +import type { Nodes, PhrasingContent, Root, RootContent } from 'mdast' +import type { Plugin } from 'unified' + +import type { SpeechPreprocessor } from './preprocessor' + +import remarkMath from 'remark-math' +import remarkParse from 'remark-parse' + +import { unified } from 'unified' + +import { detectSpeechLanguage, toSpeechLanguage } from './language' +import { normalizeMathDelimiters } from './markdown' +import { mathSpeechPreprocessor, moneySpeechPreprocessor } from './preprocessor' + +export interface SpeechTextOptions { + /** + * One character that the output keeps as often as the source has it, also inside unspoken nodes. + * The speech stream puts it where special tokens and flushes occur, then splits the output on it. + */ + marker?: string + /** + * BCP 47 code of the voice or the character, for example `zh-CN`. + * The words around a formula or a price select its language first. This code is used when they do not. + */ + language?: string + /** + * The spoken text before this markdown, when the markdown is one segment of a streamed reply, see ./segmenter. + * A formula without words around it, as a display formula that starts the segment, reads in its language. + */ + before?: string + /** + * The preprocessor plugins, in the order that their hooks run. + * The default is the money plugin, then the math plugin. A list replaces the default, so it includes them to keep them. + */ + preprocessors?: readonly SpeechPreprocessor[] +} + +/** The default preprocessor plugins: money first, so a price reads before the math plugin looks for bare TeX. */ +export function defaultSpeechPreprocessors(): SpeechPreprocessor[] { + return [moneySpeechPreprocessor(), mathSpeechPreprocessor()] +} + +/** A delimiter between two ASCII operands is an operator, for example `x**2` or `a*b*c`, not emphasis. */ +const asciiOperand = /^[A-Z0-9]$/i +/** A sentence ends at these characters. The words of the sentence select the language of its formulas. */ +const sentenceEnd = /[.!?。！？\n]/ + +/** + * Compiles markdown to the text that a TTS provider reads. + * + * Markup that has no sound is removed: emphasis delimiters, heading and list markers, link URLs, code + * blocks, and HTML. Blocks are joined with a line break, so the TTS chunker pauses between them. + * Symbols that the parser keeps as text, for example `2 * 3`, `2**10`, or `x ~= 3`, stay unchanged. + * + * TeX formulas in `$...$`, `$$...$$`, `\(...\)`, `\[...\]`, and known commands without delimiters, such as + * `\ce{H2O}`, are read as words. The words of the sentence around a formula select its language, and + * `options.language` is used when they do not. + */ +export const remarkSpeech: Plugin<[SpeechTextOptions?], Root, string> = function (options) { + this.compiler = (tree, file) => compile(tree as Root, String(file), options) +} + +/** + * Compiles markdown to the text that a TTS provider reads, with the preprocessor plugins of `options`. + * + * `\(...\)` and `\[...\]` become `$` delimiters first, because remark-math reads only `$`. Then each `prepare` hook + * rewrites the markdown, in plugin order. + */ +export function toSpeechText(markdown: string, options?: SpeechTextOptions): string { + const preprocessors = options?.preprocessors ?? defaultSpeechPreprocessors() + const prepared = preprocessors.reduce((text, preprocessor) => preprocessor.prepare?.(text) ?? text, normalizeMathDelimiters(markdown)) + return String(unified().use(remarkParse).use(remarkMath).use(remarkSpeech, { ...options, preprocessors }).processSync(prepared)) +} + +function compile(tree: Root, source: string, options: SpeechTextOptions | undefined): string { + // TODO: Read literal symbols outside formulas as words, for example `*` as "times" and `~=` as "approximately equals". + // A prose preprocessor plugin can read them with the words of the locale packs of `mathSpeech`, see ./preprocessor/math. + const marker = options?.marker + const fallback = toSpeechLanguage(options?.language) + const preprocessors = options?.preprocessors ?? defaultSpeechPreprocessors() + + function count(text: string): number { + return marker ? text.split(marker).length - 1 : 0 + } + + function range(node: Nodes): string { + const start = node.position?.start.offset + const end = node.position?.end.offset + return start === undefined || end === undefined ? '' : source.slice(start, end) + } + + /** Adds the markers that the source of a node has but its spoken text lost, for example in a link URL. */ + function keepMarkers(node: Nodes, spoken: string): string { + const missing = count(range(node)) - count(spoken) + return marker && missing > 0 ? spoken + marker.repeat(missing) : spoken + } + + function isOperator(node: Nodes): boolean { + const start = node.position?.start.offset + const end = node.position?.end.offset + if (start === undefined || end === undefined) + return false + + return asciiOperand.test(source[start - 1] ?? '') && asciiOperand.test(source[end] ?? '') + } + + /** The text of a node without its formulas. */ + function plain(node: Nodes): string { + if (node.type === 'text') + return node.value + return 'children' in node ? node.children.map(child => plain(child as Nodes)).join('') : '' + } + + /** + * Reads a formula with the first formula hook that returns words, or keeps the source when none does. + * Its markers move to its end, because words have no place for them. + */ + function speakFormula(source: string, context: string): string { + const tex = marker ? source.replaceAll(marker, '') : source + const language = detectSpeechLanguage(context, fallback) + let spoken = tex + for (const preprocessor of preprocessors) { + const words = preprocessor.formula?.(tex, { language }) + if (words !== undefined) { + spoken = words + break + } + } + return spoken + (marker ?? '').repeat(count(source)) + } + + /** Runs the prose hooks over a run of text. `context` is the words of its sentence. */ + function speakProse(text: string, kind: 'text' | 'code', context: string): string { + const language = detectSpeechLanguage(context, fallback) + return preprocessors.reduce((rest, preprocessor) => preprocessor.prose?.(rest, { language, fallback, kind, readFormula: speakFormula }) ?? rest, text) + } + + /** + * Inline math follows the Pandoc rule, so `$5 and $10` stays text: + * the content does not start or end with a space, and a digit does not follow the closing `$`. + */ + function isFormula(node: Extract): boolean { + if (range(node).startsWith('$$')) + return true + const after = sourceAfter(node) + return !/^\s|\s$/.test(node.value) && !/^\d/.test(after) + } + + function sourceAfter(node: Nodes): string { + const end = node.position?.end.offset + return end === undefined ? '' : source.slice(end, end + 1) + } + + /** The words of the sentence that holds a formula, in the paragraph `nodes`. Other sentences are the fallback. */ + function sentenceAround(nodes: readonly PhrasingContent[], index: number): string { + const before = nodes.slice(0, index).map(plain).join('') + const after = nodes.slice(index + 1).map(plain).join('') + const sentence = before.split(sentenceEnd).at(-1)! + after.split(sentenceEnd)[0]! + if (/\p{L}{2,}/u.test(sentence)) + return sentence + return /\p{L}{2,}/u.test(before + after) ? before + after : options?.before ?? '' + } + + function blocks(nodes: readonly RootContent[]): string { + return nodes.map((node, index) => block(node, nodes, index)).filter(Boolean).join('\n') + } + + function block(node: RootContent, siblings: readonly RootContent[], index: number): string { + switch (node.type) { + case 'paragraph': + case 'heading': + return phrasing(node.children) + case 'blockquote': + case 'list': + case 'listItem': + return blocks(node.children) + case 'math': { + // A display formula reads in the language of the text before it, or after it when it opens the reply. + // In a streamed segment, the text before it can be in an earlier segment. + const previous = siblings[index - 1] ? plain(siblings[index - 1]!) : options?.before ?? '' + const context = [previous, siblings[index + 1] ? plain(siblings[index + 1]!) : ''].join('\n') + return keepMarkers(node, speakFormula(node.value, context)) + } + // TODO: Tell the listener that the reply has code, instead of a skip without a sound. + default: + return keepMarkers(node, '') + } + } + + function phrasing(nodes: readonly PhrasingContent[]): string { + return nodes.map((node, index) => inline(node, nodes, index)).join('') + } + + function inline(node: PhrasingContent, siblings: readonly PhrasingContent[], index: number): string { + switch (node.type) { + case 'text': + // A voice can read `$5` in another language, so a price is read as words of its sentence. + return speakProse(node.value, 'text', sentenceAround(siblings, index) + node.value) + case 'inlineCode': + return speakProse(node.value, 'code', node.value) + case 'inlineMath': + return isFormula(node) ? keepMarkers(node, speakFormula(node.value, sentenceAround(siblings, index))) : keepMarkers(node, range(node)) + case 'emphasis': + case 'strong': + case 'delete': + // CommonMark reads `x**2 + y**2` as strong text, but the writer meant two powers. + // The source text, delimiters included, keeps the operators. + return isOperator(node) ? range(node) : phrasing(node.children) + case 'link': + case 'linkReference': + return keepMarkers(node, phrasing(node.children)) + case 'image': + case 'imageReference': + return keepMarkers(node, node.alt ?? '') + case 'break': + return '\n' + default: + return keepMarkers(node, '') + } + } + + return blocks(tree.children) +} diff --git a/packages/core-agent/src/voice/output/speech-transformation/language.test.ts b/packages/core-agent/src/voice/output/speech-transformation/language.test.ts new file mode 100644 index 0000000000..a2cf915fa7 --- /dev/null +++ b/packages/core-agent/src/voice/output/speech-transformation/language.test.ts @@ -0,0 +1,25 @@ +import { describe, expect, it } from 'vitest' + +import { detectSpeechLanguage, toSpeechLanguage } from './language' + +describe('language selection', () => { + it('maps BCP 47 codes to primary language subtags', () => { + expect(toSpeechLanguage('zh-CN')).toBe('zh') + expect(toSpeechLanguage('cmn-Hans-CN')).toBe('zh') + expect(toSpeechLanguage('yue-HK')).toBe('zh') + expect(toSpeechLanguage('en_US')).toBe('en') + expect(toSpeechLanguage('ko')).toBe('ko') + expect(toSpeechLanguage('th-TH')).toBe('th') + expect(toSpeechLanguage(undefined)).toBeUndefined() + }) + + it('reads the script of the words around a formula', () => { + expect(detectSpeechLanguage('バーゼル問題の結論は')).toBe('ja') + expect(detectSpeechLanguage('바젤 문제의 결론은')).toBe('ko') + expect(detectSpeechLanguage('巴塞尔问题的结论是')).toBe('zh') + expect(detectSpeechLanguage('巴塞尔问题', 'ja')).toBe('ja') + expect(detectSpeechLanguage('The Basel problem', 'zh')).toBe('en') + expect(detectSpeechLanguage(' , ', 'ko')).toBe('ko') + expect(detectSpeechLanguage('')).toBe('en') + }) +}) diff --git a/packages/core-agent/src/voice/output/speech-transformation/language.ts b/packages/core-agent/src/voice/output/speech-transformation/language.ts new file mode 100644 index 0000000000..b4c72604ef --- /dev/null +++ b/packages/core-agent/src/voice/output/speech-transformation/language.ts @@ -0,0 +1,33 @@ +/** + * The primary language subtag of a BCP 47 code: `zh` for `zh-CN`, `ko` for `ko_KR`. Chinese dialect codes are `zh`, + * because the readers have one Chinese pack. + */ +export function toSpeechLanguage(code: string | undefined): string | undefined { + if (code === undefined) + return undefined + const primary = code.toLowerCase().split(/[-_]/)[0] ?? '' + if (primary === 'cmn' || primary === 'yue') + return 'zh' + return primary || undefined +} + +const kana = /[\p{Script=Hiragana}\p{Script=Katakana}]/u +const hangul = /\p{Script=Hangul}/u +const han = /\p{Script=Han}/u +const latinWord = /[a-z]{2,}/i + +/** + * Selects the language of a formula or a price from the words around it, as a primary language subtag. + * Han characters without kana are Chinese, unless the voice is Japanese. + */ +export function detectSpeechLanguage(context: string, fallback?: string): string { + if (kana.test(context)) + return 'ja' + if (hangul.test(context)) + return 'ko' + if (han.test(context)) + return fallback === 'ja' ? 'ja' : 'zh' + if (latinWord.test(context)) + return 'en' + return fallback ?? 'en' +} diff --git a/packages/core-agent/src/voice/output/speech-transformation/markdown.test.ts b/packages/core-agent/src/voice/output/speech-transformation/markdown.test.ts new file mode 100644 index 0000000000..78ec7b761d --- /dev/null +++ b/packages/core-agent/src/voice/output/speech-transformation/markdown.test.ts @@ -0,0 +1,11 @@ +import { describe, expect, it } from 'vitest' + +import { normalizeMathDelimiters } from './markdown' + +describe('normalizeMathDelimiters', () => { + it('converts the parenthesis and bracket delimiters outside code', () => { + expect(normalizeMathDelimiters(String.raw`a \(x\) b \[y\]`)).toBe('a $x$ b $$y$$') + expect(normalizeMathDelimiters('`\\(x\\)` and \\(x\\)')).toBe('`\\(x\\)` and $x$') + expect(normalizeMathDelimiters('```\n\\[x\\]\n```')).toBe('```\n\\[x\\]\n```') + }) +}) diff --git a/packages/core-agent/src/voice/output/speech-transformation/markdown.ts b/packages/core-agent/src/voice/output/speech-transformation/markdown.ts new file mode 100644 index 0000000000..2611860764 --- /dev/null +++ b/packages/core-agent/src/voice/output/speech-transformation/markdown.ts @@ -0,0 +1,18 @@ +/** Code that markdown keeps as written: fenced blocks and inline code. Math delimiters inside it are not math. */ +export const codeSpans = /(```[\s\S]*?```|~~~[\s\S]*?~~~|`[^`\n]*`)/ + + +/** + * Converts `\(...\)` to `$...$` and `\[...\]` to `$$...$$`, outside code. + * CommonMark reads `\(` as an escaped parenthesis, so the backslash is gone after parsing. + */ +export function normalizeMathDelimiters(markdown: string): string { + return markdown + .split(codeSpans) + .map((part, index) => index % 2 === 1 + ? part + : part + .replace(/\\\[([\s\S]+?)\\\]/g, (_, tex: string) => `$$${tex}$$`) + .replace(/\\\(([\s\S]+?)\\\)/g, (_, tex: string) => `$${tex}$`)) + .join('') +} diff --git a/packages/core-agent/src/voice/output/speech-transformation/preprocessor/index.test.ts b/packages/core-agent/src/voice/output/speech-transformation/preprocessor/index.test.ts new file mode 100644 index 0000000000..a8db1c0b52 --- /dev/null +++ b/packages/core-agent/src/voice/output/speech-transformation/preprocessor/index.test.ts @@ -0,0 +1,54 @@ +import type { SpeechPreprocessor } from '.' + +import { describe, expect, it } from 'vitest' + +import { mathSpeechPreprocessor, moneySpeechPreprocessor } from '.' +import { defaultSpeechPreprocessors, toSpeechText } from '..' + +describe('speech preprocessor host', () => { + it('reads formulas and prices with the default plugins', () => { + expect(defaultSpeechPreprocessors().map(preprocessor => preprocessor.name)).toEqual(['money', 'math']) + expect(toSpeechText('Apples cost $3, and $x^2$ is a square.')).toBe('Apples cost three dollars, and x squared is a square.') + }) + + it('runs prose hooks in installation order, each on the output of the one before', () => { + const calls: string[] = [] + const tag = (name: string): SpeechPreprocessor => ({ + name, + prose: (text) => { + calls.push(name) + return `${text}[${name}]` + }, + }) + + expect(toSpeechText('hello', { preprocessors: [tag('a'), tag('b')] })).toBe('hello[a][b]') + expect(calls).toEqual(['a', 'b']) + }) + + it('uses the first formula hook that returns words', () => { + const quiet: SpeechPreprocessor = { name: 'quiet', formula: () => undefined } + const named: SpeechPreprocessor = { name: 'named', formula: (tex, { language }) => tex === 'E=mc^2' ? `${language}: mass energy` : undefined } + + expect(toSpeechText('See $E=mc^2$ here.', { preprocessors: [quiet, named, mathSpeechPreprocessor()] })).toBe('See en: mass energy here.') + expect(toSpeechText('See $x+1$ here.', { preprocessors: [quiet, named, mathSpeechPreprocessor()] })).toBe('See x plus one here.') + }) + + it('keeps the source of a formula that no plugin reads', () => { + expect(toSpeechText('Value $x+1$.', { preprocessors: [moneySpeechPreprocessor()] })).toBe('Value x+1.') + }) + + it('runs prepare hooks before parsing', () => { + const shout: SpeechPreprocessor = { name: 'shout', prepare: markdown => markdown.toUpperCase() } + expect(toSpeechText('*hi*', { preprocessors: [shout] })).toBe('HI') + }) + + it('reads bare TeX in prose through the formula hooks', () => { + const formula: SpeechPreprocessor = { name: 'formula', formula: () => 'water' } + expect(toSpeechText(String.raw`Drink \ce{H2O} daily.`, { preprocessors: [mathSpeechPreprocessor(), formula] })).toBe('Drink H two O daily.') + expect(toSpeechText(String.raw`Drink \ce{H2O} daily.`, { preprocessors: [formula, mathSpeechPreprocessor()] })).toBe('Drink water daily.') + }) + + it('leaves prices in inline code', () => { + expect(toSpeechText('Run `echo $5` now.')).toBe('Run echo $5 now.') + }) +}) diff --git a/packages/core-agent/src/voice/output/speech-transformation/preprocessor/index.ts b/packages/core-agent/src/voice/output/speech-transformation/preprocessor/index.ts new file mode 100644 index 0000000000..6368f2d99b --- /dev/null +++ b/packages/core-agent/src/voice/output/speech-transformation/preprocessor/index.ts @@ -0,0 +1,3 @@ +export { mathSpeech, mathSpeechPreprocessor, speakMath, toMathJson } from './math' +export { moneySpeechPreprocessor } from './money' +export type { SpeechFormulaContext, SpeechPreprocessor, SpeechProseContext } from './types' diff --git a/packages/core-agent/src/voice/output/speech-transformation/preprocessor/math/bare-latex.test.ts b/packages/core-agent/src/voice/output/speech-transformation/preprocessor/math/bare-latex.test.ts new file mode 100644 index 0000000000..215a5ba0b8 --- /dev/null +++ b/packages/core-agent/src/voice/output/speech-transformation/preprocessor/math/bare-latex.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, it } from 'vitest' + +import { findBareLatex } from './bare-latex' + +describe('findBareLatex', () => { + function spans(text: string) { + return findBareLatex(text).map(({ start, end }) => text.slice(start, end)) + } + + it('finds known commands and scripts with their operands', () => { + expect(spans(String.raw`设 \alpha 为锐角，且 a \leq b。`)).toEqual([String.raw`\alpha`, String.raw`a \leq b`]) + expect(spans(String.raw`The bare \ce{H2O} is water, and \frac{1}{2} is half.`)).toEqual([String.raw`\ce{H2O}`, String.raw`\frac{1}{2}`]) + expect(spans('E = mc^2 holds.')).toEqual(['E = mc^2']) + expect(spans('file_name and x_{i+1}')).toEqual(['x_{i+1}']) + }) + + it('ignores paths and unknown commands', () => { + expect(spans(String.raw`Open C:\Users\neko now.`)).toEqual([]) + expect(spans(String.raw`\newcommand is not read.`)).toEqual([]) + }) +}) diff --git a/packages/core-agent/src/voice/output/speech-transformation/preprocessor/math/bare-latex.ts b/packages/core-agent/src/voice/output/speech-transformation/preprocessor/math/bare-latex.ts new file mode 100644 index 0000000000..ef72897105 --- /dev/null +++ b/packages/core-agent/src/voice/output/speech-transformation/preprocessor/math/bare-latex.ts @@ -0,0 +1,89 @@ +import { mathSpeech } from './speech' + +/** Commands that start a formula in plain text. Unknown commands, such as a path `C:\Users`, do not. */ +const knownCommands = new Set([ + ...Object.keys(mathSpeech.locale('en').words), + ...Object.keys(mathSpeech.locale('en').relations), + ...Object.keys(mathSpeech.locale('en').operators), + // The commands of extensions, for example `\ce` of chemistry. + ...mathSpeech.rawCommands.map(command => `\\${command}`), + ...['frac', 'dfrac', 'tfrac', 'sqrt', 'text', 'mathrm', 'mathbb', 'overline', 'vec', 'hat', 'left', 'right', 'sum', 'prod', 'int', 'binom', 'overset', 'prime', 'circ'].map(name => `\\${name}`), +].filter(key => key.startsWith('\\'))) + +/** One symbol that can continue a formula: an operator, a digit, an uppercase name, or one lowercase letter. */ +const mathToken = /^(?:\s+|\d+(?:\.\d+)?|[A-Z]+|[a-z](?![a-z])|[=+\-*/<>()|'!^_[\]∞⊥∥≤≥≠≈∈∉⊆⊂∠△°±×÷→])/ +const trigger = /\\([a-z]+)|[a-z0-9)]\^[{\w]|_\{/gi + +function readBalanced(text: string, start: number): number { + let depth = 0 + for (let index = start; index < text.length; index++) { + if (text[index] === '\\') { + index++ + continue + } + if (text[index] === '{') + depth++ + if (text[index] === '}' && --depth === 0) + return index + 1 + } + return text.length +} + +/** Finds TeX that a reply writes without delimiters, for example `设 \alpha 为锐角` or `\ce{H2O}`. */ +export function findBareLatex(text: string): { start: number, end: number }[] { + const spans: { start: number, end: number }[] = [] + for (const match of text.matchAll(trigger)) { + if (match.index < (spans.at(-1)?.end ?? 0)) + continue + if (match[1] !== undefined && !knownCommands.has(`\\${match[1]}`)) + continue + + // Extend left over the operand before the trigger, for example `a` in `a \leq b`. + let start = match.index + while (start > 0) { + const before = text.slice(0, start) + const token = /(?:\s+|\d+(?:\.\d+)?|[A-Z]+|(?()|'!^_])$/.exec(before) + if (!token || before.length - token[0].length < (spans.at(-1)?.end ?? 0)) + break + start -= token[0].length + } + + let end = match.index + while (end < text.length) { + const rest = text.slice(end) + if (rest[0] === '\\') { + end += /^\\(?:[a-z]+|.)/i.exec(rest)![0].length + continue + } + if (rest[0] === '{') { + end = readBalanced(text, end) + continue + } + const token = mathToken.exec(rest) + if (!token) + break + end += token[0].length + } + + while (start < end && /\s/.test(text[start]!)) + start++ + while (end > start && /\s/.test(text[end - 1]!)) + end-- + if (end > start) + spans.push({ start, end }) + } + return spans +} + +/** + * Braces a multi-digit script in TeX that a reply writes without delimiters. + * TeX applies `^` and `_` to one character, but `2^10` in plain text means 2 to the 10th power. + * Formulas inside `$` keep the TeX rule, because the renderer shows them by that rule. + * + * @example + * braceBareScripts('2^10 + x_12') + * // => '2^{10} + x_{12}' + */ +export function braceBareScripts(tex: string): string { + return tex.replace(/([\^_])(\d{2,}(?:\.\d+)?|\d+\.\d+)/g, '$1{$2}') +} diff --git a/packages/core-agent/src/voice/output/speech-transformation/preprocessor/math/index.ts b/packages/core-agent/src/voice/output/speech-transformation/preprocessor/math/index.ts new file mode 100644 index 0000000000..066fac6a03 --- /dev/null +++ b/packages/core-agent/src/voice/output/speech-transformation/preprocessor/math/index.ts @@ -0,0 +1,33 @@ +import type { SpeechPreprocessor } from '../types' + +import { braceBareScripts, findBareLatex } from './bare-latex' +import { speakMath } from './speech' + +export { mathSpeech, speakMath, toMathJson } from './speech' + +/** + * Reads TeX formulas, chemistry in `\ce{}`, and units as words. + * + * Formulas go through `mathSpeech`, see ./speech. In prose, the plugin also finds TeX without delimiters, for example + * `\ce{H2O}`, and reads it as a formula. + */ +export function mathSpeechPreprocessor(): SpeechPreprocessor { + return { + name: 'math', + formula: (tex, { language }) => speakMath(tex, language), + prose: (text, { readFormula }) => { + const spans = findBareLatex(text) + if (spans.length === 0) + return text + + const context = spans.reduceRight((rest, { start, end }) => rest.slice(0, start) + rest.slice(end), text) + let result = '' + let last = 0 + for (const { start, end } of spans) { + result += text.slice(last, start) + readFormula(braceBareScripts(text.slice(start, end)), context) + last = end + } + return result + text.slice(last) + }, + } +} diff --git a/packages/core-agent/src/voice/output/speech-transformation/preprocessor/math/speech.test.ts b/packages/core-agent/src/voice/output/speech-transformation/preprocessor/math/speech.test.ts new file mode 100644 index 0000000000..ef0a5df7fb --- /dev/null +++ b/packages/core-agent/src/voice/output/speech-transformation/preprocessor/math/speech.test.ts @@ -0,0 +1,12 @@ +import { describe, expect, it } from 'vitest' + +import { speakMath } from './speech' + +// The readings of each language are tested next to their locale packs, in @proj-airi/speech-transformer-math and +// @proj-airi/speech-transformer-chem. This file tests only how AIRI uses them. +describe('speakMath', () => { + it('reads formulas and chemistry with the default instance', () => { + expect(speakMath(String.raw`x^2`, 'en')).toBe('x squared') + expect(speakMath(String.raw`\ce{2H2 + O2 -> 2H2O}`, 'zh')).toBe('氢气 和 氧气 反应生成 水') + }) +}) diff --git a/packages/core-agent/src/voice/output/speech-transformation/preprocessor/math/speech.ts b/packages/core-agent/src/voice/output/speech-transformation/preprocessor/math/speech.ts new file mode 100644 index 0000000000..9c955a942f --- /dev/null +++ b/packages/core-agent/src/voice/output/speech-transformation/preprocessor/math/speech.ts @@ -0,0 +1,20 @@ +import type { MathJson } from '@proj-airi/mathjson' + +import { chemistry } from '@proj-airi/speech-transformer-chem' +import { MathSpeech } from '@proj-airi/speech-transformer-math' + +/** + * The formula reader of AIRI: the built-in locale packs of `@proj-airi/speech-transformer-math`, with chemistry. + * A contributor adds or changes a language with `mathSpeech.register`. + */ +export const mathSpeech = new MathSpeech({ extensions: [chemistry()] }) + +/** Reads TeX math source as words of one language. `\ce{}` is read as chemistry. See {@link MathSpeech.speak}. */ +export function speakMath(tex: string, language: string): string { + return mathSpeech.speak(tex, language) +} + +/** Lowers TeX math source into MathJSON, without reading it. See {@link MathSpeech.toMathJson}. */ +export function toMathJson(tex: string): MathJson { + return mathSpeech.toMathJson(tex) +} diff --git a/packages/core-agent/src/voice/output/speech-transformation/preprocessor/money/index.ts b/packages/core-agent/src/voice/output/speech-transformation/preprocessor/money/index.ts new file mode 100644 index 0000000000..20e7833b91 --- /dev/null +++ b/packages/core-agent/src/voice/output/speech-transformation/preprocessor/money/index.ts @@ -0,0 +1,19 @@ +import type { SpeechPreprocessor } from '../types' + +import { escapePrices, speakPrices } from './prices' + +export { escapePrices, speakPrices } from './prices' + +/** + * Reads dollar amounts in prose as words of their sentence, for example `$3` as "three dollars". + * + * Before parsing, it escapes the `$` of a price with the Pandoc rule, so `$5 and $10` is not a formula. + * Inline code keeps its prices as written. + */ +export function moneySpeechPreprocessor(): SpeechPreprocessor { + return { + name: 'money', + prepare: escapePrices, + prose: (text, { kind, language }) => kind === 'text' ? speakPrices(text, language) : text, + } +} diff --git a/packages/core-agent/src/voice/output/speech-transformation/preprocessor/money/locales.ts b/packages/core-agent/src/voice/output/speech-transformation/preprocessor/money/locales.ts new file mode 100644 index 0000000000..78fa09dad7 --- /dev/null +++ b/packages/core-agent/src/voice/output/speech-transformation/preprocessor/money/locales.ts @@ -0,0 +1,24 @@ +import { spellNumber } from '@proj-airi/speech-transformer-math' + +/** The words of money in one language. */ +export interface MoneyLocale { + /** Reads an amount of US dollars from its digits, for example `1200` or `19.99`. */ + dollars: (digits: string) => string +} + +/** + * The money words of each language, by primary language subtag. + * Chinese keeps the digits, because a Chinese voice reads them. The other languages spell them, so the voice does not + * read them in another language. + */ +const locales: Readonly> = { + en: { dollars: digits => `${spellNumber(digits, 'en')} ${digits === '1' ? 'dollar' : 'dollars'}` }, + ja: { dollars: digits => `${spellNumber(digits, 'ja')}ドル` }, + ko: { dollars: digits => `${spellNumber(digits, 'ko')} 달러` }, + zh: { dollars: digits => `${digits}美元` }, +} + +/** The money words of a language, or English for a language without them. */ +export function moneyLocale(language: string): MoneyLocale { + return locales[language] ?? locales.en! +} diff --git a/packages/core-agent/src/voice/output/speech-transformation/preprocessor/money/prices.test.ts b/packages/core-agent/src/voice/output/speech-transformation/preprocessor/money/prices.test.ts new file mode 100644 index 0000000000..161a6c738e --- /dev/null +++ b/packages/core-agent/src/voice/output/speech-transformation/preprocessor/money/prices.test.ts @@ -0,0 +1,22 @@ +import { describe, expect, it } from 'vitest' + +import { escapePrices, speakPrices } from './prices' + +describe('prices', () => { + it('escapes the dollar sign of a price, and keeps formulas', () => { + // remark-math paired `$5, and the area is $` as one formula before. + expect(escapePrices('A ticket is $5, and the area is $x^2$.')).toBe('A ticket is \\$5, and the area is $x^2$.') + expect(escapePrices('It costs $5 and $10 today.')).toBe('It costs \\$5 and \\$10 today.') + expect(escapePrices('$2 + 3$ and $5$ and $3\\times10^{8}$')).toBe('$2 + 3$ and $5$ and $3\\times10^{8}$') + expect(escapePrices('Use `$5` here.')).toBe('Use `$5` here.') + }) + + it.each([ + ['Apples cost $3 and $1.', 'en', 'Apples cost three dollars and one dollar.'], + ['家賃は月 $1,200 です。', 'ja', '家賃は月 せんにひゃくドル です。'], + ['사과는 $3, 바나나는 $5입니다.', 'ko', '사과는 삼 달러, 바나나는 오 달러입니다.'], + ['这本书售价 $19.99。', 'zh', '这本书售价 19.99美元。'], + ] as const)('reads the prices of %s as words', (text, language, speech) => { + expect(speakPrices(text, language)).toBe(speech) + }) +}) diff --git a/packages/core-agent/src/voice/output/speech-transformation/preprocessor/money/prices.ts b/packages/core-agent/src/voice/output/speech-transformation/preprocessor/money/prices.ts new file mode 100644 index 0000000000..479c27ed44 --- /dev/null +++ b/packages/core-agent/src/voice/output/speech-transformation/preprocessor/money/prices.ts @@ -0,0 +1,76 @@ +import { codeSpans } from '../../markdown' +import { moneyLocale } from './locales' + +/** A dollar amount after `$`, with optional grouping commas and cents, for example `$1,200` or `$19.99`. */ +const priceSource = String.raw`\$(\d{1,3}(?:,\d{3})+|\d+)(?:\.\d+)?` +/** A price ends at a space, punctuation, CJK text, or the end. `$5\%$` and `$5$` are formulas, not prices. */ +const priceEndSource = String.raw`(?=$|[\s,.;:!?)\]、，。！？）」』\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}])` +const price = new RegExp(`^${priceSource}${priceEndSource}`, 'u') +const prices = new RegExp(`${priceSource}${priceEndSource}`, 'gu') + +/** + * Returns true when the `$` after `from` closes an inline formula by the Pandoc rule: no space before it, and no digit + * after it. The search stops at a blank line, because a formula does not cross paragraphs. + */ +function closesFormula(text: string, from: number): boolean { + for (let index = from; index < text.length; index++) { + if (text[index] === '\n' && text[index + 1] === '\n') + return false + if (text[index] === '\\') { + index++ + continue + } + if (text[index] === '$') + return !/\s/.test(text[index - 1] ?? ' ') && !/\d/.test(text[index + 1] ?? '') + } + return false +} + +/** + * Escapes the `$` of a price, outside code, so the markdown parser keeps it as text. + * + * remark-math pairs any two `$`, so `$5, and the area is $x^2$` became one formula from `$5` to `$x`. A `$` that starts + * a price, and whose next `$` cannot close a formula, is a price. `$2 + 3$` stays a formula. + * + * @example + * escapePrices('A ticket is $5, and the area is $x^2$.') + * // => 'A ticket is \\$5, and the area is $x^2$.' + */ +export function escapePrices(markdown: string): string { + return markdown + .split(codeSpans) + .map((part, index) => { + if (index % 2 === 1) + return part + let result = '' + let last = 0 + for (let at = 0; at < part.length; at++) { + if (part[at] !== '$' || part[at - 1] === '\\') + continue + if (part[at + 1] === '$') { + // `$$` opens or closes display math. + at++ + continue + } + const match = price.exec(part.slice(at)) + if (!match || closesFormula(part, at + match[0].length)) + continue + result += `${part.slice(last, at)}\\` + last = at + } + return result + part.slice(last) + }) + .join('') +} + +/** + * Reads the dollar amounts of a text as words of one language. + * + * @example + * speakPrices('Apples cost $3.', 'en') + * // => 'Apples cost three dollars.' + */ +export function speakPrices(text: string, language: string): string { + const locale = moneyLocale(language) + return text.replace(prices, match => locale.dollars(match.slice(1).replaceAll(',', ''))) +} diff --git a/packages/core-agent/src/voice/output/speech-transformation/preprocessor/types.ts b/packages/core-agent/src/voice/output/speech-transformation/preprocessor/types.ts new file mode 100644 index 0000000000..e659acf4e6 --- /dev/null +++ b/packages/core-agent/src/voice/output/speech-transformation/preprocessor/types.ts @@ -0,0 +1,37 @@ +/** What a formula hook knows about the formula that it reads. */ +export interface SpeechFormulaContext { + /** The language of the formula as a primary language subtag, for example `zh`, selected from the words around it. */ + language: string +} + +/** What a prose hook knows about the text that it rewrites. */ +export interface SpeechProseContext { + /** The language of the sentence around the text, as a primary language subtag. */ + language: string + /** The language of the voice, used when the words do not select one. */ + fallback?: string + /** `text` for plain text, `code` for inline code. The text of inline code keeps its symbols. */ + kind: 'text' | 'code' + /** Reads a formula found in the text through the formula hooks. `context` is the words around it. */ + readFormula: (tex: string, context: string) => string +} + +/** + * A preprocessor plugin of `toSpeechText`. One plugin owns one kind of content, for example formulas or money. + * + * The host runs the hooks of all plugins in installation order: + * + * - `prepare` rewrites the markdown before it is parsed, for example to escape `$5` so it is not a formula. + * - `formula` reads one TeX formula. The first plugin that returns a string wins. + * - `prose` rewrites a run of text. Each plugin gets the output of the plugin before it. + * + * A plugin is trusted in-process code. A new reading or a special pronunciation is a new plugin, so the host and the + * other plugins stay unchanged. + */ +export interface SpeechPreprocessor { + /** A unique name, for logs and tests, for example `math`. */ + name: string + prepare?: (markdown: string) => string + formula?: (tex: string, context: SpeechFormulaContext) => string | undefined + prose?: (text: string, context: SpeechProseContext) => string +} diff --git a/packages/core-agent/src/voice/output/speech-transformation/segmenter.test.ts b/packages/core-agent/src/voice/output/speech-transformation/segmenter.test.ts new file mode 100644 index 0000000000..c649571909 --- /dev/null +++ b/packages/core-agent/src/voice/output/speech-transformation/segmenter.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, it } from 'vitest' + +import { MarkdownSegmenter } from './segmenter' + +/** Streams `text` one character at a time, as a model can, and returns every segment, the rest included. */ +function segmentsOf(text: string): string[] { + const segmenter = new MarkdownSegmenter() + const segments = [...text].flatMap(char => segmenter.push(char)) + return [...segments, ...segmenter.end()] +} + +describe('markdownSegmenter', () => { + it('ends a segment after each sentence, and keeps the text whole', () => { + const text = '第一句。第二句！First one. Second one? Last' + const segments = segmentsOf(text) + expect(segments).toEqual(['第一句。', '第二句！', 'First one. ', 'Second one? ', 'Last']) + expect(segments.join('')).toBe(text) + }) + + it('keeps a formula, inline code, and a link whole', () => { + expect(segmentsOf('Area: $x = 3. y$. Then')).toEqual(['Area: $x = 3. y$. ', 'Then']) + expect(segmentsOf('Run `a. b` now. Then')).toEqual(['Run `a. b` now. ', 'Then']) + expect(segmentsOf('See [a. b](https://x.y/a. b). Then')).toEqual(['See [a. b](https://x.y/a. b). ', 'Then']) + expect(segmentsOf('结果是 \\(x。y\\)。然后')).toEqual(['结果是 \\(x。y\\)。', '然后']) + }) + + it('keeps display math and a code block whole across lines', () => { + expect(segmentsOf('Before.\n$$\nx = 1.\n$$\nAfter.\nEnd')).toEqual(['Before.\n', '$$\nx = 1.\n$$\n', 'After.\n', 'End']) + expect(segmentsOf('Code:\n```js\na. b\n\nc. d\n```\nAfter. End')).toEqual(['Code:\n', '```js\na. b\n\nc. d\n```\n', 'After. ', 'End']) + }) + + it('does not open a formula at a price', () => { + expect(segmentsOf('It costs $5. Then $x$. End')).toEqual(['It costs $5. ', 'Then $x$. ', 'End']) + }) + + it('keeps emphasis whole, and reads an asterisk between spaces as an operator', () => { + expect(segmentsOf('**Bold. Still bold.** Then')).toEqual(['**Bold. Still bold.** ', 'Then']) + expect(segmentsOf('So 2 * 3. Then')).toEqual(['So 2 * 3. ', 'Then']) + }) + + it('keeps a table, an indented continuation, and a setext heading with their lines', () => { + expect(segmentsOf('Table:\n| a. | b |\n| - | - |\n| 1. | 2 |\nAfter.\nEnd')).toEqual(['Table:\n', '| a. | b |\n| - | - |\n| 1. | 2 |\n', 'After.\n', 'End']) + expect(segmentsOf('- Item one\n more\n- Item two\nEnd')).toEqual(['- Item one\n more\n', '- Item two\n', 'End']) + expect(segmentsOf('Title\n===\nBody')).toEqual(['Title\n===\n', 'Body']) + }) + + it('does not end a sentence at a list number or a decimal', () => { + expect(segmentsOf('1. First\n2. Pi is 3.14 here. End')).toEqual(['1. First\n', '2. Pi is 3.14 here. ', 'End']) + }) + + it('ends a segment at a flush only when nothing is open', () => { + const segmenter = new MarkdownSegmenter() + expect(segmenter.push('Half a sentence')).toEqual([]) + expect(segmenter.flush()).toEqual(['Half a sentence']) + expect(segmenter.push('In $x')).toEqual([]) + expect(segmenter.flush()).toEqual([]) + expect(segmenter.push('^2$')).toEqual([]) + expect(segmenter.flush()).toEqual(['In $x^2$']) + }) +}) diff --git a/packages/core-agent/src/voice/output/speech-transformation/segmenter.ts b/packages/core-agent/src/voice/output/speech-transformation/segmenter.ts new file mode 100644 index 0000000000..4f804e07a1 --- /dev/null +++ b/packages/core-agent/src/voice/output/speech-transformation/segmenter.ts @@ -0,0 +1,240 @@ +/** Sentence ends of CJK text. A segment can end right after them. */ +const cjkSentenceEnds = new Set(['。', '！', '？']) +/** Sentence ends of Latin text. A segment can end after them when a space follows. */ +const latinSentenceEnds = new Set(['.', '!', '?']) +/** Closing quotes and brackets that stay with the sentence before them, as in `他说：“好。”`. */ +const closers = new Set(['"', '\'', '”', '’', '」', '』', '）', ')', '】']) +/** + * A `$` that starts a price, as `$5` or `$1,200.50`, followed by an end of the amount. It does not open a formula. + * The same rule as the money plugin, see ./preprocessor/money/prices. + */ +const price = /^\$(?:\d{1,3}(?:,\d{3})+|\d+)(?:\.\d+)?(?=$|[\s,.;:!?)\]、，。！？）」』\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}])/u + +/** What is open at a position of the buffer. A segment can end only where nothing is open. */ +interface ScanState { + /** The fence of a code block, as ``` or ~~~, or undefined outside one. */ + fence?: string + /** The backtick run that opened inline code. */ + code?: string + /** The delimiter of an open formula: `$`, `$$`, `\(`, or `\[`. */ + math?: string + /** Open square brackets of links. */ + brackets: number + /** True between `](` and its `)`. */ + url: boolean + /** True inside `*emphasis*`. Operator asterisks, as `2 * 3`, do not count. */ + emphasis: boolean + /** True inside `**strong**`. `***` opens or closes both. */ + strong: boolean + /** True on the lines of a table. */ + table: boolean +} + +function isOpen(state: ScanState): boolean { + return state.fence !== undefined || state.code !== undefined || state.math !== undefined || state.brackets > 0 || state.url || state.emphasis || state.strong || state.table +} + +/** + * Splits streamed markdown into segments that parse alone as they would inside the whole reply. + * + * `toSpeechText` needs whole markdown constructs: a formula, a code block, a link, or a table that is cut in two reads + * as its source. The model streams text in pieces, so the segmenter holds the text after the last safe end. A segment + * ends after a sentence end or a line, when no code, formula, link, emphasis, or table is open. + * + * Markdown is not fully streamable. A construct that refers to later text, as a reference link definition, reads as + * text when its segment has ended. Model replies rarely use them. + * + * @example + * const segmenter = new MarkdownSegmenter() + * segmenter.push('The area is $x^') // => [] + * segmenter.push('2$. Next') // => ['The area is $x^2$. '] + * segmenter.end() // => ['Next'] + */ +export class MarkdownSegmenter { + private buffer = '' + + /** Adds streamed text. Returns the segments that ended. */ + push(text: string): string[] { + this.buffer += text + const end = this.safeEnd(false) + return end > 0 ? [this.take(end)] : [] + } + + /** + * Ends a segment at the end of the buffer when nothing is open there, for a flush of the producer. A sentence can end + * in the middle then, which is what a flush asks for. + */ + flush(): string[] { + const end = this.safeEnd(true) + return end > 0 ? [this.take(end)] : [] + } + + /** Returns the rest of the text. The model has ended, so an open construct stays as written. */ + end(): string[] { + const rest = this.take(this.buffer.length) + return rest ? [rest] : [] + } + + private take(end: number): string { + const segment = this.buffer.slice(0, end) + this.buffer = this.buffer.slice(end) + return segment + } + + /** + * Scans the buffer and returns the last index where a segment can end, or 0. `atEnd` also accepts the end of the + * buffer. + */ + private safeEnd(atEnd: boolean): number { + const text = this.buffer + const state: ScanState = { brackets: 0, url: false, emphasis: false, strong: false, table: false } + let last = 0 + let lineStart = true + + for (let index = 0; index < text.length; index++) { + const char = text[index]! + + if (lineStart) { + lineStart = false + const line = text.slice(index, text.indexOf('\n', index) === -1 ? undefined : text.indexOf('\n', index)) + const fence = /^ {0,3}(`{3,}|~{3,})/.exec(line) + if (fence && (state.fence === undefined || fence[1]!.startsWith(state.fence))) { + state.fence = state.fence === undefined ? fence[1] : undefined + index += line.length - 1 + continue + } + if (state.fence !== undefined) { + index += line.length - 1 + continue + } + state.table = /^ {0,3}\|/.test(line) + } + + if (char === '\n') { + lineStart = true + // Inline code and emphasis do not cross a blank line. + if (text[index + 1] === '\n') { + state.code = undefined + state.emphasis = false + state.strong = false + if (state.math === '$') + state.math = undefined + } + const next = text[index + 1] + // The next line decides: an indented line can continue a list item, and parsed alone it is a code block. A line + // of `=` makes the line before a heading. The last line waits for the next one. + // A table ends at a line that is not a row. + if (next !== undefined && next !== ' ' && next !== '\t' && next !== '=') { + if (state.table && next !== '|') + state.table = false + if (!isOpen(state)) + last = index + 1 + } + continue + } + + if (state.fence !== undefined) + continue + + if (char === '\\') { + const pair = text.slice(index, index + 2) + if (state.code === undefined && (pair === '\\(' || pair === '\\[') && state.math === undefined) + state.math = pair + else if (state.code === undefined && ((pair === '\\)' && state.math === '\\(') || (pair === '\\]' && state.math === '\\['))) + state.math = undefined + // An escaped character, as `\$` or `\*`, is text. + index++ + continue + } + + if (char === '`') { + let run = '`' + while (text[index + run.length] === '`') + run += '`' + if (state.math === undefined) { + if (state.code === undefined) + state.code = run + else if (state.code === run) + state.code = undefined + } + index += run.length - 1 + continue + } + + if (state.code !== undefined) + continue + + if (char === '$') { + if (text[index + 1] === '$') { + if (state.math === undefined) + state.math = '$$' + else if (state.math === '$$') + state.math = undefined + index++ + continue + } + if (state.math === '$') + state.math = undefined + // Pandoc rule: an opening `$` has no space after it. A price, as `$5 and`, is not a formula. + else if (state.math === undefined && text[index + 1] !== undefined && !/\s/.test(text[index + 1]!) && !price.test(text.slice(index))) + state.math = '$' + continue + } + + if (state.math !== undefined) + continue + + // The character that ends a sentence. Closing emphasis keeps the end before it, as in `**Done.** Next`. + let terminal = char + if (char === '[') + state.brackets++ + else if (char === ']' && state.brackets > 0) { + state.brackets-- + if (text[index + 1] === '(') { + state.url = true + index++ + } + } + else if (char === ')' && state.url) + state.url = false + else if (char === '*') { + let run = 1 + while (text[index + run] === '*') + run++ + // A run between two spaces is an operator, as `2 * 3`. + if (!(/\s/.test(text[index - 1] ?? ' ') && /\s/.test(text[index + run] ?? ' '))) { + if (run >= 2) + state.strong = !state.strong + if (run % 2 === 1) + state.emphasis = !state.emphasis + } + terminal = text[index - 1] ?? '' + index += run - 1 + } + + if (isOpen(state)) + continue + + // A sentence ends after CJK punctuation, or after Latin punctuation and a space. `1.` that starts a list item and + // `3.14` are not ends. + let end: number | undefined + if (cjkSentenceEnds.has(terminal)) + end = index + 1 + else if (latinSentenceEnds.has(terminal) && (text[index + 1] === ' ' || text[index + 1] === '\t') && !/^\s*\d+$/.test(text.slice(text.lastIndexOf('\n', index) + 1, index))) + end = index + 1 + if (end === undefined) + continue + while (closers.has(text[end] ?? '')) + end++ + while (text[end] === ' ' || text[end] === '\t') + end++ + // The end of the buffer cannot show a closing quote yet, so it waits for more text. + if (end < text.length) + last = end + } + + if (atEnd && !isOpen(state) && state.fence === undefined && text.length > 0) + return text.length + return last + } +} diff --git a/packages/electron-screen-capture/package.json b/packages/electron-screen-capture/package.json index 10b9bf57a2..6706bf3298 100644 --- a/packages/electron-screen-capture/package.json +++ b/packages/electron-screen-capture/package.json @@ -54,6 +54,6 @@ "@moeru/eventa": "1.0.2", "async-mutex": "0.5.0", "nanoid": "6.0.1", - "picomatch": "4.0.5" + "picomatch": "4.0.7" } } diff --git a/packages/mathjson/README.md b/packages/mathjson/README.md new file mode 100644 index 0000000000..1c2c4f2e30 --- /dev/null +++ b/packages/mathjson/README.md @@ -0,0 +1,30 @@ +# @proj-airi/mathjson + +Types and small helpers for the MathJSON of the speech transformers. The package has no dependency. + +## Use it for + +- The type of a lowered formula: `MathJson`, `MathJsonFunction`, and `MathJsonNumber`. +- Reading or building nodes: `headOf`, `argsOf`, `isFunction`, `isNumber`, `isSymbol`, `text`, `textOf`, and `nothing`. + +## Do not use it for + +- Parsing or reading formulas. Use `@proj-airi/speech-transformer-math`. +- Full MathJSON with evaluation. Use `@cortex-js/compute-engine`. + +## Shape + +The shape follows [MathJSON](https://mathlive.io/math-json/). A function is an array with its head first, a string is a +symbol, and a quoted string, for example `'kg'`, is text. The speech transformers add heads for reading, for example +`Apply`, `Bracket`, and `Chemistry`. + +```ts +const expression: MathJson = ['Add', 'x', { num: '1' }] +headOf(expression) // => 'Add' +``` + +## Checks + +```sh +pnpm -F @proj-airi/mathjson typecheck +``` diff --git a/packages/mathjson/package.json b/packages/mathjson/package.json new file mode 100644 index 0000000000..2e4afbabcb --- /dev/null +++ b/packages/mathjson/package.json @@ -0,0 +1,35 @@ +{ + "name": "@proj-airi/mathjson", + "type": "module", + "version": "0.12.0-beta.5", + "private": true, + "description": "MathJSON types and helpers, with the heads that speech readings add", + "author": { + "name": "Moeru AI Project AIRI Team", + "email": "airi@moeru.ai", + "url": "https://github.com/moeru-ai" + }, + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/moeru-ai/airi.git", + "directory": "packages/mathjson" + }, + "exports": { + ".": { + "types": "./dist/index.d.mts", + "default": "./dist/index.mjs" + } + }, + "main": "./dist/index.mjs", + "types": "./dist/index.d.mts", + "files": [ + "README.md", + "dist", + "package.json" + ], + "scripts": { + "build": "tsdown", + "typecheck": "tsc --noEmit" + } +} diff --git a/packages/mathjson/src/index.ts b/packages/mathjson/src/index.ts new file mode 100644 index 0000000000..36c7d104d1 --- /dev/null +++ b/packages/mathjson/src/index.ts @@ -0,0 +1,58 @@ +/** + * The MathJSON expressions of the speech pipeline. + * + * The format follows MathJSON: a number object, a symbol string, or an array whose first item is the head. Standard + * heads, for example `Divide`, `Power`, `Limit`, `D`, and `Interval`, keep their MathJSON meaning, so the formula speech + * eval can compare them with Compute Engine. Speech also needs what the writer wrote, so the pipeline adds heads such + * as `Percent`, `Group`, `Unit`, and `Chemistry`. See docs/ai/adr/2026-10-20-speech-text-preprocessor.md. + * + * Two representation choices differ from Compute Engine: + * - A symbol keeps its TeX token, for example `\pi` or `x`, so locale packs key their words by TeX. + * - A number keeps its digits as a string, for example `{ num: '3.14' }`, so `007` and `3.10` keep their digits. + */ +export type MathJson = MathJsonNumber | string | MathJsonFunction + +/** A number with the digits that the writer wrote. */ +export interface MathJsonNumber { + readonly num: string +} + +/** A function expression: the head, then its arguments. */ +export type MathJsonFunction = readonly [head: string, ...args: MathJson[]] + +/** The placeholder of an argument that the writer left out, for example the upper bound of `\sum_{i}`. */ +export const nothing = 'Nothing' + +/** Wraps literal text as a MathJSON string, which MathJSON quotes with apostrophes: `'card'`. */ +export function text(value: string): string { + return `'${value}'` +} + +export function isText(value: MathJson): value is string { + return typeof value === 'string' && value.length >= 2 && value.startsWith('\'') && value.endsWith('\'') +} + +/** Reads the content of a MathJSON string. */ +export function textOf(value: MathJson): string { + return isText(value) ? value.slice(1, -1) : '' +} + +export function isNumber(value: MathJson | undefined): value is MathJsonNumber { + return typeof value === 'object' && value !== null && !Array.isArray(value) && 'num' in value +} + +export function isFunction(value: MathJson | undefined, head?: string): value is MathJsonFunction { + return Array.isArray(value) && (head === undefined || value[0] === head) +} + +export function isSymbol(value: MathJson | undefined): value is string { + return typeof value === 'string' && !isText(value) && value !== nothing +} + +export function headOf(value: MathJson | undefined): string | undefined { + return isFunction(value) ? value[0] : undefined +} + +export function argsOf(value: MathJson): readonly MathJson[] { + return isFunction(value) ? value.slice(1) : [] +} diff --git a/packages/mathjson/tsconfig.json b/packages/mathjson/tsconfig.json new file mode 100644 index 0000000000..389367200d --- /dev/null +++ b/packages/mathjson/tsconfig.json @@ -0,0 +1,14 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "lib": [ + "DOM", + "DOM.Iterable", + "ESNext" + ], + "types": [] + }, + "include": [ + "./src/**/*" + ] +} diff --git a/packages/mathjson/tsdown.config.ts b/packages/mathjson/tsdown.config.ts new file mode 100644 index 0000000000..a51052c07e --- /dev/null +++ b/packages/mathjson/tsdown.config.ts @@ -0,0 +1,6 @@ +import { defineConfig } from 'tsdown' + +export default defineConfig({ + entry: ['src/index.ts'], + dts: true, +}) diff --git a/packages/pipelines-audio/src/processors/tts-chunker.test.ts b/packages/pipelines-audio/src/processors/tts-chunker.test.ts index c7900609e1..2f993e7db7 100644 --- a/packages/pipelines-audio/src/processors/tts-chunker.test.ts +++ b/packages/pipelines-audio/src/processors/tts-chunker.test.ts @@ -158,6 +158,20 @@ describe('tTS Chunker Logic Cleanup', () => { expect(chunks.join('')).toBe(input) }) + it('keeps a kept punctuation once, with its own sentence, at the length limit', async () => { + const chunks = await collectChunkText('直觉上，模型在学习：“处于位置 x下标t、时间为 t 时，应该往哪个方向、以多快的速度移动？”生成时，从噪声出发。') + + expect(chunks.join('').split('？')).toHaveLength(2) + expect(chunks.some(chunk => chunk.endsWith('方向、？'))).toBe(false) + }) + + it('does not send a chunk without sound, as a lone closing quote', async () => { + const chunks = await collectChunkText('以多快的速度移动？”\n生成时。') + + expect(chunks).not.toContain('”') + expect(chunks.every(chunk => /[\p{L}\p{N}]/u.test(chunk))).toBe(true) + }) + it('preserves emoji grapheme clusters while chunking', async () => { const input = '👩‍💻👍🏽!' diff --git a/packages/pipelines-audio/src/processors/tts-chunker.ts b/packages/pipelines-audio/src/processors/tts-chunker.ts index a55a304154..e4d92dbbcc 100644 --- a/packages/pipelines-audio/src/processors/tts-chunker.ts +++ b/packages/pipelines-audio/src/processors/tts-chunker.ts @@ -10,7 +10,11 @@ export const TTS_SPECIAL_TOKEN = '\u2063' const regexpAnySingleDigit = /\d/ -const keptPunctuations = new Set('?？!！') +/** + * A chunk without a letter, a digit, or an emoji, as a lone closing quote `”`, has no sound and is not a TTS request. + * Emoji stay, because some providers read them. + */ +const regexpSound = /[\p{L}\p{N}\p{Extended_Pictographic}]/u const hardPunctuations = new Set('.。?？!！…⋯～~\n\t\r') const softPunctuations = new Set(',，、–—:：;；《》「」') @@ -92,7 +96,6 @@ export async function* chunkTtsInput( const special = value === TTS_SPECIAL_TOKEN const hard = hardPunctuations.has(value) const soft = softPunctuations.has(value) - const kept = keptPunctuations.has(value) let next: IteratorResult | undefined let afterNext: IteratorResult | undefined @@ -142,13 +145,17 @@ export async function* chunkTtsInput( const words = [...segmenter.segment(buffer)].filter(w => w.isWordLike) if (chunkWordsCount > minimumWords && chunkWordsCount + words.length > maximumWords) { - const text = kept ? chunk.trim() + value : chunk.trim() - yield { - text, - words: chunkWordsCount, - reason: 'limit', + // The punctuation ends the buffer, not the chunk before it, so it stays with the buffer. Adding a `？` here too + // read it twice, as 方向、？ and then 移动？. + const text = chunk.trim() + if (regexpSound.test(text)) { + yield { + text, + words: chunkWordsCount, + reason: 'limit', + } + yieldCount++ } - yieldCount++ chunk = '' chunkWordsCount = 0 } @@ -172,12 +179,14 @@ export async function* chunkTtsInput( // own TTS request. A shorter opening clause stays in the chunk and joins the next one. else if (flush || hard || chunkWordsCount > maximumWords || (yieldCount < boost && chunkWordsCount >= minimumWords)) { const text = chunk.trim() - yield { - text, - words: chunkWordsCount, - reason: flush ? 'flush' : hard ? 'hard' : chunkWordsCount > maximumWords ? 'limit' : 'boost', + if (regexpSound.test(text)) { + yield { + text, + words: chunkWordsCount, + reason: flush ? 'flush' : hard ? 'hard' : chunkWordsCount > maximumWords ? 'limit' : 'boost', + } + yieldCount++ } - yieldCount++ chunk = '' chunkWordsCount = 0 } @@ -206,7 +215,7 @@ export async function* chunkTtsInput( current = next } - if (chunk.length > 0 || buffer.length > 0) { + if (regexpSound.test(chunk + buffer)) { const text = (chunk + buffer).trim() yield { text, diff --git a/packages/speech-transformer-chem/README.md b/packages/speech-transformer-chem/README.md new file mode 100644 index 0000000000..36dab58bb0 --- /dev/null +++ b/packages/speech-transformer-chem/README.md @@ -0,0 +1,47 @@ +# @proj-airi/speech-transformer-chem + +Reads mhchem `\ce{}` and `\pu{}` as chemistry for a listener, in Chinese, English, Japanese, and Korean. It is an +extension of `@proj-airi/speech-transformer-math`. + +## Use it for + +- Chemical formulas and equations in a reply that a TTS voice reads: `chemistry()`. +- New words of chemistry for a language: a chemistry locale (`ChemLocale`). + +## Do not use it for + +- Formulas without `\ce{}` or `\pu{}`. The math transformer reads them. +- Naming any compound. The names cover the formulas of school chemistry. Other formulas keep their symbols. + +## Usage + +```ts +import { chemistry } from '@proj-airi/speech-transformer-chem' +import { MathSpeech } from '@proj-airi/speech-transformer-math' + +const speech = new MathSpeech({ extensions: [chemistry()] }) +speech.speak(String.raw`\ce{2H2 + O2 -> 2H2O}`, 'zh') // => '氢气 和 氧气 反应生成 水' +``` + +Install `@proj-airi/speech-transformer-math` too. It is a peer dependency, so the reader and the extension share one +`MathSpeech`. + +A chemistry locale matches the math locale pack by its language. A language without one is read by the fallback +locale, English by default. + +- Chinese, Japanese, and Korean read the species of an equation by their names, as a textbook does. +- English reads the symbols, for example "H two O". +- The built-in locales are exported as `zhChem`, `enChem`, `jaChem`, and `koChem`, apart from the math packs. +- Chinese, Japanese, and Korean do not read coefficients. A substance has no count, so 2个氢气 is wrong, and 分子 or + 분자 is wrong for metals, ionic solids, and ions. + +## Tests + +The tests of each language are next to its locale, for example `src/locales/ja.test.ts`. + +## Checks + +```sh +pnpm -F @proj-airi/speech-transformer-chem typecheck +pnpm -F @proj-airi/speech-transformer-chem exec vitest run +``` diff --git a/packages/speech-transformer-chem/package.json b/packages/speech-transformer-chem/package.json new file mode 100644 index 0000000000..f6df5c8106 --- /dev/null +++ b/packages/speech-transformer-chem/package.json @@ -0,0 +1,45 @@ +{ + "name": "@proj-airi/speech-transformer-chem", + "type": "module", + "version": "0.12.0-beta.5", + "private": true, + "description": "Reads mhchem chemistry as words, as an extension of speech-transformer-math", + "author": { + "name": "Moeru AI Project AIRI Team", + "email": "airi@moeru.ai", + "url": "https://github.com/moeru-ai" + }, + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/moeru-ai/airi.git", + "directory": "packages/speech-transformer-chem" + }, + "exports": { + ".": { + "types": "./dist/index.d.mts", + "default": "./dist/index.mjs" + } + }, + "main": "./dist/index.mjs", + "types": "./dist/index.d.mts", + "files": [ + "README.md", + "dist", + "package.json" + ], + "scripts": { + "build": "tsdown", + "typecheck": "tsc --noEmit" + }, + "peerDependencies": { + "@proj-airi/speech-transformer-math": "workspace:^" + }, + "dependencies": { + "@proj-airi/mathjson": "workspace:^", + "mhchemparser": "catalog:" + }, + "devDependencies": { + "@proj-airi/speech-transformer-math": "workspace:^" + } +} diff --git a/packages/speech-transformer-chem/src/chem.test.ts b/packages/speech-transformer-chem/src/chem.test.ts new file mode 100644 index 0000000000..f002e93d65 --- /dev/null +++ b/packages/speech-transformer-chem/src/chem.test.ts @@ -0,0 +1,20 @@ +import { describe, expect, it } from 'vitest' + +import { lowerChemistry } from './chem' + +describe('lowerChemistry', () => { + it('rebuilds species from the TeX of mhchemParser', () => { + expect(lowerChemistry('2H2 + O2 -> 2H2O')).toEqual(['Chemistry', ['Reaction', '\'->\'', ['Side', ['Species', '\'H2\'', { num: '2' }, 'Nothing'], ['Species', '\'O2\'', 'Nothing', 'Nothing']], ['Side', ['Species', '\'H2O\'', { num: '2' }, 'Nothing']], 'Nothing']]) + expect(lowerChemistry('Ca(OH)2')).toEqual(['Chemistry', ['Side', ['Species', '\'Ca(OH)2\'', 'Nothing', 'Nothing']]]) + expect(lowerChemistry('SO4^2-')).toEqual(['Chemistry', ['Side', ['Species', '\'SO4^2-\'', 'Nothing', 'Nothing']]]) + }) + + it('keeps states, hydrates, gas, precipitates, and arrow conditions', () => { + expect(lowerChemistry('H2O(l)')).toEqual(['Chemistry', ['Side', ['Species', '\'H2O\'', 'Nothing', '\'l\'']]]) + expect(lowerChemistry('CuSO4*5H2O')).toEqual(['Chemistry', ['Side', ['Hydrate', ['Species', '\'CuSO4\'', 'Nothing', 'Nothing'], ['Species', '\'H2O\'', { num: '5' }, 'Nothing']]]]) + expect(lowerChemistry('AgCl v')).toEqual(['Chemistry', ['Side', ['Precipitate', ['Species', '\'AgCl\'', 'Nothing', 'Nothing']]]]) + expect(lowerChemistry('CaCO3 ->[\\Delta] CaO + CO2 ^')).toMatchObject(['Chemistry', ['Reaction', '\'->\'', expect.anything(), ['Side', expect.anything(), ['Gas', expect.anything()]], ['Heat']]]) + const [, reaction] = lowerChemistry('N2 + 3H2 <=> 2NH3') as [string, [string, string]] + expect(reaction[1]).toBe('\'<=>\'') + }) +}) diff --git a/packages/speech-transformer-chem/src/chem.ts b/packages/speech-transformer-chem/src/chem.ts new file mode 100644 index 0000000000..c1322207dc --- /dev/null +++ b/packages/speech-transformer-chem/src/chem.ts @@ -0,0 +1,287 @@ +import type { MathJson } from '@proj-airi/mathjson' +import type { MathLocale, TexNode } from '@proj-airi/speech-transformer-math' + +import type { ChemLocale, ChemSpecies } from './locale' + +import { argsOf, headOf, isFunction, isNumber, nothing, text, textOf } from '@proj-airi/mathjson' +import { parseMath } from '@proj-airi/speech-transformer-math' +import { mhchemParser } from 'mhchemparser' + +/** The mhchem arrows that mhchemParser writes, mapped to the three readings that the lexicons have. */ +const arrows: Readonly' | '<=>' | '<-'>> = { + '\\longrightarrow': '->', + '\\rightarrow': '->', + '\\xrightarrow': '->', + '\\longleftarrow': '<-', + '\\xleftarrow': '<-', + '\\longrightleftharpoons': '<=>', + '\\xrightleftharpoons': '<=>', + '\\rightleftharpoons': '<=>', + '\\longleftrightarrow': '<=>', + '\\xleftrightarrow': '<=>', + '\\longLeftrightharpoons': '<=>', + '\\longRightleftharpoons': '<=>', +} +const gasMarks = new Set(['\\uparrow', '↑']) +const precipitateMarks = new Set(['\\downarrow', '↓']) +const states = new Set(['s', 'l', 'g', 'aq']) +const heat = new Set(['\\Delta', 'Δ']) + +/** One species while its tokens are read. `core` is the formula without charge, for example `Ca(OH)2`. */ +interface Draft { + coefficient?: string + core: string + charge?: string + state?: string +} + +/** Reads the characters of a script, for example `2` of `_{2}` or `3+` of `^{3+}`. */ +function scriptText(nodes: readonly TexNode[] | undefined): string { + return (nodes ?? []).map(node => node.type === 'symbol' ? node.value : node.type === 'group' ? scriptText(node.children) : '').join('') +} + +/** Rebuilds the formula of tokens, for example `(OH)2` from a fence of `\mathrm{OH}` and its count. */ +function formulaText(nodes: readonly TexNode[]): string { + return nodes.map((node) => { + if (node.type === 'text') + return node.value + if (node.type === 'fence') + return `(${formulaText(node.children)})${scriptText(node.sub)}` + if (node.type === 'scripts') + return scriptText(node.sub) + return '' + }).join('') +} + +function species(draft: Draft): MathJson { + const formula = draft.charge ? `${draft.core}^${draft.charge}` : draft.core + return ['Species', text(formula), draft.coefficient ? { num: draft.coefficient } : nothing, draft.state ? text(draft.state) : nothing] +} + +/** + * Lowers the content of `\ce{}` or `\pu{}` into a `Chemistry` expression. + * + * mhchemParser turns the mhchem source into TeX, and KaTeX parses that TeX, so this module only rebuilds species from + * the tokens. The shape is: + * + * - `["Chemistry", ["Reaction", arrow, ["Side", ...terms], ["Side", ...terms], condition]]` for an equation. + * - `["Chemistry", ["Side", ...terms]]` for formulas alone. + * - A term is `["Species", formula, coefficient, state]`, or `["Hydrate", ...species]`, wrapped in `Gas` or + * `Precipitate` when mhchem marks it. + */ +export function lowerChemistry(source: string): MathJson { + let tex: string + try { + tex = mhchemParser.toTex(source, 'ce') + } + catch { + // mhchemParser rejects some input. The source is then read as written. + return ['Chemistry', ['Side', ['Text', text(source)]]] + } + + const tokens = parseMath(tex).flatMap(node => node.type === 'group' ? node.children : [node]) + const sides: MathJson[][] = [[]] + let arrow: '->' | '<=>' | '<-' | undefined + let condition: MathJson = nothing + let hydrate: MathJson[] = [] + let draft: Draft | undefined + let coefficient: string | undefined + + const endSpecies = () => { + if (draft) + hydrate.push(species(draft)) + draft = undefined + } + const endTerm = () => { + endSpecies() + if (hydrate.length === 1) + sides.at(-1)!.push(hydrate[0]!) + else if (hydrate.length > 1) + sides.at(-1)!.push(['Hydrate', ...hydrate]) + hydrate = [] + } + const mark = (head: 'Gas' | 'Precipitate') => { + endTerm() + const side = sides.at(-1)! + if (side.length > 0) + side.push([head, side.pop()!]) + } + const arrowOf = (token: TexNode): string | undefined => { + if (token.type === 'symbol' || token.type === 'text') + return token.value.trim() + const label = token.type === 'command' && token.name === 'xarrow' ? token.args[0]?.[0] : undefined + return label?.type === 'symbol' ? label.value : undefined + } + + for (const token of tokens) { + if (token.type === 'group' && token.children.length === 0) + continue + + // An arrow ends the left side. A labeled arrow carries its condition. + const arrowName = arrowOf(token) + if (arrowName !== undefined && arrows[arrowName] !== undefined) { + endTerm() + arrow = arrows[arrowName] + if (token.type === 'command') { + const label = token.args[1] ?? [] + const only = label.length === 1 ? label[0]! : undefined + const labelText = only?.type === 'text' || only?.type === 'symbol' ? only.value.trim() : undefined + if (labelText !== undefined && heat.has(labelText)) + condition = ['Heat'] + else if (label.length > 0) + condition = ['Text', text(formulaText(label) || scriptText(label))] + } + sides.push([]) + continue + } + + if (token.type === 'symbol' && token.value === '+') { + endTerm() + continue + } + if (token.type === 'symbol' && gasMarks.has(token.value)) { + mark('Gas') + continue + } + if (token.type === 'symbol' && precipitateMarks.has(token.value)) { + mark('Precipitate') + continue + } + // The dot of a hydrate, `CuSO4*5H2O`, starts the next formula of the same term. + if (token.type === 'group' && token.children.some(child => child.type === 'symbol' && child.value === '\\cdot')) { + endSpecies() + continue + } + if (token.type === 'symbol' && /^\d+(?:\/\d+)?$/.test(token.value) && !draft) { + coefficient = token.value + continue + } + // A state follows its formula in parentheses, as `(l)` of `H2O(l)`. + const only = token.type === 'fence' && token.children.length === 1 ? token.children[0]! : undefined + if (draft && only?.type === 'text' && states.has(only.value)) { + draft.state = only.value + continue + } + if (token.type === 'scripts' && draft) { + if (token.sub?.length) + draft.core += scriptText(token.sub) + if (token.sup?.length) + draft.charge = scriptText(token.sup) + continue + } + if (token.type === 'text' || token.type === 'fence') { + draft ??= { core: '', coefficient } + coefficient = undefined + draft.core += formulaText([token]) + } + } + endTerm() + + if (arrow === undefined) + return ['Chemistry', ['Side', ...sides[0]!]] + return ['Chemistry', ['Reaction', text(arrow), ['Side', ...sides[0]!], ['Side', ...(sides[1] ?? [])], condition]] +} + +/** + * Reads a formula symbol by symbol: each element by its letters, each count as a number, and the parentheses of a + * group as words, for example `Ca(OH)2` as "C A open parenthesis O H close parenthesis two". + */ +function spellFormula(core: string, lexicon: MathLocale, chem: ChemLocale): string { + const words: string[] = [] + for (const match of core.matchAll(/([A-Z][a-z]?)(\d*)|\((?=[A-Z])|\)(\d*)/g)) { + const [token, element, count, groupCount] = match + if (element !== undefined) { + words.push(...[...element].map(letter => lexicon.letter(letter.toUpperCase()))) + if (count) + words.push(lexicon.number(count)) + } + else if (token === '(') { + words.push(chem.open) + } + else { + words.push(chem.close) + if (groupCount) + words.push(lexicon.number(groupCount)) + } + } + return words.join(' ') +} + +function chemSpecies(expression: MathJson): ChemSpecies { + const [formula = nothing, coefficient = nothing, state = nothing] = argsOf(expression) + const source = textOf(formula) + const [core = '', charge] = source.split('^') + return { formula: source, core, charge, coefficient: isNumber(coefficient) ? coefficient.num : undefined, state: textOf(state) || undefined } +} + +/** + * Reads a `Chemistry` expression. + * + * A formula alone keeps its symbols, because the text around it usually names it already, as in "the formula of + * sulfuric acid is H2SO4". An equation reads each species by its name when the lexicon has one, and adds the + * coefficient 1 where the language reads it. + */ +export function speakChemistry(expression: MathJson, lexicon: MathLocale, chem: ChemLocale): string { + function symbols(item: ChemSpecies): string { + return chem.spellsFormula ? spellFormula(item.core, lexicon, chem) : chem.formula({ ...item, formula: item.core }) + } + + function name(item: ChemSpecies): string { + return chem.names[item.formula] ?? symbols(item) + (item.charge ? ` ${chem.ion('', item.charge, true).trim()}` : '') + } + + function alone(item: ChemSpecies): string { + const formula = symbols(item) + return item.charge ? chem.ion(formula, item.charge, false) : formula + } + + function term(value: MathJson, inEquation: boolean): string { + const head = headOf(value) + if (head === 'Gas' || head === 'Precipitate') + return lexicon.join([term(argsOf(value)[0] ?? nothing, inEquation), head === 'Gas' ? chem.gas : chem.precipitate]) + if (head === 'Text') + return textOf(argsOf(value)[0] ?? nothing) + const parts = head === 'Hydrate' ? argsOf(value) : [value] + // A school hydrate in an equation has one name for the whole formula, as 五水合硫酸铜 for `CuSO4*5H2O`. A hydrate + // alone keeps its symbols: the formula speech eval heard 五水和物 as 無水和物, the opposite meaning. + if (inEquation && parts.length > 1) { + const species = parts.map(chemSpecies) + const whole = chem.names[species.map((item, index) => index === 0 ? item.formula : `${item.coefficient ?? ''}${item.formula}`).join('*')] + if (whole !== undefined) + return chem.term(whole, species[0]!.coefficient && lexicon.number(species[0]!.coefficient)) + } + const spoken = parts.map((part, index) => { + const item = chemSpecies(part) + const reading = inEquation ? name(item) : alone(item) + const coefficient = item.coefficient && lexicon.number(item.coefficient) + // The first coefficient of a hydrate belongs to the term, the others to their formula. + if (inEquation && index === 0) + return chem.term(reading, coefficient) + return coefficient ? lexicon.join([coefficient, reading]) : reading + }) + // Spaces keep the dot apart also in a compact language, where よんドットご would run together. + return spoken.join(` ${chem.hydrate} `) + } + + function side(value: MathJson, inEquation: boolean): string { + const terms = argsOf(value).map(item => term(item, inEquation)) + if (terms.length === 0) + return '' + // Three or more names join as a list, then "and" before the last: 锰酸钾、二氧化锰 和 氧气. A language that spells + // formulas joins them with plus, as "A plus B plus C", so it keeps the pairwise join. + return terms.length > 2 && !chem.spellsFormula ? chem.and(lexicon.list(terms.slice(0, -1)), terms.at(-1)!) : terms.reduce((left, right) => chem.and(left, right)) + } + + const content = argsOf(expression)[0] ?? nothing + if (!isFunction(content, 'Reaction')) + return side(content, false) + + const [arrow = nothing, left = nothing, right = nothing, condition = nothing] = argsOf(content) + const decomposition = textOf(arrow) === '->' && argsOf(left).length === 1 && argsOf(right).length >= 2 + const read = (decomposition ? chem.decompose : undefined) ?? chem.arrow[textOf(arrow)]! + const equation = read(side(left, true), side(right, true)) + if (condition === nothing) + return equation + const spokenCondition = headOf(condition) === 'Heat' ? chem.heat : textOf(argsOf(condition)[0] ?? nothing) + return chem.condition(spokenCondition, equation) +} diff --git a/packages/speech-transformer-chem/src/index.ts b/packages/speech-transformer-chem/src/index.ts new file mode 100644 index 0000000000..14a684c0d6 --- /dev/null +++ b/packages/speech-transformer-chem/src/index.ts @@ -0,0 +1,50 @@ +import type { MathSpeechExtension } from '@proj-airi/speech-transformer-math' + +import type { ChemLocale } from './locale' + +import { lowerChemistry, speakChemistry } from './chem' +import { en } from './locales/en' +import { ja } from './locales/ja' +import { ko } from './locales/ko' +import { zh } from './locales/zh' + +export { lowerChemistry, speakChemistry } from './chem' +export type { ChemLocale, ChemSpecies } from './locale' +// Named apart from the math packs, so one file can import both. +export { en as enChem, ja as jaChem, ko as koChem, zh as zhChem } + +/** The built-in chemistry locales: Chinese, English, Japanese, and Korean. */ +export const builtinChemLocales: readonly ChemLocale[] = [en, ja, ko, zh] + +export interface ChemistryOptions { + /** The chemistry locales, matched to a math locale pack by language. The default is {@link builtinChemLocales}. */ + locales?: readonly ChemLocale[] + /** The language of the locale that reads a language without one. The default is `en`. */ + fallbackLanguage?: string +} + +/** + * Reads mhchem `\ce{}` and `\pu{}` as chemistry, as an extension of `MathSpeech`. + * + * @example + * const speech = new MathSpeech({ extensions: [chemistry()] }) + * speech.speak(String.raw`\ce{2H2 + O2 -> 2H2O}`, 'zh') + * // => '氢气 和 氧气 反应生成 水' + */ +export function chemistry(options: ChemistryOptions = {}): MathSpeechExtension { + const locales = new Map((options.locales ?? builtinChemLocales).map(locale => [locale.language, locale])) + const fallback = options.fallbackLanguage ?? 'en' + return { + name: 'chemistry', + rawCommands: ['ce', 'pu'], + lower: (_, source) => lowerChemistry(source), + render: { + Chemistry: (expression, { lexicon }) => { + const locale = locales.get(lexicon.language) ?? locales.get(fallback) + if (!locale) + throw new Error(`No chemistry locale reads ${lexicon.language}, and the fallback ${fallback} has none.`) + return speakChemistry(expression, lexicon, locale) + }, + }, + } +} diff --git a/packages/speech-transformer-chem/src/locale.ts b/packages/speech-transformer-chem/src/locale.ts new file mode 100644 index 0000000000..384b0a93ea --- /dev/null +++ b/packages/speech-transformer-chem/src/locale.ts @@ -0,0 +1,49 @@ +/** Reads a chemical species. `coefficient` is set only inside an equation. */ +export interface ChemSpecies { + formula: string + /** Formula without charge, for example `Fe` for `Fe^{3+}`. */ + core: string + /** Signed charge, for example `3+` or `2-`. */ + charge?: string + coefficient?: string + state?: string +} + +/** + * The words of chemistry in one language. The words around them, as numbers and letters, come from the math locale + * pack of the same language. + */ +export interface ChemLocale { + /** The language of the math locale pack that this locale goes with, for example `zh`. */ + language: string + /** + * Names for whole formulas. An equation reads the name. A formula alone keeps its symbols, because the text names it. + */ + names: Readonly> + formula: (species: ChemSpecies) => string + /** Reads an ion. `inEquation` is true for a species of an equation, where more words follow it. */ + ion: (formula: string, charge: string, inEquation: boolean) => string + term: (name: string, coefficient: string | undefined) => string + and: (left: string, right: string) => string + arrow: Readonly string>> + /** + * Reads a decomposition: one reactant that yields two or more products, as `2KClO3 -> 2KCl + 3O2`. A textbook says + * 分解 there, not 反应. A locale without it reads the `->` arrow. + */ + decompose?: (reactant: string, products: string) => string + condition: (condition: string, equation: string) => string + gas: string + precipitate: string + /** `\Delta` over an arrow. */ + heat: string + /** + * True when a formula is read symbol by symbol: letter names, spelled counts, and parentheses. + * False keeps the formula as written, for example `H2SO4`, because the voice reads it. + */ + spellsFormula: boolean + /** Words for the parentheses of a group, for example `(OH)2`. */ + open: string + close: string + /** The dot of a hydrate, for example `CuSO4*5H2O`. */ + hydrate: string +} diff --git a/packages/speech-transformer-chem/src/locales/en.test.ts b/packages/speech-transformer-chem/src/locales/en.test.ts new file mode 100644 index 0000000000..e324b5fddd --- /dev/null +++ b/packages/speech-transformer-chem/src/locales/en.test.ts @@ -0,0 +1,44 @@ +import { MathSpeech } from '@proj-airi/speech-transformer-math' +import { describe, expect, it } from 'vitest' + +import { chemistry } from '..' + +const speech = new MathSpeech({ extensions: [chemistry()] }) + +describe('speakMath', () => { + it.each([ + [String.raw`\ce{2H2 + O2 -> 2H2O}`, 'two H two plus O two yields two H two O'], + [String.raw`\ce{N2 + 3H2 <=> 2NH3}`, 'N two plus three H two is in equilibrium with two N H three'], + [String.raw`\ce{Fe^{3+}}`, 'F E three plus'], + [String.raw`\ce{CaCO3 ->[\Delta] CaO + CO2 ^}`, 'with heat C A C O three yields C A O plus C O two gas'], + ])('reads the chemistry %s', (tex, expected) => { + expect(speech.speak(tex, 'en')).toBe(expected) + }) +}) + +// The cases below come from the formula speech eval, where a StepFun voice read the rule output. +// Each one failed to reach the listener before, because the text left a choice to the voice or to the listener. +describe('speakMath readings that a voice cannot get wrong', () => { + it.each([ + [String.raw`\ce{Ca(OH)2 + CO2 -> CaCO3 + H2O}`, 'C A open parenthesis O H close parenthesis two plus C O two yields C A C O three plus H two O'], + [String.raw`\ce{CuSO4*5H2O}`, 'C U S O four dot five H two O'], + ])('reads the chemistry %s symbol by symbol where it has no name', (tex, expected) => { + expect(speech.speak(tex, 'en')).toBe(expected) + }) +}) + +describe('chemistry readings', () => { + it.each([ + [String.raw`\ce{AgNO3 + NaCl -> AgCl v + NaNO3}`, 'A G N O three plus N A C L yields A G C L precipitate plus N A N O three'], + [String.raw`\ce{SO4^2-}`, 'S O four two minus'], + [String.raw`\ce{H2O(l)}`, 'H two O'], + ])('reads %s in English', (tex, expected) => { + expect(speech.speak(tex, 'en')).toBe(expected) + }) +}) + +describe('mappings of the review', () => { + it('ends an ion of an equation with the word ion, so its plus is not the plus between species', () => { + expect(speech.speak(String.raw`\ce{Fe^3+ + 3OH- -> Fe(OH)3 v}`, 'en')).toBe('F E three plus ion plus three O H minus ion yields F E open parenthesis O H close parenthesis three precipitate') + }) +}) diff --git a/packages/speech-transformer-chem/src/locales/en.ts b/packages/speech-transformer-chem/src/locales/en.ts new file mode 100644 index 0000000000..d1a4fbe15c --- /dev/null +++ b/packages/speech-transformer-chem/src/locales/en.ts @@ -0,0 +1,29 @@ +import type { ChemLocale } from '../locale' + +import { spellNumber } from '@proj-airi/speech-transformer-math' + +/** The English words of chemistry. */ +export const en: ChemLocale = { + language: 'en', + // English reads the symbols of a formula, also inside an equation. + names: {}, + formula: species => species.formula, + // In an equation, the word ion ends the charge, so its plus is not heard as the plus between species: F E three plus + // ion plus O H minus ion. + ion: (formula, charge, inEquation) => `${formula} ${charge.slice(0, -1) ? spellNumber(charge.slice(0, -1), 'en') : ''} ${charge.endsWith('+') ? 'plus' : 'minus'}${inEquation ? ' ion' : ''}`.replace(/\s+/g, ' ').trim(), + term: (name, coefficient) => coefficient ? `${coefficient} ${name}` : name, + and: (l, r) => `${l} plus ${r}`, + arrow: { + '->': (r, p) => `${r} yields ${p}`, + '<=>': (r, p) => `${r} is in equilibrium with ${p}`, + '<-': (r, p) => `${p} yields ${r}`, + }, + condition: (c, e) => `with ${c} ${e}`, + gas: 'gas', + heat: 'heat', + precipitate: 'precipitate', + spellsFormula: true, + open: 'open parenthesis', + close: 'close parenthesis', + hydrate: 'dot', +} diff --git a/packages/speech-transformer-chem/src/locales/ja.test.ts b/packages/speech-transformer-chem/src/locales/ja.test.ts new file mode 100644 index 0000000000..cb2d29708d --- /dev/null +++ b/packages/speech-transformer-chem/src/locales/ja.test.ts @@ -0,0 +1,51 @@ +import { MathSpeech } from '@proj-airi/speech-transformer-math' +import { describe, expect, it } from 'vitest' + +import { chemistry } from '..' + +const speech = new MathSpeech({ extensions: [chemistry()] }) + +describe('speakMath', () => { + it.each([ + [String.raw`\ce{2H2 + O2 -> 2H2O}`, '水素と酸素が反応して 水ができる'], + [String.raw`\ce{N2 + 3H2 <=> 2NH3}`, '窒素と水素が可逆的に反応して アンモニアができる'], + [String.raw`\ce{Fe^{3+}}`, 'エフ イー さん価の陽イオン'], + [String.raw`\ce{CaCO3 ->[\Delta] CaO + CO2 ^}`, '加熱の条件で 炭酸カルシウムが分解して 酸化カルシウムと二酸化炭素気体ができる'], + ])('reads the chemistry %s', (tex, expected) => { + expect(speech.speak(tex, 'ja')).toBe(expected) + }) +}) + +// The cases below come from the formula speech eval, where a StepFun voice read the rule output. +// Each one failed to reach the listener before, because the text left a choice to the voice or to the listener. +describe('speakMath readings that a voice cannot get wrong', () => { + it.each([ + [String.raw`\ce{Ca(OH)2 + CO2 -> CaCO3 + H2O}`, '水酸化カルシウムと二酸化炭素が反応して 炭酸カルシウムと水ができる'], + ])('reads the chemistry %s symbol by symbol where it has no name', (tex, expected) => { + expect(speech.speak(tex, 'ja')).toBe(expected) + }) +}) + +describe('chemistry readings', () => { + it('reads school compounds of an equation by their names', () => { + expect(speech.speak(String.raw`\ce{Fe2O3 + 3CO -> 2Fe + 3CO2}`, 'ja')).toContain('酸化第二鉄') + }) +}) + +describe('chemistry of the review', () => { + // The formula speech eval heard the name of a hydrate alone wrongly in every language, as 無水和物 for 五水和物, the + // opposite meaning. Its symbols were heard right, so a hydrate alone keeps them. + it('reads a hydrate alone by its symbols, because its name was heard wrongly', () => { + expect(speech.speak(String.raw`\ce{CuSO4*5H2O}`, 'ja')).toBe('シー ユー エス オー よん ドット ごエイチ に オー') + }) + + it('reads one reactant that yields several products as a decomposition', () => { + expect(speech.speak(String.raw`\ce{2KClO3 -> 2KCl + 3O2}`, 'ja')).toBe('塩素酸カリウムが分解して 塩化カリウムと酸素ができる') + }) +}) + +describe('mappings of the review', () => { + it('names a charge with a word, because a voice reads (III) as letters', () => { + expect(speech.speak(String.raw`\ce{Fe^3+ + 3OH- -> Fe(OH)3 v}`, 'ja')).toBe('三価の鉄イオンと水酸化物イオンが反応して 水酸化第二鉄沈殿ができる') + }) +}) diff --git a/packages/speech-transformer-chem/src/locales/ja.ts b/packages/speech-transformer-chem/src/locales/ja.ts new file mode 100644 index 0000000000..0f5e8a2ee5 --- /dev/null +++ b/packages/speech-transformer-chem/src/locales/ja.ts @@ -0,0 +1,31 @@ +import type { ChemLocale } from '../locale' + +import { spellNumber } from '@proj-airi/speech-transformer-math' + +import { compoundNames } from './names-ja' + +/** The Japanese words of chemistry. */ +export const ja: ChemLocale = { + language: 'ja', + names: compoundNames, + formula: species => species.formula, + ion: (formula, charge) => `${formula} ${spellNumber(charge.slice(0, -1) || '1', 'ja')}価の${charge.endsWith('+') ? '陽' : '陰'}イオン`, + // A textbook reads an equation by its substances, as Chinese does. 分子 counts molecules, but metals, ionic solids, + // and ions are not molecules, so に分子のナトリウム is wrong. Coefficients are not read. + term: name => name, + and: (l, r) => `${l}と${r}`, + arrow: { + '->': (r, p) => `${r}が反応して ${p}ができる`, + '<=>': (r, p) => `${r}が可逆的に反応して ${p}ができる`, + '<-': (r, p) => `${p}が反応して ${r}ができる`, + }, + decompose: (r, p) => `${r}が分解して ${p}ができる`, + condition: (c, e) => `${c}の条件で ${e}`, + gas: '気体', + heat: '加熱', + precipitate: '沈殿', + spellsFormula: true, + open: 'かっこ', + close: 'かっことじ', + hydrate: 'ドット', +} diff --git a/packages/speech-transformer-chem/src/locales/ko.test.ts b/packages/speech-transformer-chem/src/locales/ko.test.ts new file mode 100644 index 0000000000..dc91b20243 --- /dev/null +++ b/packages/speech-transformer-chem/src/locales/ko.test.ts @@ -0,0 +1,39 @@ +import { MathSpeech } from '@proj-airi/speech-transformer-math' +import { describe, expect, it } from 'vitest' + +import { chemistry } from '..' + +const speech = new MathSpeech({ extensions: [chemistry()] }) + +describe('speakMath', () => { + it.each([ + [String.raw`\ce{2H2 + O2 -> 2H2O}`, '수소와 산소가 반응하여 물이 생성된다'], + [String.raw`\ce{N2 + 3H2 <=> 2NH3}`, '질소와 수소가 가역적으로 반응하여 암모니아가 생성된다'], + [String.raw`\ce{Fe^{3+}}`, '에프 E 삼가 양이온'], + [String.raw`\ce{CaCO3 ->[\Delta] CaO + CO2 ^}`, '가열 조건에서 탄산 칼슘이 분해되어 산화 칼슘과 이산화 탄소 기체가 생성된다'], + ])('reads the chemistry %s', (tex, expected) => { + expect(speech.speak(tex, 'ko')).toBe(expected) + }) +}) + +// The cases below come from the formula speech eval, where a StepFun voice read the rule output. +// Each one failed to reach the listener before, because the text left a choice to the voice or to the listener. +describe('speakMath readings that a voice cannot get wrong', () => { + it.each([ + [String.raw`\ce{Ca(OH)2 + CO2 -> CaCO3 + H2O}`, '수산화 칼슘과 이산화 탄소가 반응하여 탄산 칼슘과 물이 생성된다'], + ])('reads the chemistry %s symbol by symbol where it has no name', (tex, expected) => { + expect(speech.speak(tex, 'ko')).toBe(expected) + }) +}) + +describe('chemistry of the review', () => { + // The formula speech eval heard the name of a hydrate alone wrongly in every language, as 無水和物 for 五水和物, the + // opposite meaning. Its symbols were heard right, so a hydrate alone keeps them. + it('reads a hydrate alone by its symbols, because its name was heard wrongly', () => { + expect(speech.speak(String.raw`\ce{CuSO4*5H2O}`, 'ko')).toBe('씨 유 에스 오 사 점 오 에이치 이 오') + }) + + it('reads one reactant that yields several products as a decomposition', () => { + expect(speech.speak(String.raw`\ce{2KClO3 -> 2KCl + 3O2}`, 'ko')).toBe('염소산 칼륨이 분해되어 염화 칼륨과 산소가 생성된다') + }) +}) diff --git a/packages/speech-transformer-chem/src/locales/ko.ts b/packages/speech-transformer-chem/src/locales/ko.ts new file mode 100644 index 0000000000..7d48ec1da8 --- /dev/null +++ b/packages/speech-transformer-chem/src/locales/ko.ts @@ -0,0 +1,31 @@ +import type { ChemLocale } from '../locale' + +import { josa, spellNumber } from '@proj-airi/speech-transformer-math' + +import { compoundNames } from './names-ko' + +/** The Korean words of chemistry. */ +export const ko: ChemLocale = { + language: 'ko', + names: compoundNames, + formula: species => species.formula, + ion: (formula, charge) => `${formula} ${spellNumber(charge.slice(0, -1) || '1', 'ko')}가 ${charge.endsWith('+') ? '양' : '음'}이온`, + // A textbook reads an equation by its substances, as Chinese does. 분자 counts molecules, but metals, ionic solids, + // and ions are not molecules, so 나트륨 이분자 is wrong. Coefficients are not read. + term: name => name, + and: (l, r) => `${josa(l, '과', '와')} ${r}`, + arrow: { + '->': (r, p) => `${josa(r, '이', '가')} 반응하여 ${josa(p, '이', '가')} 생성된다`, + '<=>': (r, p) => `${josa(r, '이', '가')} 가역적으로 반응하여 ${josa(p, '이', '가')} 생성된다`, + '<-': (r, p) => `${josa(p, '이', '가')} 반응하여 ${josa(r, '이', '가')} 생성된다`, + }, + decompose: (r, p) => `${josa(r, '이', '가')} 분해되어 ${josa(p, '이', '가')} 생성된다`, + condition: (c, e) => `${c} 조건에서 ${e}`, + gas: '기체', + heat: '가열', + precipitate: '침전', + spellsFormula: true, + open: '괄호 열고', + close: '괄호 닫고', + hydrate: '점', +} diff --git a/packages/speech-transformer-chem/src/locales/names-ja.ts b/packages/speech-transformer-chem/src/locales/names-ja.ts new file mode 100644 index 0000000000..cfdfd0f965 --- /dev/null +++ b/packages/speech-transformer-chem/src/locales/names-ja.ts @@ -0,0 +1,134 @@ +/** + * Japanese names of the formulas of school chemistry, keyed as the chemistry reader writes a species: the formula, a + * charge after `^`, and the parts of a hydrate joined with `*`. An equation reads a species by this name. A formula + * outside the table keeps its symbols. + */ +export const compoundNames: Readonly> = { + 'H2': '水素', + 'O2': '酸素', + 'N2': '窒素', + 'Cl2': '塩素', + 'O3': 'オゾン', + 'I2': 'ヨウ素', + 'Br2': '臭素', + 'He': 'ヘリウム', + 'C': '炭素', + 'S': '硫黄', + 'P': 'リン', + 'Si': 'ケイ素', + 'Fe': '鉄', + 'Cu': '銅', + 'Zn': '亜鉛', + 'Mg': 'マグネシウム', + 'Al': 'アルミニウム', + 'Na': 'ナトリウム', + 'K': 'カリウム', + 'Ca': 'カルシウム', + 'Ag': '銀', + 'Hg': '水銀', + 'H2O': '水', + 'H2O2': '過酸化水素', + 'CO': '一酸化炭素', + 'CO2': '二酸化炭素', + 'SO2': '二酸化硫黄', + 'SO3': '三酸化硫黄', + 'NO': '一酸化窒素', + 'NO2': '二酸化窒素', + 'P2O5': '五酸化二リン', + 'SiO2': '二酸化ケイ素', + 'CaO': '酸化カルシウム', + 'MgO': '酸化マグネシウム', + 'Al2O3': '酸化アルミニウム', + 'Fe2O3': '酸化第二鉄', + 'Fe3O4': '四酸化三鉄', + 'CuO': '酸化銅', + 'MnO2': '二酸化マンガン', + 'Na2O': '酸化ナトリウム', + 'ZnO': '酸化亜鉛', + 'HCl': '塩化水素', + 'H2SO4': '硫酸', + 'HNO3': '硝酸', + 'H2CO3': '炭酸', + 'H3PO4': 'リン酸', + 'H2S': '硫化水素', + 'CH3COOH': '酢酸', + 'NaOH': '水酸化ナトリウム', + 'KOH': '水酸化カリウム', + 'Ca(OH)2': '水酸化カルシウム', + 'Ba(OH)2': '水酸化バリウム', + 'Mg(OH)2': '水酸化マグネシウム', + 'Al(OH)3': '水酸化アルミニウム', + 'Fe(OH)3': '水酸化第二鉄', + 'Cu(OH)2': '水酸化銅', + 'NH3': 'アンモニア', + 'NaCl': '塩化ナトリウム', + 'KCl': '塩化カリウム', + 'CaCl2': '塩化カルシウム', + 'MgCl2': '塩化マグネシウム', + 'FeCl3': '塩化第二鉄', + 'FeCl2': '塩化第一鉄', + 'CuCl2': '塩化銅', + 'AlCl3': '塩化アルミニウム', + 'ZnCl2': '塩化亜鉛', + 'BaCl2': '塩化バリウム', + 'AgCl': '塩化銀', + 'NH4Cl': '塩化アンモニウム', + 'Na2CO3': '炭酸ナトリウム', + 'NaHCO3': '炭酸水素ナトリウム', + 'CaCO3': '炭酸カルシウム', + 'K2CO3': '炭酸カリウム', + 'BaCO3': '炭酸バリウム', + 'Na2SO4': '硫酸ナトリウム', + 'CuSO4': '硫酸銅', + 'BaSO4': '硫酸バリウム', + 'ZnSO4': '硫酸亜鉛', + 'FeSO4': '硫酸第一鉄', + '(NH4)2SO4': '硫酸アンモニウム', + 'NaNO3': '硝酸ナトリウム', + 'KNO3': '硝酸カリウム', + 'AgNO3': '硝酸銀', + 'NH4NO3': '硝酸アンモニウム', + 'Cu(NO3)2': '硝酸銅', + 'KMnO4': '過マンガン酸カリウム', + 'K2MnO4': 'マンガン酸カリウム', + 'KClO3': '塩素酸カリウム', + 'KI': 'ヨウ化カリウム', + 'Na2S': '硫化ナトリウム', + 'CH4': 'メタン', + 'C2H6': 'エタン', + 'C2H4': 'エチレン', + 'C2H2': 'アセチレン', + 'C3H8': 'プロパン', + 'C6H6': 'ベンゼン', + 'CH3OH': 'メタノール', + 'C2H5OH': 'エタノール', + 'HCHO': 'ホルムアルデヒド', + 'C6H12O6': 'グルコース', + 'C12H22O11': 'スクロース', + 'CH3COOC2H5': '酢酸エチル', + 'CO(NH2)2': '尿素', + 'H^+': '水素イオン', + 'OH^-': '水酸化物イオン', + 'Na^+': 'ナトリウムイオン', + 'K^+': 'カリウムイオン', + 'Ca^2+': 'カルシウムイオン', + 'Mg^2+': 'マグネシウムイオン', + 'Al^3+': 'アルミニウムイオン', + // A voice reads (III) as letters, so the charge is a word: 三価の鉄イオン for 鉄(III)イオン. + 'Fe^2+': '二価の鉄イオン', + 'Fe^3+': '三価の鉄イオン', + 'Cu^2+': '銅イオン', + 'Zn^2+': '亜鉛イオン', + 'Ag^+': '銀イオン', + 'Ba^2+': 'バリウムイオン', + 'NH4^+': 'アンモニウムイオン', + 'Cl^-': '塩化物イオン', + 'SO4^2-': '硫酸イオン', + 'CO3^2-': '炭酸イオン', + 'NO3^-': '硝酸イオン', + 'HCO3^-': '炭酸水素イオン', + 'S^2-': '硫化物イオン', + 'MnO4^-': '過マンガン酸イオン', + 'CuSO4*5H2O': '硫酸銅五水和物', + 'Na2CO3*10H2O': '炭酸ナトリウム十水和物', +} diff --git a/packages/speech-transformer-chem/src/locales/names-ko.ts b/packages/speech-transformer-chem/src/locales/names-ko.ts new file mode 100644 index 0000000000..7c028a8bbf --- /dev/null +++ b/packages/speech-transformer-chem/src/locales/names-ko.ts @@ -0,0 +1,134 @@ +/** + * Korean names of the formulas of school chemistry, keyed as the chemistry reader writes a species: the formula, a + * charge after `^`, and the parts of a hydrate joined with `*`. An equation reads a species by this name. A formula + * outside the table keeps its symbols. + */ +export const compoundNames: Readonly> = { + 'H2': '수소', + 'O2': '산소', + 'N2': '질소', + 'Cl2': '염소', + 'O3': '오존', + 'I2': '아이오딘', + 'Br2': '브로민', + 'He': '헬륨', + 'C': '탄소', + 'S': '황', + 'P': '인', + 'Si': '규소', + 'Fe': '철', + 'Cu': '구리', + 'Zn': '아연', + 'Mg': '마그네슘', + 'Al': '알루미늄', + 'Na': '나트륨', + 'K': '칼륨', + 'Ca': '칼슘', + 'Ag': '은', + 'Hg': '수은', + 'H2O': '물', + 'H2O2': '과산화 수소', + 'CO': '일산화 탄소', + 'CO2': '이산화 탄소', + 'SO2': '이산화 황', + 'SO3': '삼산화 황', + 'NO': '일산화 질소', + 'NO2': '이산화 질소', + 'P2O5': '오산화 이인', + 'SiO2': '이산화 규소', + 'CaO': '산화 칼슘', + 'MgO': '산화 마그네슘', + 'Al2O3': '산화 알루미늄', + 'Fe2O3': '산화 제이철', + 'Fe3O4': '사산화 삼철', + 'CuO': '산화 구리', + 'MnO2': '이산화 망가니즈', + 'Na2O': '산화 나트륨', + 'ZnO': '산화 아연', + 'HCl': '염화 수소', + 'H2SO4': '황산', + 'HNO3': '질산', + 'H2CO3': '탄산', + 'H3PO4': '인산', + 'H2S': '황화 수소', + 'CH3COOH': '아세트산', + 'NaOH': '수산화 나트륨', + 'KOH': '수산화 칼륨', + 'Ca(OH)2': '수산화 칼슘', + 'Ba(OH)2': '수산화 바륨', + 'Mg(OH)2': '수산화 마그네슘', + 'Al(OH)3': '수산화 알루미늄', + 'Fe(OH)3': '수산화 제이철', + 'Cu(OH)2': '수산화 구리', + 'NH3': '암모니아', + 'NaCl': '염화 나트륨', + 'KCl': '염화 칼륨', + 'CaCl2': '염화 칼슘', + 'MgCl2': '염화 마그네슘', + 'FeCl3': '염화 제이철', + 'FeCl2': '염화 제일철', + 'CuCl2': '염화 구리', + 'AlCl3': '염화 알루미늄', + 'ZnCl2': '염화 아연', + 'BaCl2': '염화 바륨', + 'AgCl': '염화 은', + 'NH4Cl': '염화 암모늄', + 'Na2CO3': '탄산 나트륨', + 'NaHCO3': '탄산수소 나트륨', + 'CaCO3': '탄산 칼슘', + 'K2CO3': '탄산 칼륨', + 'BaCO3': '탄산 바륨', + 'Na2SO4': '황산 나트륨', + 'CuSO4': '황산 구리', + 'BaSO4': '황산 바륨', + 'ZnSO4': '황산 아연', + 'FeSO4': '황산 제일철', + '(NH4)2SO4': '황산 암모늄', + 'NaNO3': '질산 나트륨', + 'KNO3': '질산 칼륨', + 'AgNO3': '질산 은', + 'NH4NO3': '질산 암모늄', + 'Cu(NO3)2': '질산 구리', + 'KMnO4': '과망가니즈산 칼륨', + 'K2MnO4': '망가니즈산 칼륨', + 'KClO3': '염소산 칼륨', + 'KI': '아이오딘화 칼륨', + 'Na2S': '황화 나트륨', + 'CH4': '메테인', + 'C2H6': '에테인', + 'C2H4': '에틸렌', + 'C2H2': '아세틸렌', + 'C3H8': '프로페인', + 'C6H6': '벤젠', + 'CH3OH': '메탄올', + 'C2H5OH': '에탄올', + 'HCHO': '폼알데하이드', + 'C6H12O6': '포도당', + 'C12H22O11': '수크로스', + 'CH3COOC2H5': '아세트산 에틸', + 'CO(NH2)2': '요소', + 'H^+': '수소 이온', + 'OH^-': '수산화 이온', + 'Na^+': '나트륨 이온', + 'K^+': '칼륨 이온', + 'Ca^2+': '칼슘 이온', + 'Mg^2+': '마그네슘 이온', + 'Al^3+': '알루미늄 이온', + // A voice reads (III) as letters, so the charge is a word: 철 삼가 이온 for 철(III) 이온. + 'Fe^2+': '철 이가 이온', + 'Fe^3+': '철 삼가 이온', + 'Cu^2+': '구리 이온', + 'Zn^2+': '아연 이온', + 'Ag^+': '은 이온', + 'Ba^2+': '바륨 이온', + 'NH4^+': '암모늄 이온', + 'Cl^-': '염화 이온', + 'SO4^2-': '황산 이온', + 'CO3^2-': '탄산 이온', + 'NO3^-': '질산 이온', + 'HCO3^-': '탄산수소 이온', + 'S^2-': '황화 이온', + 'MnO4^-': '과망가니즈산 이온', + 'CuSO4*5H2O': '황산 구리 오수화물', + 'Na2CO3*10H2O': '탄산 나트륨 십수화물', +} diff --git a/packages/speech-transformer-chem/src/locales/names-zh.ts b/packages/speech-transformer-chem/src/locales/names-zh.ts new file mode 100644 index 0000000000..1770ff26c0 --- /dev/null +++ b/packages/speech-transformer-chem/src/locales/names-zh.ts @@ -0,0 +1,133 @@ +/** + * Mandarin Chinese names of the formulas of school chemistry, keyed as the chemistry reader writes a species: the formula, a + * charge after `^`, and the parts of a hydrate joined with `*`. An equation reads a species by this name. A formula + * outside the table keeps its symbols. + */ +export const compoundNames: Readonly> = { + 'H2': '氢气', + 'O2': '氧气', + 'N2': '氮气', + 'Cl2': '氯气', + 'O3': '臭氧', + 'I2': '碘', + 'Br2': '溴', + 'He': '氦气', + 'C': '碳', + 'S': '硫', + 'P': '磷', + 'Si': '硅', + 'Fe': '铁', + 'Cu': '铜', + 'Zn': '锌', + 'Mg': '镁', + 'Al': '铝', + 'Na': '钠', + 'K': '钾', + 'Ca': '钙', + 'Ag': '银', + 'Hg': '汞', + 'H2O': '水', + 'H2O2': '过氧化氢', + 'CO': '一氧化碳', + 'CO2': '二氧化碳', + 'SO2': '二氧化硫', + 'SO3': '三氧化硫', + 'NO': '一氧化氮', + 'NO2': '二氧化氮', + 'P2O5': '五氧化二磷', + 'SiO2': '二氧化硅', + 'CaO': '氧化钙', + 'MgO': '氧化镁', + 'Al2O3': '氧化铝', + 'Fe2O3': '氧化铁', + 'Fe3O4': '四氧化三铁', + 'CuO': '氧化铜', + 'MnO2': '二氧化锰', + 'Na2O': '氧化钠', + 'ZnO': '氧化锌', + 'HCl': '氯化氢', + 'H2SO4': '硫酸', + 'HNO3': '硝酸', + 'H2CO3': '碳酸', + 'H3PO4': '磷酸', + 'H2S': '硫化氢', + 'CH3COOH': '乙酸', + 'NaOH': '氢氧化钠', + 'KOH': '氢氧化钾', + 'Ca(OH)2': '氢氧化钙', + 'Ba(OH)2': '氢氧化钡', + 'Mg(OH)2': '氢氧化镁', + 'Al(OH)3': '氢氧化铝', + 'Fe(OH)3': '氢氧化铁', + 'Cu(OH)2': '氢氧化铜', + 'NH3': '氨气', + 'NaCl': '氯化钠', + 'KCl': '氯化钾', + 'CaCl2': '氯化钙', + 'MgCl2': '氯化镁', + 'FeCl3': '氯化铁', + 'FeCl2': '氯化亚铁', + 'CuCl2': '氯化铜', + 'AlCl3': '氯化铝', + 'ZnCl2': '氯化锌', + 'BaCl2': '氯化钡', + 'AgCl': '氯化银', + 'NH4Cl': '氯化铵', + 'Na2CO3': '碳酸钠', + 'NaHCO3': '碳酸氢钠', + 'CaCO3': '碳酸钙', + 'K2CO3': '碳酸钾', + 'BaCO3': '碳酸钡', + 'Na2SO4': '硫酸钠', + 'CuSO4': '硫酸铜', + 'BaSO4': '硫酸钡', + 'ZnSO4': '硫酸锌', + 'FeSO4': '硫酸亚铁', + '(NH4)2SO4': '硫酸铵', + 'NaNO3': '硝酸钠', + 'KNO3': '硝酸钾', + 'AgNO3': '硝酸银', + 'NH4NO3': '硝酸铵', + 'Cu(NO3)2': '硝酸铜', + 'KMnO4': '高锰酸钾', + 'K2MnO4': '锰酸钾', + 'KClO3': '氯酸钾', + 'KI': '碘化钾', + 'Na2S': '硫化钠', + 'CH4': '甲烷', + 'C2H6': '乙烷', + 'C2H4': '乙烯', + 'C2H2': '乙炔', + 'C3H8': '丙烷', + 'C6H6': '苯', + 'CH3OH': '甲醇', + 'C2H5OH': '乙醇', + 'HCHO': '甲醛', + 'C6H12O6': '葡萄糖', + 'C12H22O11': '蔗糖', + 'CH3COOC2H5': '乙酸乙酯', + 'CO(NH2)2': '尿素', + 'H^+': '氢离子', + 'OH^-': '氢氧根离子', + 'Na^+': '钠离子', + 'K^+': '钾离子', + 'Ca^2+': '钙离子', + 'Mg^2+': '镁离子', + 'Al^3+': '铝离子', + 'Fe^2+': '亚铁离子', + 'Fe^3+': '三价铁离子', + 'Cu^2+': '铜离子', + 'Zn^2+': '锌离子', + 'Ag^+': '银离子', + 'Ba^2+': '钡离子', + 'NH4^+': '铵根离子', + 'Cl^-': '氯离子', + 'SO4^2-': '硫酸根离子', + 'CO3^2-': '碳酸根离子', + 'NO3^-': '硝酸根离子', + 'HCO3^-': '碳酸氢根离子', + 'S^2-': '硫离子', + 'MnO4^-': '高锰酸根离子', + 'CuSO4*5H2O': '五水合硫酸铜', + 'Na2CO3*10H2O': '十水合碳酸钠', +} diff --git a/packages/speech-transformer-chem/src/locales/names.test.ts b/packages/speech-transformer-chem/src/locales/names.test.ts new file mode 100644 index 0000000000..544a43ed36 --- /dev/null +++ b/packages/speech-transformer-chem/src/locales/names.test.ts @@ -0,0 +1,12 @@ +import { describe, expect, it } from 'vitest' + +import { compoundNames as ja } from './names-ja' +import { compoundNames as ko } from './names-ko' +import { compoundNames as zh } from './names-zh' + +describe('compoundNames', () => { + it('names the same formulas in every language', () => { + expect(Object.keys(ja).sort()).toEqual(Object.keys(zh).sort()) + expect(Object.keys(ko).sort()).toEqual(Object.keys(zh).sort()) + }) +}) diff --git a/packages/speech-transformer-chem/src/locales/zh.test.ts b/packages/speech-transformer-chem/src/locales/zh.test.ts new file mode 100644 index 0000000000..c27b48d77e --- /dev/null +++ b/packages/speech-transformer-chem/src/locales/zh.test.ts @@ -0,0 +1,50 @@ +import { MathSpeech } from '@proj-airi/speech-transformer-math' +import { describe, expect, it } from 'vitest' + +import { chemistry } from '..' + +const speech = new MathSpeech({ extensions: [chemistry()] }) + +describe('speakMath', () => { + it.each([ + [String.raw`\ce{2H2 + O2 -> 2H2O}`, '氢气 和 氧气 反应生成 水'], + [String.raw`\ce{N2 + 3H2 <=> 2NH3}`, '氮气 和 氢气 可逆反应生成 氨气'], + [String.raw`\ce{Fe^{3+}}`, 'Fe 3价正离子'], + [String.raw`\ce{CaCO3 ->[\Delta] CaO + CO2 ^}`, '在加热条件下 碳酸钙 分解生成 氧化钙 和 二氧化碳气体'], + ])('reads the chemistry %s', (tex, expected) => { + expect(speech.speak(tex, 'zh')).toBe(expected) + }) +}) + +// The cases below come from the formula speech eval, where a StepFun voice read the rule output. +// Each one failed to reach the listener before, because the text left a choice to the voice or to the listener. +describe('speakMath readings that a voice cannot get wrong', () => { + it.each([ + [String.raw`\ce{Ca(OH)2 + CO2 -> CaCO3 + H2O}`, '氢氧化钙 和 二氧化碳 反应生成 碳酸钙 和 水'], + ])('reads the chemistry %s symbol by symbol where it has no name', (tex, expected) => { + expect(speech.speak(tex, 'zh')).toBe(expected) + }) +}) + +describe('chemistry readings', () => { + it('reads the names of species in an equation', () => { + expect(speech.speak(String.raw`\ce{Ca(OH)2 + CO2 -> CaCO3 + H2O}`, 'zh')).toBe('氢氧化钙 和 二氧化碳 反应生成 碳酸钙 和 水') + }) + + it('reads school compounds of an equation by their names', () => { + expect(speech.speak(String.raw`\ce{2KMnO4 -> K2MnO4 + MnO2 + O2}`, 'zh')).toBe('高锰酸钾 分解生成 锰酸钾、二氧化锰 和 氧气') + expect(speech.speak(String.raw`\ce{CuSO4*5H2O -> CuSO4 + 5H2O}`, 'zh')).toBe('五水合硫酸铜 分解生成 硫酸铜 和 水') + }) +}) + +describe('chemistry of the review', () => { + // The formula speech eval heard the name of a hydrate alone wrongly in every language, as 無水和物 for 五水和物, the + // opposite meaning. Its symbols were heard right, so a hydrate alone keeps them. + it('reads a hydrate alone by its symbols, because its name was heard wrongly', () => { + expect(speech.speak(String.raw`\ce{CuSO4*5H2O}`, 'zh')).toBe('CuSO4 点 5 H2O') + }) + + it('reads one reactant that yields several products as a decomposition', () => { + expect(speech.speak(String.raw`\ce{2KClO3 -> 2KCl + 3O2}`, 'zh')).toBe('氯酸钾 分解生成 氯化钾 和 氧气') + }) +}) diff --git a/packages/speech-transformer-chem/src/locales/zh.ts b/packages/speech-transformer-chem/src/locales/zh.ts new file mode 100644 index 0000000000..a34a5e7dcb --- /dev/null +++ b/packages/speech-transformer-chem/src/locales/zh.ts @@ -0,0 +1,29 @@ +import type { ChemLocale } from '../locale' + +import { compoundNames } from './names-zh' + +/** The Chinese words of chemistry. */ +export const zh: ChemLocale = { + language: 'zh', + names: compoundNames, + formula: species => species.formula, + ion: (formula, charge) => `${formula} ${charge.slice(0, -1) || '1'}价${charge.endsWith('+') ? '正' : '负'}离子`, + // A Chinese textbook reads an equation by its substances: 氢气和氧气反应生成水. A substance has no count, so 2个氢气 + // is wrong. Counting needs 分子 or 原子 for each species, which the names do not tell, so coefficients are not read. + term: name => name, + and: (l, r) => `${l} 和 ${r}`, + arrow: { + '->': (r, p) => `${r} 反应生成 ${p}`, + '<=>': (r, p) => `${r} 可逆反应生成 ${p}`, + '<-': (r, p) => `${p} 反应生成 ${r}`, + }, + decompose: (r, p) => `${r} 分解生成 ${p}`, + condition: (c, e) => `在${c}条件下 ${e}`, + gas: '气体', + heat: '加热', + precipitate: '沉淀', + spellsFormula: false, + open: '左括号', + close: '右括号', + hydrate: '点', +} diff --git a/packages/speech-transformer-chem/tsconfig.json b/packages/speech-transformer-chem/tsconfig.json new file mode 100644 index 0000000000..389367200d --- /dev/null +++ b/packages/speech-transformer-chem/tsconfig.json @@ -0,0 +1,14 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "lib": [ + "DOM", + "DOM.Iterable", + "ESNext" + ], + "types": [] + }, + "include": [ + "./src/**/*" + ] +} diff --git a/packages/speech-transformer-chem/tsdown.config.ts b/packages/speech-transformer-chem/tsdown.config.ts new file mode 100644 index 0000000000..a51052c07e --- /dev/null +++ b/packages/speech-transformer-chem/tsdown.config.ts @@ -0,0 +1,6 @@ +import { defineConfig } from 'tsdown' + +export default defineConfig({ + entry: ['src/index.ts'], + dts: true, +}) diff --git a/packages/speech-transformer-chem/vitest.config.ts b/packages/speech-transformer-chem/vitest.config.ts new file mode 100644 index 0000000000..ceafc24120 --- /dev/null +++ b/packages/speech-transformer-chem/vitest.config.ts @@ -0,0 +1,8 @@ +import { defineConfig } from 'vitest/config' + +export default defineConfig({ + test: { + environment: 'node', + include: ['src/**/*.test.ts'], + }, +}) diff --git a/packages/speech-transformer-math/README.md b/packages/speech-transformer-math/README.md new file mode 100644 index 0000000000..5ee4a45626 --- /dev/null +++ b/packages/speech-transformer-math/README.md @@ -0,0 +1,65 @@ +# @proj-airi/speech-transformer-math + +Reads TeX formulas as words for a listener, in Chinese, English, Japanese, and Korean. It is a rule system. It has no +speech pipeline, no model, and no network, so any project can use it. + +## Use it for + +- Text for a TTS voice from a TeX formula: `new MathSpeech().speak(tex, language)`. +- The meaning of a formula as MathJSON, without words: `speech.toMathJson(tex)`. +- A new language, or new words for a language: a locale pack (`MathLocale`). +- New notation, for example chemistry: an extension (`MathSpeechExtension`). + +## Do not use it for + +- Finding formulas in markdown or prose. `@proj-airi/core-agent` does that before it calls this package. +- Evaluating or simplifying formulas. +- mhchem chemistry alone. Add `@proj-airi/speech-transformer-chem` as an extension. + +## Usage + +```ts +import { MathSpeech } from '@proj-airi/speech-transformer-math' + +const speech = new MathSpeech() +speech.speak('x^2', 'en') // => 'x squared' +speech.speak('x^2', 'zh-CN') // => 'x的平方' +``` + +The language is a BCP 47 code. A language without a pack is read by the fallback pack, English by default. + +Each instance owns its packs and extensions. `register(pack)` adds a pack or replaces the pack of its language, and +returns a function that restores the earlier pack. + +## Layers + +1. `parseMath`: KaTeX parses the formula, and the parse tree becomes a small syntax tree. Raw commands of extensions + keep their source. +2. `lowerMath`: the syntax tree becomes MathJSON. This layer holds every rule of meaning, for example a derivative or + an interval. +3. Extensions lower their raw commands. +4. `renderMath`: MathJSON becomes the words of one language. This layer holds only the wording. + +## Extensions + +```ts +const shout: MathSpeechExtension = { + name: 'shout', + rawCommands: ['shout'], + lower: (_, source) => ['Shout', `'${source}'`], + render: { Shout: ([, value]) => String(value).slice(1, -1).toUpperCase() }, +} +new MathSpeech({ extensions: [shout] }).speak(String.raw`x = \shout{hi}`, 'en') // => 'x equals HI' +``` + +## Tests + +The tests of each language are next to its pack, for example `src/locales/ko.test.ts`. Tests of a layer are next to +the layer, for example `src/lower.test.ts`. + +## Checks + +```sh +pnpm -F @proj-airi/speech-transformer-math typecheck +pnpm -F @proj-airi/speech-transformer-math exec vitest run +``` diff --git a/packages/speech-transformer-math/package.json b/packages/speech-transformer-math/package.json new file mode 100644 index 0000000000..eaec958332 --- /dev/null +++ b/packages/speech-transformer-math/package.json @@ -0,0 +1,39 @@ +{ + "name": "@proj-airi/speech-transformer-math", + "type": "module", + "version": "0.12.0-beta.5", + "private": true, + "description": "Reads TeX formulas as words in Chinese, English, Japanese, and Korean, through MathJSON", + "author": { + "name": "Moeru AI Project AIRI Team", + "email": "airi@moeru.ai", + "url": "https://github.com/moeru-ai" + }, + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/moeru-ai/airi.git", + "directory": "packages/speech-transformer-math" + }, + "exports": { + ".": { + "types": "./dist/index.d.mts", + "default": "./dist/index.mjs" + } + }, + "main": "./dist/index.mjs", + "types": "./dist/index.d.mts", + "files": [ + "README.md", + "dist", + "package.json" + ], + "scripts": { + "build": "tsdown", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@proj-airi/mathjson": "workspace:^", + "katex": "catalog:" + } +} diff --git a/packages/speech-transformer-math/src/extension.ts b/packages/speech-transformer-math/src/extension.ts new file mode 100644 index 0000000000..a1ae144f90 --- /dev/null +++ b/packages/speech-transformer-math/src/extension.ts @@ -0,0 +1,44 @@ +import type { MathJson, MathJsonFunction } from '@proj-airi/mathjson' + +import type { MathLocale } from './locale' + +/** What a render handler gets besides its expression. */ +export interface RenderContext { + /** The locale pack that reads the formula. */ + lexicon: MathLocale + /** Reads a part of the expression with the same pack and handlers. */ + render: (expression: MathJson) => string +} + +/** Reads one MathJSON head of an extension as words. */ +export type RenderHandler = (expression: MathJsonFunction, context: RenderContext) => string + +/** Render handlers by MathJSON head, for example `Chemistry`. */ +export type RenderHandlers = Readonly> + +/** + * Adds notation that the core rules do not read, for example mhchem chemistry. + * + * An extension can work in two steps: + * + * 1. Lowering. KaTeX does not parse the commands in `rawCommands`, for example `\ce`. The parser keeps their source, + * and `lower` turns the source into MathJSON with the heads of the extension. + * 2. Rendering. `render` reads those heads with the words of a language. + * + * @example + * const shout: MathSpeechExtension = { + * name: 'shout', + * rawCommands: ['shout'], + * lower: (_, source) => ['Shout', `'${source}'`], + * render: { Shout: ([, value]) => String(value).toUpperCase() }, + * } + */ +export interface MathSpeechExtension { + /** A name for errors and debugging, for example `chemistry`. */ + name: string + /** TeX commands without the backslash whose argument is source for `lower`, for example `ce`. */ + rawCommands?: readonly string[] + /** Lowers the source of one raw command. `command` is one of `rawCommands`. */ + lower?: (command: string, source: string) => MathJson + render?: RenderHandlers +} diff --git a/packages/speech-transformer-math/src/heads.ts b/packages/speech-transformer-math/src/heads.ts new file mode 100644 index 0000000000..13d11a0275 --- /dev/null +++ b/packages/speech-transformer-math/src/heads.ts @@ -0,0 +1,67 @@ +/** + * The MathJSON head of each relation, keyed by the canonical TeX spelling of {@link canonicalSymbol}. + * The renderer maps the head back to the words of the relation. + */ +export const relationHeads: Readonly> = { + '=': 'Equal', + '\\neq': 'NotEqual', + '<': 'Less', + '>': 'Greater', + '\\ll': 'MuchLess', + '\\gg': 'MuchGreater', + '\\leq': 'LessEqual', + '\\geq': 'GreaterEqual', + '\\approx': 'Approx', + '\\equiv': 'Equivalent', + '\\perp': 'Perpendicular', + '\\parallel': 'Parallel', + '\\in': 'Element', + '\\notin': 'NotElement', + '\\ni': 'Contains', + '\\subseteq': 'SubsetEqual', + '\\subset': 'Subset', + '\\supseteq': 'SupersetEqual', + '\\supset': 'Superset', + '\\nsubseteq': 'NotSubsetEqual', + '\\nsupseteq': 'NotSupersetEqual', + '\\sim': 'Similar', + '\\cong': 'Congruent', + '\\to': 'To', + '\\mapsto': 'MapsTo', + '\\Rightarrow': 'Implies', + '\\Leftrightarrow': 'Iff', + '\\propto': 'Proportional', + '\\gtrsim': 'GreaterApprox', + '\\lesssim': 'LessApprox', + '\\mid': 'Divides', + '\\nmid': 'NotDivides', + '\\vdash': 'Proves', + '\\coloneqq': 'Assign', +} + +/** The MathJSON head of each binary operator, keyed by its canonical TeX spelling. */ +export const operatorHeads: Readonly> = { + '+': 'Add', + '-': 'Subtract', + '\\times': 'Multiply', + '\\cdot': 'Multiply', + // `a / b` and `a \div b` are read as a division, unlike `\frac`, which is `Divide`. + '\\div': 'InlineDivide', + '\\pm': 'PlusMinus', + '\\mp': 'MinusPlus', + '\\cup': 'Union', + '\\cap': 'Intersection', + '\\setminus': 'SetMinus', + '\\sqcup': 'DisjointUnion', + '\\triangle': 'SymmetricDifference', + // A ring between two operands composes functions, as in `f \\circ g`. As a superscript it is degrees, see lowerScripts. + '\\circ': 'Compose', +} + +/** Operators that bind tighter than the others, so `a + b \times c` is `a + (b c)`. */ + +/** + * The head of a raw command that no extension lowered yet: `["Raw", 'command', 'source']`, see ./extension. + * {@link MathSpeech} replaces it with the MathJSON of the extension that owns the command. + */ +export const rawHead = 'Raw' diff --git a/packages/speech-transformer-math/src/index.ts b/packages/speech-transformer-math/src/index.ts new file mode 100644 index 0000000000..d915220962 --- /dev/null +++ b/packages/speech-transformer-math/src/index.ts @@ -0,0 +1,17 @@ +export type { MathSpeechExtension, RenderContext, RenderHandler, RenderHandlers } from './extension' +export { operatorHeads, rawHead, relationHeads } from './heads' +export type { FractionPart, FractionShape, MathLanguage, MathLocale, Relation, UnitLocale } from './locale' +export { aliases, canonicalSymbol, greek, greekNames, joinCompact, joinSpaced, josa, letterName, symbolAliases, unitPower } from './locale' +export { en } from './locales/en' +export { ja } from './locales/ja' +export { ko } from './locales/ko' +export { zh } from './locales/zh' +export { lowerMath } from './lower' +export { spellNumber } from './numbers' +export { parseMath } from './parse' +export { renderMath } from './render' +export { builtinLocales, MathSpeech } from './speech' +export type { MathSpeechOptions } from './speech' +export { texCommands } from './tex' +export type { TexNode } from './tex' +export { speakUnit } from './units' diff --git a/packages/speech-transformer-math/src/locale.ts b/packages/speech-transformer-math/src/locale.ts new file mode 100644 index 0000000000..b48886d339 --- /dev/null +++ b/packages/speech-transformer-math/src/locale.ts @@ -0,0 +1,389 @@ +/** + * The language of a locale pack: a BCP 47 primary language subtag, for example `zh` or `ko`. + * The built-in packs are `zh`, `en`, `ja`, and `ko`. A registered pack adds its own, see ./locales. + */ +export type MathLanguage = string + +/** Builds a relation from its two spoken operands. Some languages put the words around both operands. */ +export type Relation = (left: string, right: string) => string + +/** + * How hard one part of a fraction is to hear. + * + * - `simple`: a number, a letter, or a product such as `2a`. + * - `scripted`: it has a power or a subscript, for example `\pi^2`. + * - `compound`: it has an operator, a relation, a parenthesized sum, or another fraction, for example `n-1`. + */ +export type FractionPart = 'simple' | 'scripted' | 'compound' + +/** What the reader knows about a fraction, so each lexicon can choose the reading that the listener parses. */ +export interface FractionShape { + numerator: FractionPart + denominator: FractionPart + /** + * True when an operator comes before the fraction and its denominator is not a number, as in `1+\frac{1}{x}`. + * A language that reads the denominator first then cannot show where the operand before it ends: いちプラスエックス分のいち. + */ + followsOperator: boolean +} + +/** + * The words of units in one language. ./units decides which symbols are units: the SI prefixes, the SI units, and the + * units accepted for use with SI. A locale only names them. + */ +export interface UnitLocale { + /** Words of the SI prefixes, keyed by symbol, for example `k` or `µ`. */ + prefixes: Readonly> + /** Words of the units that take a prefix, keyed by symbol, for example `m` or `Hz`. */ + units: Readonly> + /** Words of the units that do not take a prefix, for example `min` or `ha`. */ + plain: Readonly> + /** Words for whole symbols that the composition of prefix and unit reads badly, for example `kg` in Chinese. */ + overrides?: Readonly> + /** Joins the words of a product of units, as `kg m^2`. The default is a space. Chinese and Japanese write none. */ + joinFactors?: (words: readonly string[]) => string + /** Words of a unit after "per", where they differ from `plain`, as Japanese 時 in 毎時 for `km/h`. */ + denominators?: Readonly> + /** Optional: the singular of a unit word, after the number one, as "one millimeter", or after "per". */ + singular?: (words: string) => string + /** Adds the power of a unit, for example square meters or 平方米. */ + power: (words: string, power: '2' | '3') => string +} + +/** + * A locale pack: every word that the speech pipeline needs in one language. + * + * A pack is data and small functions. A contributor adds a language with one pack and registers it, see ./locales. + * `language` is the BCP 47 primary language subtag of the pack, for example `ko`. + */ +export interface MathLocale { + language: string + /** Joins spoken words. Chinese and Japanese words join without a space, other words with one space. */ + join: (parts: readonly string[]) => string + /** Reads a decimal number, for example `3.14`. Chinese keeps the digits, see ./numbers. */ + number: (digits: string) => string + /** + * Reads one Latin letter, for example a variable. Japanese and Korean use the letter name, such as エム or 엠, + * because a voice can read a bare letter in another language. + */ + letter: (letter: string) => string + /** Marks the case of a letter, when one formula has both cases of it, for example `a \in A` or `\Omega \neq \omega`. */ + letterCase: (spoken: string, upper: boolean) => string + /** Joins clauses that a comma separates, for example `f: X \to Y, x \mapsto y`. */ + clauses: (parts: readonly string[]) => string + /** Reads `cases`. A case without a condition is the last one, "otherwise". */ + cases: (items: readonly { value: string, condition?: string }[]) => string + /** Reads the rows of a matrix, or of a determinant when `determinant` is true. Each row lists its cells. */ + matrix: (rows: readonly (readonly string[])[], determinant: boolean) => string + /** + * Joins the items of a list, for example `r, s` or the conditions of `\mathbb{E}_{t \sim U, x \sim p}`. + * Chinese and Japanese use 、 here, because a full comma sounds like the end of a clause. + */ + list: (items: readonly string[]) => string + /** Words for single symbols and commands. A missing key is read by its name. */ + words: Readonly> + operators: Readonly> + /** The unary minus. */ + negative: string + /** The unary plus, for example in `+\infty`. */ + positive: string + relations: Readonly> + /** + * Reads a fraction. A part that is hard to hear makes the reading name the numerator and the denominator, so the + * listener knows where each one ends. Which parts are hard depends on the word order of the language. + */ + fraction: (numerator: string, denominator: string, shape: FractionShape) => string + /** + * Reads `\frac{d}{dx} f`, the derivative of `body` with respect to `variable`. `order` is the digits of a higher + * derivative, as `2` in `\frac{d^2 y}{dx^2}`. + */ + derivative: (variable: string, body: string, order?: string) => string + power: (base: string, exponent: string) => string + squared: (base: string) => string + cubed: (base: string) => string + degrees: (base: string) => string + prime: (base: string) => string + /** + * A superscript that is a mark, not a power, for example `\mathbb{N}^{+}` or `f^{(n)}`. + * `parenthesized` is true for `^{(n)}`, so the listener hears the parentheses of an order or a label. + */ + superscript: (base: string, mark: string, parenthesized: boolean) => string + /** + * Words of the marks that follow a symbol as a superscript, as star in `x^*` or dagger in `K^\dagger`. A mark is said + * right after its symbol, as a presenter says it, never as a superscript. + */ + marks: Readonly> + /** + * Reads a function call, for example `f(x)` or `H_k(M, N)`. + * `grouped` is true when the arguments are hard to hear without their parentheses: more than one argument, or one + * argument with an operator, as in `f(x+1)`. A simple call such as `f(x)` reads without them. + * `decorated` is true when the name has an accent, a subscript, or several letters, as `\\hat H(t)`, `q_m(t)`, or + * `\\mathcal O(n)`. A pack that + * reads a simple call as the product does, as Chinese H帽t, says the brackets then, so it is not heard as a product. + */ + apply: (name: string, args: readonly string[], grouped: boolean, decorated: boolean) => string + /** The last item of a list after an ellipsis, as `d` in `k = 1, \dots, d`: "up to d". */ + upTo: (item: string) => string + /** + * Reads the item before an ellipsis and the last item as one range, for example Japanese いちからエヌまで for + * `1, \dots, n`. The ellipsis has no word then. A pack without it reads the ellipsis word and {@link MathLocale.upTo}. + */ + range?: (from: string, to: string) => string + /** + * Reads a chain of relations, as `a < b < c` or `x_1 \leq \cdots \leq x_n`. `operands` has one more item than + * `relations`, and an ellipsis operand is undefined. A pack without it nests the relations, which reads well when a + * relation is a word between its operands, as English "is less than". + */ + chain?: (operands: readonly (string | undefined)[], relations: readonly Relation[]) => string + /** The `\cdot` that marks an empty argument slot, as in `p(\cdot)`. */ + placeholder: string + /** + * The times word after a spoken group, as `(1-t)\\epsilon`, when it differs from the operator word. Chinese says 乘以, + * because 差乘 after 1与t的差 is heard as 叉乘, the cross product. + */ + groupTimes?: string + /** A number under a letter, for example `x_0`, said without a subscript word: x zero. */ + numberIndex: (base: string, index: string) => string + /** A simple subscript, for example `x_t`. */ + index: (base: string, index: string) => string + /** A subscript with more than one symbol, for example `x_{i+1}`. */ + subscript: (base: string, subscript: string) => string + /** `compound` is true when the radicand has an operator, as `a^2+b^2`, so the reading can say where it starts. */ + root: (radicand: string, degree?: string, compound?: boolean) => string + /** The inverse of a function, as `f^{-1}` or `\\cos^{-1}`. It is not a power. */ + inverse: (name: string) => string + /** A condition inside an argument, as `P(A \\mid B)`: `event` given `condition`. */ + given: (event: string, condition: string) => string + /** `\\pmod{n}` alone. */ + modulo: (modulus: string) => string + /** `a \\equiv b \\pmod{n}`: a congruence. */ + congruent: (left: string, right: string, modulus: string) => string + /** An ordinal written with a superscript, as `n^{\\text{th}}`. */ + ordinal: (value: string) => string + /** `side` is set for a one-sided limit, for example `x \to 0^{+}`, which approaches from the right. */ + limit: (body: string, variable?: string, target?: string, condition?: string, side?: 'left' | 'right') => string + /** + * Reads an arrow between two objects that do not approach each other, for example the map `H_k(M) \to H_k(M, N)`. + * An arrow toward a number or a variable is a limit and reads with `relations['\\to']`, see ./lower. + */ + arrow: (from: string, to: string) => string + /** + * Reads a norm, for example `\|x\|`. `index` names the norm, as `2` in `\|x\|_2`. `squared` is true for `\|x\|^2`, + * so the reading says "the squared norm of x" instead of the ambiguous "the norm of x squared". + */ + norm: (inner: string, index: string | undefined, squared: boolean) => string + /** + * Reads a difference inside a norm, as `\\|a - b\\|`, so the listener hears where the norm ends. English says "the + * norm of the difference between a and b": in "the norm of a minus b", the minus can be outside the norm. + */ + normDifference?: (left: string, right: string) => string + /** + * Reads an operator of probability, for example `\mathbb{E}_{x \sim p}[f(x)]`. `condition` is its subscript: the + * variables or the distributions that it averages over. + */ + statistic: (kind: 'Expectation' | 'Probability' | 'Variance' | 'Covariance', body: string, condition?: string) => string + /** Reads a quantity after a leading `\sim`, as in `\sim 50`: about fifty. */ + about: (quantity: string) => string + /** Reads `x \sim p`: x is drawn from the distribution p. */ + distributed: (variable: string, distribution: string) => string + /** `\\lceil x \\rceil` and `\\lfloor x \\rfloor`. */ + ceil: (inner: string) => string + floor: (inner: string) => string + /** A pair or a tuple in parentheses, for example `(V, E)`. */ + tuple: (items: readonly string[]) => string + /** + * Optional: the square of a group as one phrase, as "the square of the quantity a plus b". Without it, the square comes + * after the group, where it can sound like the square of its last term. + */ + squareOf?: (group: string) => string + /** Optional: a squared absolute value, when the language reads it as one phrase, as "the squared absolute value of q". */ + absoluteSquared?: (inner: string) => string + /** Square brackets that are not an interval, for example the class `[M]`. */ + bracket: (inner: string) => string + /** Reads a function signature, for example `f: X \to Y`. */ + mapping: (name: string, from: string, to: string) => string + /** + * `kind` is the operator: `sum`, `prod`, or `int`. + * `improper` is set for an integral with an infinite bound. + */ + bigOperator: (kind: 'sum' | 'prod' | 'int' | 'union' | 'intersection', body: string, bounds: { variable?: string, from?: string, to?: string, condition?: string, improper?: boolean }) => string + absolute: (inner: string) => string + /** A parenthesized group that is the base of a power. */ + quantity: (inner: string) => string + /** + * Optional words for a group of two simple terms, for example Chinese a 与 b 的和 for `(a+b)`. A pack without them + * reads every group with {@link MathLocale.quantity}. With them, a product with a group also says its times word, as + * in n 乘 n 与 1 的和 for `n(n+1)`, because the group no longer has spoken brackets that end the first factor. + */ + groupOf?: { sum: (left: string, right: string) => string, difference: (left: string, right: string) => string } + set: (inner: string, condition?: string) => string + /** `closed` tells whether each bound belongs to the interval: `[` and `]` include it, `(` and `)` do not. */ + interval: (from: string, to: string, closed: { left: boolean, right: boolean }) => string + /** `universe` is the set that the complement is taken in, for example `U` in `\complement_U A`. */ + complement: (set: string, universe?: string) => string + powerSet: (set: string) => string + cardinality: (set: string) => string + percent: (number: string) => string + accent: Readonly string>> + /** A segment of geometry, for example `\overline{AB}`. */ + segment: (name: string) => string + binomial: (n: string, k: string) => string + units: UnitLocale + /** + * Reads a unit with a denominator, for example `m/s^2`. The words come from `units`. + * `power` is the exponent of the denominator unit, for example `2` in `s^2`. + */ + unitRatio: (numerator: string, denominator: string, power?: string) => string + /** Reads a temperature in degrees Celsius, for example `25^\circ\mathrm{C}`. */ + celsius: (value: string) => string + numberSets: Readonly> +} + +const alphanumeric = /[a-z0-9]/i + +/** Joins words without a space, except between two Latin letters or digits, so `AB` and `CD` stay apart. */ +export function joinCompact(parts: readonly string[]): string { + let result = '' + for (const part of parts.map(part => part.trim()).filter(Boolean)) { + const last = result.at(-1) ?? '' + if (result && alphanumeric.test(last) && alphanumeric.test(part[0]!)) + result += ' ' + result += part + } + return result +} + +export function joinSpaced(parts: readonly string[]): string { + return parts.map(part => part.trim()).filter(Boolean).join(' ') +} + +export const greekNames = ['alpha', 'beta', 'gamma', 'delta', 'epsilon', 'varepsilon', 'zeta', 'eta', 'theta', 'vartheta', 'iota', 'kappa', 'lambda', 'mu', 'nu', 'xi', 'omicron', 'pi', 'rho', 'sigma', 'tau', 'upsilon', 'phi', 'varphi', 'chi', 'psi', 'omega'] as const +const greekLetters = 'αβγδεεζηθϑικλμνξοπρστυφφχψω' + +/** Maps each Greek command and its Unicode letter to one word. Capitals use the same word. */ +export function greek(words: readonly string[]): Record { + const result: Record = {} + greekNames.forEach((name, index) => { + const word = words[index]! + result[`\\${name}`] = word + result[`\\${name[0]!.toUpperCase()}${name.slice(1)}`] = word + result[greekLetters[index]!] = word + }) + return result +} + +/** + * The spellings of each relation and operator, keyed by one canonical spelling. Lexicons key their words by the + * canonical spelling, and lowering uses it as the identity of the symbol. + */ +export const symbolAliases: Readonly> = { + '=': ['='], + '\\neq': ['\\neq', '\\ne', '≠'], + '<': ['<', '\\lt'], + '>': ['>', '\\gt'], + '\\leq': ['\\leq', '\\le', '\\leqq', '\\leqslant', '≤', '⩽'], + '\\geq': ['\\geq', '\\ge', '\\geqq', '\\geqslant', '≥', '⩾'], + '\\approx': ['\\approx', '≈'], + '\\equiv': ['\\equiv', '≡'], + '\\perp': ['\\perp', '\\bot', '⊥'], + '\\parallel': ['\\parallel', '\\|', '∥'], + '\\ll': ['\\ll', '≪'], + '\\gg': ['\\gg', '≫'], + '\\in': ['\\in', '∈'], + '\\notin': ['\\notin', '∉'], + '\\ni': ['\\ni', '\\owns', '∋'], + '\\subseteq': ['\\subseteq', '⊆'], + '\\subset': ['\\subset', '\\subsetneq', '⊂', '⊊'], + '\\supseteq': ['\\supseteq', '⊇'], + '\\supset': ['\\supset', '\\supsetneq', '⊃', '⊋'], + '\\nsubseteq': ['\\nsubseteq', '⊈'], + '\\nsupseteq': ['\\nsupseteq', '⊉'], + '\\sim': ['\\sim', '\\backsim', '∽', '∼', '~'], + '\\cong': ['\\cong', '≅'], + '\\to': ['\\to', '\\rightarrow', '→', '\\longrightarrow'], + '\\mapsto': ['\\mapsto', '↦'], + '\\Rightarrow': ['\\Rightarrow', '\\implies', '⇒'], + '\\Leftrightarrow': ['\\Leftrightarrow', '\\iff', '⇔'], + '\\propto': ['\\propto', '∝'], + '\\gtrsim': ['\\gtrsim', '≳'], + '\\lesssim': ['\\lesssim', '≲'], + '\\nmid': ['\\nmid', '∤'], + '\\vdash': ['\\vdash', '⊢'], + '\\coloneqq': ['\\coloneqq', '≔', '\\coloneq', ':='], + '+': ['+'], + '-': ['-', '−'], + '\\times': ['\\times', '×', '*', '\\ast'], + '\\cdot': ['\\cdot', '·', '⋅'], + '\\div': ['\\div', '÷', '/'], + '\\pm': ['\\pm', '±'], + '\\mp': ['\\mp', '∓'], + '\\cup': ['\\cup', '∪'], + '\\cap': ['\\cap', '∩'], + '\\setminus': ['\\setminus', '\\smallsetminus', '∖'], + '\\sqcup': ['\\sqcup', '⊔'], + // Only between two operands. A triangle that starts an operand is the shape, see `words`. + '\\triangle': ['\\triangle', '△', '\\bigtriangleup'], + '!': ['!'], +} + +/** Each spelling with its canonical spelling, built once, because lowering looks up every symbol. */ +const canonicalSpellings = new Map(Object.entries(symbolAliases).flatMap(([key, spellings]) => spellings.map(spelling => [spelling, key] as const))) + +/** The canonical spelling of a symbol, for example `\\leq` for `≤`. Other symbols stay unchanged. */ +export function canonicalSymbol(value: string): string { + return canonicalSpellings.get(value) ?? value +} + +/** Maps every alias of a key, for example `\leq`, `\le`, `≤`, and `<=`, to one value. */ +export function aliases(entries: Record): Record { + const result: Record = {} + for (const [key, value] of Object.entries(entries)) { + for (const alias of symbolAliases[key] ?? [key]) + result[alias] = value + } + return result +} + +/** Korean particles change with the last sound of the word before them. */ +const finalConsonantLetters = new Set('lmnr'.split('')) +// 0 영, 1 일, 3 삼, 6 육, 7 칠, 8 팔 end in a consonant. 10 십, 100 백, 1000 천 and 10000 만 do too. +const finalConsonantDigits = new Set('013678'.split('')) +const hangulStart = 0xAC00 +const hangulEnd = 0xD7A3 + +function finalConsonant(word: string): 'none' | 'rieul' | 'other' { + const last = word.trim().at(-1) ?? '' + const code = last.codePointAt(0) ?? 0 + if (code >= hangulStart && code <= hangulEnd) { + const jong = (code - hangulStart) % 28 + return jong === 0 ? 'none' : jong === 8 ? 'rieul' : 'other' + } + if (/\d/.test(last)) { + if (last === '1' || last === '7' || last === '8') + return 'rieul' + return finalConsonantDigits.has(last) || /\d0$/.test(word.trim()) ? 'other' : 'none' + } + if (finalConsonantLetters.has(last.toLowerCase())) + return last.toLowerCase() === 'l' || last.toLowerCase() === 'r' ? 'rieul' : 'other' + return 'none' +} + +/** Appends the Korean particle that fits the word, for example 극한은 and 파이는. */ +export function josa(word: string, withConsonant: string, withVowel: string): string { + const sound = finalConsonant(word) + // 로 also follows a final ㄹ. + if (withVowel === '로') + return `${word}${sound === 'other' ? withConsonant : withVowel}` + return `${word}${sound === 'none' ? withVowel : withConsonant}` +} + +/** Names a Latin letter from a table. Other characters stay unchanged. */ +export function letterName(names: readonly string[], letter: string): string { + const index = letter.toUpperCase().charCodeAt(0) - 65 + return letter.length === 1 && index >= 0 && index < 26 ? names[index]! : letter +} + +/** Adds a power word to a denominator unit, for example 平方 before 秒. */ +export function unitPower(power: string | undefined, words: { 2: string, 3: string }): string { + return power === '2' ? words[2] : power === '3' ? words[3] : '' +} diff --git a/packages/speech-transformer-math/src/locales/en.test.ts b/packages/speech-transformer-math/src/locales/en.test.ts new file mode 100644 index 0000000000..347f16ff9c --- /dev/null +++ b/packages/speech-transformer-math/src/locales/en.test.ts @@ -0,0 +1,263 @@ +import { describe, expect, it } from 'vitest' + +import { MathSpeech } from '../speech' + +const speech = new MathSpeech() +const basel = String.raw`\lim_{n \to \infty}\sum_{i=1}^{n}\frac{1}{i^2}=\frac{\pi^2}{6}` + +describe('speakMath', () => { + it('reads the Basel sum', () => { + expect(speech.speak(basel, 'en')).toBe('the limit as n approaches infinity of the sum from i equals one to n of the fraction with numerator one and denominator i squared equals pi squared over six') + }) + + it.each([ + [String.raw`AB \perp CD`, 'A B is perpendicular to C D'], + [String.raw`l_1 \parallel l_2`, 'l one is parallel to l two'], + [String.raw`a \leq b`, 'a is less than or equal to b'], + [String.raw`x \in A`, 'x is an element of A'], + [String.raw`A \subseteq B`, 'A is a subset of or equal to B'], + [String.raw`\angle ABC = 90^\circ`, 'angle A B C equals ninety degrees'], + [String.raw`\triangle ABC \sim \triangle DEF`, 'triangle A B C is similar to triangle D E F'], + [String.raw`(a+b)^2`, 'the square of the quantity a plus b'], + [String.raw`\sqrt[3]{8} = 2`, 'the cube root of eight equals two'], + [String.raw`|x| \geq 0`, 'the absolute value of x is greater than or equal to zero'], + [String.raw`\int_0^1 x^2 dx`, 'the integral from zero to one of x squared d x'], + [String.raw`-x^{-1}`, 'negative x to the power of negative one'], + [String.raw`50\%`, 'fifty percent'], + ])('reads %s', (tex, expected) => { + expect(speech.speak(tex, 'en')).toBe(expected) + }) + + it.each([ + [String.raw`t \in [0,1]`, 't is an element of the closed interval from zero to one'], + [String.raw`x \in (0,1)`, 'x is an element of the open interval from zero to one'], + [String.raw`x \in [0, 1)`, 'x is an element of the interval from zero inclusive to one exclusive'], + [String.raw`x \in \left(0, 1\right]`, 'x is an element of the interval from zero exclusive to one inclusive'], + ])('reads the interval %s', (tex, expected) => { + expect(speech.speak(tex, 'en')).toBe(expected) + }) + + it('reads a pair as an interval only when a set symbol or an infinite bound shows it', () => { + expect(speech.speak(String.raw`A \cup (1, 2)`, 'en')).toBe('A union the open interval from one to two') + expect(speech.speak(String.raw`f(x, y) = 0`, 'en')).toBe('f of x, y equals zero') + }) + + it('reads ceilings, near relations, composition, big set operators, and labels in subscripts', () => { + expect(speech.speak(String.raw`m = \lceil \alpha \rceil`, 'en')).toBe('m equals the ceiling of alpha') + expect(speech.speak(String.raw`n \gtrsim 10`, 'en')).toBe('n is greater than or approximately ten') + expect(speech.speak(String.raw`\pi \nmid m`, 'en')).toBe('pi does not divide m') + expect(speech.speak(String.raw`\varphi \coloneqq \iota \circ \mu`, 'en')).toBe('phi is defined as iota composed with mu') + expect(speech.speak(String.raw`\bigcap_{i=1}^{r} E_i`, 'en')).toBe('the intersection from i equals one to r of E sub i') + // A word in a subscript is a label: `t` is not tonnes. + expect(speech.speak(String.raw`I_{\rm t} = 100`, 'en')).toBe('I sub t equals one hundred') + // Every digit before a scripted digit joins the number. + expect(speech.speak(String.raw`45.1_{\pm 0.5}`, 'en')).toBe('forty-five point one plus or minus zero point five') + }) + + it('reads parentheses that a neighbor binds to, so the grouping is not lost', () => { + expect(speech.speak(String.raw`v-(x-\epsilon)`, 'en')).toBe('v minus the quantity x minus epsilon') + // The times word ends a spoken group, so epsilon is not heard inside it. + expect(speech.speak(String.raw`(1-t)\epsilon`, 'en')).toBe('the quantity one minus t times epsilon') + // After a plus, the brackets change nothing, so they stay silent. + expect(speech.speak(String.raw`a+(b+c)`, 'en')).toBe('a plus b plus c') + }) + + it('reads arXiv notation that the MathBridge review found', () => { + // A blackboard letter without an argument names a space or a measure, not an expectation. + expect(speech.speak(String.raw`(\Omega, \mathcal{F}, \mathbb{P})`, 'en')).toBe('the tuple omega, F, P') + expect(speech.speak(String.raw`\mathbb{E}[X]`, 'en')).toBe('the expected value of X') + // In measurements, `\sim` before a quantity means about. + expect(speech.speak(String.raw`\sim 50`, 'en')).toBe('about fifty') + expect(speech.speak(String.raw`z \sim 1`, 'en')).toBe('z is approximately equal to one') + expect(speech.speak(String.raw`\|\cdot\|`, 'en')).toBe('the norm') + expect(speech.speak(String.raw`x := y + 1`, 'en')).toBe('x is defined as y plus one') + expect(speech.speak(String.raw`F(A, {\bf B})`, 'en')).toBe('F of A, B') + expect(speech.speak(String.raw`1.4 M_\odot`, 'en')).toBe('one point four solar masses') + expect(speech.speak(String.raw`\cap_{i=1}^{r} E_i`, 'en')).toBe('the intersection from i equals one to r of E sub i') + expect(speech.speak(String.raw`Z \sim \mathcal{CN}(0, 1)`, 'en')).toBe('Z drawn from C N of zero, one') + // Superscripts that label instead of raising, and boundaries that a listener needs. + expect(speech.speak(String.raw`x \in \mathbb{P}^n`, 'en')).toBe('x is an element of P n') + expect(speech.speak(String.raw`X \in L^p(\Omega)`, 'en')).toBe('X is an element of L p of omega') + expect(speech.speak(String.raw`K^\dagger`, 'en')).toBe('K dagger') + expect(speech.speak(String.raw`G = (V, E)`, 'en')).toBe('G equals the pair V, E') + expect(speech.speak(String.raw`\frac{7}{60} k`, 'en')).toBe('seven over sixty times k') + expect(speech.speak(String.raw`|q|^2`, 'en')).toBe('the squared absolute value of q') + expect(speech.speak(String.raw`(e_1, \dots, e_n)`, 'en')).toBe('the tuple e one, and so on, up to e sub n') + expect(speech.speak(String.raw`vMF(\mu, \kappa)`, 'en')).toBe('v M F of mu, kappa') + }) + + it('reads like a presenter, not like a dictation', () => { + // A mark after a symbol is said with it, without the word superscript. + expect(speech.speak(String.raw`\Theta^*`, 'en')).toBe('theta star') + expect(speech.speak(String.raw`K_i^\dag`, 'en')).toBe('K sub i dagger') + expect(speech.speak(String.raw`\mathbb{L}^\infty(X)`, 'en')).toBe('L infinity of X') + // Only the capital says its case, and the norm itself has no argument to read. + expect(speech.speak(String.raw`\sum_{k=1}^{K} x_k`, 'en')).toBe('the sum from k equals one to capital K of x sub k') + expect(speech.speak(String.raw`\|\cdot\|_2`, 'en')).toBe('the two-norm') + // Measurements read as a person says them. + expect(speech.speak(String.raw`\approx 7`, 'en')).toBe('about seven') + expect(speech.speak(String.raw`\sim 1\,\mathrm{mm}`, 'en')).toBe('about one millimeter') + expect(speech.speak(String.raw`45.1_{\pm 0.5}`, 'en')).toBe('forty-five point one plus or minus zero point five') + // The square of a group is said before it, so it does not sound like the square of its last term. + expect(speech.speak(String.raw`(1+\sqrt{\delta})^2`, 'en')).toBe('the square of the quantity one plus the square root of delta') + }) + + it('reads like a Chinese teacher', () => { + expect(speech.speak(String.raw`2.5e10`, 'en')).toBe('two point five times ten to the power of ten') + }) + + it('reads an integral with an infinite bound as an improper integral', () => { + const tex = String.raw`\int_0^{+\infty} e^{-x}\,dx` + expect(speech.speak(tex, 'en')).toBe('the improper integral from zero to positive infinity of e to the power of negative x d x') + }) + + it.each([ + [String.raw`A \setminus B`, 'A minus B'], + [String.raw`A \triangle B`, 'A symmetric difference B'], + [String.raw`A \sqcup B`, 'A disjoint union B'], + [String.raw`A^c`, 'the complement of A'], + [String.raw`A^{\complement}`, 'the complement of A'], + [String.raw`\complement_U A`, 'the complement of A in U'], + [String.raw`\mathcal{P}(A)`, 'the power set of A'], + [String.raw`\#A`, 'the number of elements of A'], + [String.raw`\operatorname{card}(A)`, 'the number of elements of A'], + [String.raw`A \ni x`, 'A contains x'], + [String.raw`A \supsetneq B`, 'A is a proper superset of B'], + [String.raw`A \nsubseteq B`, 'A is not a subset of B'], + [String.raw`\nexists x`, 'there does not exist x'], + [String.raw`a \ll b`, 'a is much less than b'], + [String.raw`a \gg b`, 'a is much greater than b'], + ])('reads the set notation %s', (tex, expected) => { + expect(speech.speak(tex, 'en')).toBe(expected) + }) + + it('reads an unknown command by its name and does not fail on unbalanced input', () => { + expect(speech.speak(String.raw`\foo{x}`, 'en')).toBe('foo x') + expect(speech.speak(String.raw`\frac{1}{`, 'en')).toBe('one over') + }) +}) + +// The cases below come from the formula speech eval, where a StepFun voice read the rule output. +// Each one failed to reach the listener before, because the text left a choice to the voice or to the listener. +describe('speakMath readings that a voice cannot get wrong', () => { + it('spells numbers in English, Japanese, and Korean, and keeps Chinese digits', () => { + const tex = String.raw`\frac{1}{2}+\frac{1}{3}=\frac{5}{6}` + // A Chinese voice read the 1 of `1 over 2` as yī, which an English listener heard as "E". + expect(speech.speak(tex, 'en')).toBe('one over two plus one over three equals five over six') + }) + + it.each([ + [String.raw`a \in A`, 'a is an element of capital A'], + [String.raw`\Omega \neq \omega`, 'capital omega is not equal to omega'], + ])('says the case of a letter that %s uses in both cases', (tex, expected) => { + expect(speech.speak(tex, 'en')).toBe(expected) + }) + + it('keeps the case unsaid when each letter has one case', () => { + expect(speech.speak('E=mc^2', 'en')).toBe('E equals m c squared') + }) + + it.each([ + [String.raw`9.8\,\mathrm{m/s^2}`, 'nine point eight meters per second squared'], + [String.raw`3\times10^{8}\,\mathrm{m/s}`, 'three times ten to the power of eight meters per second'], + [String.raw`25^\circ\mathrm{C}`, 'twenty-five degrees Celsius'], + [String.raw`5\,\mathrm{kg}`, 'five kilograms'], + ])('reads the unit of %s as words', (tex, expected) => { + expect(speech.speak(tex, 'en')).toBe(expected) + }) + + it.each([ + // A compound part names the numerator and the denominator, so the listener knows where each one ends. + [String.raw`\frac{1}{1+\frac{1}{x}}`, 'the fraction with numerator one and denominator one plus one over x'], + [String.raw`\frac{f-1}{2}`, 'the fraction with numerator f minus one and denominator two'], + // A power after the second-read part could bind to the whole fraction, so it names both parts too. + [String.raw`\frac{\pi^2}{6}`, 'pi squared over six'], + [String.raw`\frac{6}{\pi^2}`, 'the fraction with numerator six and denominator pi squared'], + ])('reads the fraction %s with clear bounds', (tex, expected) => { + expect(speech.speak(tex, 'en')).toBe(expected) + }) + + it('reads a sum in parentheses that multiplies as a quantity', () => { + expect(speech.speak(String.raw`\frac{n(n+1)}{2}`, 'en')).toBe('the fraction with numerator n the quantity n plus one and denominator two') + expect(speech.speak(String.raw`f(x, y) = 0`, 'en')).toBe('f of x, y equals zero') + }) + + it('reads a derivative, a one-sided limit, a subscript, and a function signature', () => { + expect(speech.speak(String.raw`\frac{dy}{dx}`, 'en')).toBe('the derivative with respect to x of y') + expect(speech.speak(String.raw`\lim_{x \to 0^{-}} x`, 'en')).toBe('the limit as x approaches zero from the left of x') + expect(speech.speak(String.raw`x_1 \leq \cdots \leq x_n`, 'en')).toBe('x one is less than or equal to and so on is less than or equal to x sub n') + expect(speech.speak(String.raw`k = 1, \dots, d`, 'en')).toBe('k equals one, and so on, up to d') + expect(speech.speak(String.raw`f: X \to Y, x \mapsto y`, 'en')).toBe('f is a function from capital X to capital Y, x maps to y') + }) +}) + +describe('readings of the second review', () => { + it.each([ + // A function command before parentheses calls them, also with a sum inside. + [String.raw`\ln(x+1)`, 'natural log of the quantity x plus one'], + [String.raw`\gcd(a,b)`, 'gcd of a, b'], + [String.raw`\operatorname{tr}(A+B)`, 'tr of the quantity A plus B'], + [String.raw`\mathrm{GL}(n,\mathbb{R})`, 'GL of n, the real numbers'], + // Upright d, e, and i are letters, and an upright T over a matrix is its transpose. + [String.raw`\int_0^1 f(x)\,\mathrm{d}x`, 'the integral from zero to one of f of x d x'], + [String.raw`A^\mathrm{T}`, 'A T'], + [String.raw`f''(x)`, 'f prime prime of x'], + [String.raw`f^{-1}(x)`, 'f inverse of x'], + [String.raw`x^{-1}`, 'x to the power of negative one'], + [String.raw`x^{2}(t)`, 'x squared t'], + // Scientific notation needs its parts together. With spaces, e is Euler's number. + [String.raw`2e - 1`, 'two e minus one'], + [String.raw`2.5e10`, 'two point five times ten to the power of ten'], + [String.raw`1,000,000`, 'one million'], + [String.raw`[1,100]`, 'the closed interval from one to one hundred'], + [String.raw`x = 2 \quad 3`, 'x equals two three'], + [String.raw`.5`, 'zero point five'], + [String.raw`007`, 'zero zero seven'], + [String.raw`n^{\text{th}}`, 'n-th'], + [String.raw`|x - |y||`, 'the absolute value of x minus the absolute value of y'], + [String.raw`\frac{d^2 y}{dx^2}`, 'the second derivative with respect to x of y'], + [String.raw`f(x)=\begin{cases}1 & x>0\\0 & \text{otherwise}\end{cases}`, 'f of x equals one if x is greater than zero, zero otherwise'], + [String.raw`\begin{pmatrix}1&2\\3&4\end{pmatrix}`, 'the matrix with row one: one, two; row two: three, four'], + [String.raw`\begin{aligned}a&=b\\&=c\end{aligned}`, 'a equals b equals c'], + [String.raw`\mathrm{Rt}\triangle ABC`, 'right triangle A B C'], + ])('reads %s', (tex, expected) => { + expect(speech.speak(tex, 'en')).toBe(expected) + }) + + it.each([ + [String.raw`\mathrm{s^{-1}}`, 'per second'], + [String.raw`3\,\mathrm{mol\,L^{-1}}`, 'three moles per liter'], + [String.raw`5\,\mathrm{kg\cdot m^2}`, 'five kilogram square meters'], + [String.raw`8.31\,\mathrm{J\,mol^{-1}\,K^{-1}}`, 'eight point three one joules per mole kelvin'], + [String.raw`10\,\mathrm{dB}`, 'ten decibels'], + [String.raw`3\,\mathrm{ft}`, 'three feet'], + ])('reads the unit of %s', (tex, expected) => { + expect(speech.speak(tex, 'en')).toBe(expected) + }) +}) + +describe('mappings of the review', () => { + it.each([ + // The bar inside an argument is a condition. Between two operands, it divides. + [String.raw`P(A \mid B)`, 'P of A given B'], + [String.raw`\mathbb{E}[X \mid Y]`, 'the expected value of X given Y'], + [String.raw`a \mid b`, 'a divides b'], + [String.raw`R = 10\,\Omega`, 'R equals ten ohms'], + [String.raw`\mathbb{N}^{+}`, 'the positive integers'], + [String.raw`a \equiv b \pmod{n}`, 'a is congruent to b modulo n'], + [String.raw`\sqrt[4]{x}`, 'the fourth root of x'], + [String.raw`A \supseteq B`, 'A is a superset of or equal to B'], + ])('reads %s', (tex, expected) => { + expect(speech.speak(tex, 'en')).toBe(expected) + }) +}) + +describe('readings of the eval review', () => { + it.each([ + // In "the squared norm of v minus u", two of three listeners put the minus outside the norm. + [String.raw`\|v_\theta(x_t,t)-(x-\epsilon)\|^2`, 'the squared norm of the difference between v sub theta of x sub t, t and the quantity x minus epsilon'], + [String.raw`E=mc^2`, 'E equals m c squared'], + ])('reads %s', (tex, expected) => { + expect(speech.speak(tex, 'en')).toBe(expected) + }) +}) diff --git a/packages/speech-transformer-math/src/locales/en.ts b/packages/speech-transformer-math/src/locales/en.ts new file mode 100644 index 0000000000..3ae0520507 --- /dev/null +++ b/packages/speech-transformer-math/src/locales/en.ts @@ -0,0 +1,219 @@ +import type { MathLocale, Relation } from '../locale' + +import { aliases, greek, greekNames, joinSpaced } from '../locale' +import { spellNumber } from '../numbers' + +/** Ordinal words of a derivative order. Other orders add -th to the spelled number. */ +const ordinals: Readonly> = { 2: 'second', 3: 'third' } + +/** Ordinal words of a root degree, after the number is spelled. */ +const rootOrdinals: Readonly> = { four: 'fourth', five: 'fifth', six: 'sixth', seven: 'seventh', eight: 'eighth', nine: 'ninth', ten: 'tenth' } + +export const en: MathLocale = { + language: 'en', + join: joinSpaced, + number: digits => spellNumber(digits, 'en'), + letter: letter => letter, + // Only the capital says its case: "k from one to capital K" is clear, and "lowercase" on every k is heavy. + letterCase: (spoken, upper) => upper ? `capital ${spoken}` : spoken, + clauses: parts => parts.join(', '), + cases: items => items.map(({ value, condition }) => condition === undefined ? `${value} otherwise` : `${value} if ${condition}`).join(', '), + matrix: (rows, determinant) => `the ${determinant ? 'determinant' : 'matrix'} with ${rows.map((row, index) => `row ${spellNumber(String(index + 1), 'en')}: ${row.join(', ')}`).join('; ')}`, + list: items => items.join(', '), + words: { + ...greek(greekNames.map(name => name.replace(/^var/, ''))), + '\\nabla': 'nabla', + '∇': 'nabla', + // `\varsigma` is another letter, the final sigma, not a variant glyph as `\varphi` is. + '\\varsigma': 'final sigma', + 'M_\\odot': 'solar masses', + 'L_\\odot': 'solar luminosities', + 'R_\\odot': 'solar radii', + '\\checkmark': 'check mark', + // Between two operands, `\frown` is the cap product of topology. Over letters it is an arc, see ../lower. + '\\frown': 'cap product', + '\\ast': 'star', + '\\infty': 'infinity', + '∞': 'infinity', + '\\angle': 'angle', + '∠': 'angle', + '\\triangle': 'triangle', + '△': 'triangle', + '\\odot': 'circle', + '⊙': 'circle', + '\\circ': 'degrees', + '°': 'degrees', + '\\cdots': 'and so on', + '\\ldots': 'and so on', + '\\dots': 'and so on', + '…': 'and so on', + '\\emptyset': 'the empty set', + '\\varnothing': 'the empty set', + '∅': 'the empty set', + '\\forall': 'for all', + '\\exists': 'there exists', + '\\nexists': 'there does not exist', + '∄': 'there does not exist', + '\\aleph': 'aleph', + 'ℵ': 'aleph', + '\\therefore': 'therefore', + '\\because': 'because', + '\\partial': 'partial', + '\\sin': 'sine', + '\\cos': 'cosine', + '\\tan': 'tangent', + '\\cot': 'cotangent', + '\\sec': 'secant', + '\\csc': 'cosecant', + '\\log': 'log', + '\\lg': 'log', + '\\ln': 'natural log', + '\\lim': 'limit', + '\\max': 'max', + '\\min': 'min', + '\\mid': 'such that', + '\\%': 'percent', + '%': 'percent', + }, + operators: aliases({ '+': 'plus', '-': 'minus', '\\times': 'times', '\\cdot': 'times', '\\div': 'divided by', '\\pm': 'plus or minus', '\\mp': 'minus or plus', '\\cup': 'union', '\\cap': 'intersection', '\\setminus': 'minus', '\\sqcup': 'disjoint union', '\\triangle': 'symmetric difference', '\\circ': 'composed with', '!': 'factorial' }), + negative: 'negative', + positive: 'positive', + relations: aliases({ + '=': (l, r) => `${l} equals ${r}`, + '\\neq': (l, r) => `${l} is not equal to ${r}`, + '<': (l, r) => `${l} is less than ${r}`, + '>': (l, r) => `${l} is greater than ${r}`, + '\\ll': (l, r) => `${l} is much less than ${r}`, + '\\gg': (l, r) => `${l} is much greater than ${r}`, + '\\leq': (l, r) => `${l} is less than or equal to ${r}`, + '\\geq': (l, r) => `${l} is greater than or equal to ${r}`, + '\\approx': (l, r) => `${l} is approximately equal to ${r}`, + '\\equiv': (l, r) => `${l} is equivalent to ${r}`, + '\\perp': (l, r) => `${l} is perpendicular to ${r}`, + '\\parallel': (l, r) => `${l} is parallel to ${r}`, + '\\in': (l, r) => `${l} is an element of ${r}`, + '\\notin': (l, r) => `${l} is not an element of ${r}`, + '\\ni': (l, r) => `${l} contains ${r}`, + // `\subseteq` says "or equal to", because a listener writes a plain "subset" as `\subset`. + '\\subseteq': (l, r) => `${l} is a subset of or equal to ${r}`, + '\\subset': (l, r) => `${l} is a proper subset of ${r}`, + '\\supseteq': (l, r) => `${l} is a superset of or equal to ${r}`, + '\\supset': (l, r) => `${l} is a proper superset of ${r}`, + '\\nsubseteq': (l, r) => `${l} is not a subset of ${r}`, + '\\nsupseteq': (l, r) => `${l} is not a superset of ${r}`, + '\\sim': (l, r) => `${l} is similar to ${r}`, + '\\cong': (l, r) => `${l} is congruent to ${r}`, + '\\to': (l, r) => `${l} approaches ${r}`, + '\\mapsto': (l, r) => `${l} maps to ${r}`, + '\\Rightarrow': (l, r) => `${l} implies ${r}`, + '\\Leftrightarrow': (l, r) => `${l} if and only if ${r}`, + '\\propto': (l, r) => `${l} is proportional to ${r}`, + '\\gtrsim': (l, r) => `${l} is greater than or approximately ${r}`, + '\\lesssim': (l, r) => `${l} is less than or approximately ${r}`, + '\\mid': (l, r) => `${l} divides ${r}`, + '\\nmid': (l, r) => `${l} does not divide ${r}`, + '\\vdash': (l, r) => `${l} proves ${r}`, + '\\coloneqq': (l, r) => `${l} is defined as ${r}`, + }), + // The numerator comes first, so a power on the denominator can bind to the whole fraction: 6 over pi squared. + fraction: (n, d, shape) => shape.numerator === 'compound' || shape.denominator === 'compound' || shape.denominator === 'scripted' + ? `the fraction with numerator ${n} and denominator ${d}` + : `${n} over ${d}`, + derivative: (v, body, order) => `the ${order ? `${ordinals[order] ?? `${spellNumber(order, 'en')}th`} ` : ''}derivative with respect to ${v} of ${body}`, + power: (b, e) => `${b} to the power of ${e}`, + squared: b => `${b} squared`, + cubed: b => `${b} cubed`, + degrees: b => `${b} degrees`, + prime: b => `${b} prime`, + // A presenter says "theta star", "P n", and "G p". The word superscript names the layout, not the meaning. + superscript: (b, m) => `${b} ${m}`, + marks: { '\\ast': 'star', '\\dagger': 'dagger', '\\vee': 'dual' }, + // "of" marks the call, so `f(x)` is not heard as the product `f x`. + apply: (f, args, grouped) => grouped && args.length === 1 ? `${f} of the quantity ${args[0]}` : `${f} of ${args.join(', ')}`, + upTo: x => `up to ${x}`, + placeholder: 'dot', + numberIndex: (b, i) => `${b} ${i}`, + index: (b, i) => `${b} sub ${i}`, + subscript: (b, s) => `${b} sub ${s}`, + ordinal: value => `${value}-th`, + given: (event, condition) => `${event} given ${condition}`, + modulo: m => `modulo ${m}`, + congruent: (l, r, m) => `${l} is congruent to ${r} modulo ${m}`, + inverse: name => `${name} inverse`, + root: (x, n) => { + if (n === undefined) + return `the square root of ${x}` + if (n === 'three') + return `the cube root of ${x}` + // A letter takes an ordinal suffix, as in the n-th root. A spelled number takes its ordinal word, as the fourth root. + if (/^[a-z]$/i.test(n)) + return `the ${n}th root of ${x}` + return rootOrdinals[n] ? `the ${rootOrdinals[n]} root of ${x}` : `the root of degree ${n} of ${x}` + }, + limit: (body, v, t, c, side) => { + if (v !== undefined && t !== undefined) + return `the limit as ${v} approaches ${t}${side ? ` from the ${side}` : ''} of ${body}` + return c ? `the limit over ${c} of ${body}` : `the limit of ${body}` + }, + normDifference: (left, right) => `the difference between ${left} and ${right}`, + norm: (x, index, squared) => `the ${squared ? 'squared ' : ''}${index ? `${index}-norm` : 'norm'}${x ? ` of ${x}` : ''}`, + statistic: (kind, body, condition) => { + const noun = { Expectation: 'the expected value', Probability: 'the probability', Variance: 'the variance', Covariance: 'the covariance' }[kind] + return condition ? `${noun}, over ${condition}, of ${body}` : `${noun} of ${body}` + }, + about: x => `about ${x}`, + distributed: (x, d) => `${x} drawn from ${d}`, + arrow: (l, r) => `${l} to ${r}`, + ceil: x => `the ceiling of ${x}`, + floor: x => `the floor of ${x}`, + tuple: items => `${items.length === 2 ? 'the pair' : 'the tuple'} ${items.join(', ')}`, + squareOf: x => `the square of ${x}`, + absoluteSquared: x => `the squared absolute value of ${x}`, + bracket: x => `open bracket ${x} close bracket`, + mapping: (f, from, to) => `${f} is a function from ${from} to ${to}`, + bigOperator: (kind, body, { variable, from, to, condition, improper }) => { + const noun = { sum: 'the sum', prod: 'the product', int: improper ? 'the improper integral' : 'the integral', union: 'the union', intersection: 'the intersection' }[kind] + if (kind === 'int') + return from !== undefined || to !== undefined ? `${noun} from ${from ?? ''} to ${to ?? ''} of ${body}` : `${noun} of ${body}` + if (variable !== undefined && from !== undefined) + return `${noun} from ${variable} equals ${from} to ${to ?? 'infinity'} of ${body}` + return condition ? `${noun} over ${condition} of ${body}` : `${noun} of ${body}` + }, + absolute: x => `the absolute value of ${x}`, + quantity: x => `the quantity ${x}`, + set: (x, c) => c ? `the set of ${x} such that ${c}` : `the set ${x}`, + interval: (a, b, { left, right }) => left === right + ? `the ${left ? 'closed' : 'open'} interval from ${a} to ${b}` + : `the interval from ${a} ${left ? 'inclusive' : 'exclusive'} to ${b} ${right ? 'inclusive' : 'exclusive'}`, + complement: (set, universe) => universe ? `the complement of ${set} in ${universe}` : `the complement of ${set}`, + powerSet: set => `the power set of ${set}`, + cardinality: set => `the number of elements of ${set}`, + percent: n => `${n} percent`, + accent: { + ket: b => `ket ${b}`, + bra: b => `bra ${b}`, + overline: b => `${b} bar`, + bar: b => `${b} bar`, + vec: b => `vector ${b}`, + overrightarrow: b => `vector ${b}`, + hat: b => `${b} hat`, + widehat: b => `${b} hat`, + tilde: b => `${b} tilde`, + dot: b => `${b} dot`, + overarc: b => `arc ${b}`, + wideparen: b => `arc ${b}`, + }, + segment: name => `segment ${name}`, + binomial: (n, k) => `${n} choose ${k}`, + // The denominator unit is singular: meters per second. + unitRatio: (n, d, power) => `${n} per ${d}${power === '2' ? ' squared' : power === '3' ? ' cubed' : ''}`, + celsius: value => `${value} degrees Celsius`, + numberSets: { 'R': 'the real numbers', 'N': 'the natural numbers', 'Z': 'the integers', 'Q': 'the rational numbers', 'C': 'the complex numbers', 'N+': 'the positive integers', 'N*': 'the positive integers', 'Z+': 'the positive integers', 'Z-': 'the negative integers', 'Z*': 'the nonzero integers', 'R+': 'the positive real numbers', 'R-': 'the negative real numbers', 'R*': 'the nonzero real numbers', 'Q+': 'the positive rational numbers' }, + units: { + prefixes: { Q: 'quetta', R: 'ronna', Y: 'yotta', Z: 'zetta', E: 'exa', P: 'peta', T: 'tera', G: 'giga', M: 'mega', k: 'kilo', h: 'hecto', da: 'deca', d: 'deci', c: 'centi', m: 'milli', µ: 'micro', n: 'nano', p: 'pico', f: 'femto', a: 'atto', z: 'zepto', y: 'yocto', r: 'ronto', q: 'quecto' }, + units: { m: 'meters', g: 'grams', s: 'seconds', A: 'amperes', K: 'kelvins', mol: 'moles', cd: 'candelas', Hz: 'hertz', N: 'newtons', Pa: 'pascals', J: 'joules', W: 'watts', C: 'coulombs', V: 'volts', F: 'farads', Ω: 'ohms', S: 'siemens', Wb: 'webers', T: 'teslas', H: 'henries', lm: 'lumens', lx: 'lux', Bq: 'becquerels', Gy: 'grays', Sv: 'sieverts', kat: 'katals', L: 'liters', l: 'liters', t: 'tonnes', eV: 'electronvolts', Da: 'daltons', bar: 'bars', B: 'bytes', bit: 'bits', cal: 'calories' }, + plain: { min: 'minutes', h: 'hours', d: 'days', ha: 'hectares', au: 'astronomical units', Å: 'angstroms', atm: 'atmospheres', Rt: 'right', dB: 'decibels', ft: 'feet', mi: 'miles' }, + power: (words, power) => `${power === '2' ? 'square' : 'cubic'} ${words}`, + singular: words => words === 'feet' ? 'foot' : words.endsWith('ies') ? `${words.slice(0, -3)}y` : /(?:hertz|siemens|lux)$/.test(words) || !words.endsWith('s') ? words : words.slice(0, -1), + }, +} diff --git a/packages/speech-transformer-math/src/locales/ja.test.ts b/packages/speech-transformer-math/src/locales/ja.test.ts new file mode 100644 index 0000000000..d3b4494c59 --- /dev/null +++ b/packages/speech-transformer-math/src/locales/ja.test.ts @@ -0,0 +1,151 @@ +import { describe, expect, it } from 'vitest' + +import { MathSpeech } from '../speech' + +const speech = new MathSpeech() +const basel = String.raw`\lim_{n \to \infty}\sum_{i=1}^{n}\frac{1}{i^2}=\frac{\pi^2}{6}` + +describe('speakMath', () => { + it('reads the Basel sum', () => { + expect(speech.speak(basel, 'ja')).toBe('エヌを無限大に近づけたときのアイがいちからエヌまでのアイの二乗分のいちの和の極限 イコール 分子がパイの二乗、分母がろくの分数') + }) + + it.each([ + [String.raw`AB \perp CD`, 'エービーとシーディーは垂直'], + [String.raw`l_1 \parallel l_2`, 'エルいちとエルには平行'], + [String.raw`a \leq b`, 'エー 小なりイコール ビー'], + [String.raw`x \in A`, 'エックスはエーに属する'], + [String.raw`A \subseteq B`, 'エーはビーに含まれる'], + [String.raw`\angle ABC = 90^\circ`, '角エービーシー イコール きゅうじゅう度'], + [String.raw`\triangle ABC \sim \triangle DEF`, '三角形エービーシーと三角形ディーイーエフは相似'], + [String.raw`(a+b)^2`, 'かっこエープラスビーかっことじの二乗'], + [String.raw`\sqrt[3]{8} = 2`, 'はちのさん乗根 イコール に'], + [String.raw`|x| \geq 0`, 'エックスの絶対値 大なりイコール ゼロ'], + [String.raw`\int_0^1 x^2 dx`, 'ゼロからいちまでのエックスの二乗ディーエックスの積分'], + [String.raw`-x^{-1}`, 'マイナスエックスのマイナスいち乗'], + [String.raw`50\%`, 'ごじゅうパーセント'], + ])('reads %s', (tex, expected) => { + expect(speech.speak(tex, 'ja')).toBe(expected) + }) + + it.each([ + [String.raw`t \in [0,1]`, 'ティーはゼロからいちまでの閉区間に属する'], + [String.raw`x \in (0,1)`, 'エックスはゼロからいちまでの開区間に属する'], + [String.raw`x \in [0, 1)`, 'エックスはゼロ以上いち未満の区間に属する'], + [String.raw`x \in \left(0, 1\right]`, 'エックスはゼロより大きくいち以下の区間に属する'], + ])('reads the interval %s', (tex, expected) => { + expect(speech.speak(tex, 'ja')).toBe(expected) + }) + + it('reads an integral with an infinite bound as an improper integral', () => { + const tex = String.raw`\int_0^{+\infty} e^{-x}\,dx` + expect(speech.speak(tex, 'ja')).toBe('ゼロからプラス無限大までのイーのマイナスエックス乗ディーエックスの広義積分') + }) + + it.each([ + [String.raw`A \setminus B`, 'エーマイナスビー'], + [String.raw`A \triangle B`, 'エー対称差ビー'], + [String.raw`A \sqcup B`, 'エー直和ビー'], + [String.raw`A^c`, 'エーの補集合'], + [String.raw`A^{\complement}`, 'エーの補集合'], + [String.raw`\complement_U A`, 'ユーにおけるエーの補集合'], + [String.raw`\mathcal{P}(A)`, 'エーの冪集合'], + [String.raw`\#A`, 'エーの要素の個数'], + [String.raw`\operatorname{card}(A)`, 'エーの要素の個数'], + [String.raw`A \ni x`, 'エーはエックスを要素に持つ'], + [String.raw`A \supsetneq B`, 'エーはビーを真に含む'], + [String.raw`A \nsubseteq B`, 'エーはビーに含まれない'], + [String.raw`\nexists x`, '存在しないエックス'], + [String.raw`a \ll b`, 'エーはビーよりずっと小さい'], + [String.raw`a \gg b`, 'エーはビーよりずっと大きい'], + ])('reads the set notation %s', (tex, expected) => { + expect(speech.speak(tex, 'ja')).toBe(expected) + }) +}) + +// The cases below come from the formula speech eval, where a StepFun voice read the rule output. +// Each one failed to reach the listener before, because the text left a choice to the voice or to the listener. +describe('speakMath readings that a voice cannot get wrong', () => { + it('spells numbers in English, Japanese, and Korean, and keeps Chinese digits', () => { + const tex = String.raw`\frac{1}{2}+\frac{1}{3}=\frac{5}{6}` + expect(speech.speak(tex, 'ja')).toBe('に分のいちプラスさん分のいち イコール ろく分のご') + }) + + it.each([ + [String.raw`a \in A`, '小文字のエーは大文字のエーに属する'], + [String.raw`\Omega \neq \omega`, '大文字のオメガ ノットイコール 小文字のオメガ'], + ])('says the case of a letter that %s uses in both cases', (tex, expected) => { + expect(speech.speak(tex, 'ja')).toBe(expected) + }) + + it.each([ + [String.raw`9.8\,\mathrm{m/s^2}`, 'きゅうてんはちメートル毎平方秒'], + [String.raw`3\times10^{8}\,\mathrm{m/s}`, 'さんかけるじゅうのはち乗メートル毎秒'], + [String.raw`25^\circ\mathrm{C}`, '摂氏にじゅうご度'], + [String.raw`5\,\mathrm{kg}`, 'ごキログラム'], + ])('reads the unit of %s as words', (tex, expected) => { + expect(speech.speak(tex, 'ja')).toBe(expected) + }) + + it.each([ + // A compound part names the numerator and the denominator, so the listener knows where each one ends. + [String.raw`\frac{1}{1+\frac{1}{x}}`, '分子がいち、分母がいちプラス分子がいち、分母がエックスの分数の分数'], + [String.raw`\frac{f-1}{2}`, '分子がエフマイナスいち、分母がにの分数'], + // A power after the second-read part could bind to the whole fraction, so it names both parts too. + [String.raw`\frac{\pi^2}{6}`, '分子がパイの二乗、分母がろくの分数'], + [String.raw`\frac{6}{\pi^2}`, 'パイの二乗分のろく'], + ])('reads the fraction %s with clear bounds', (tex, expected) => { + expect(speech.speak(tex, 'ja')).toBe(expected) + }) +}) + +describe('readings of the review', () => { + it.each([ + // The subscript of a name runs into its argument without かっこ: キュー下付きエムティー. + [String.raw`q_m(t)`, 'キュー下付きエムかっこティーかっことじ'], + [String.raw`x_1, x_2, \dots, x_n`, 'エックスいち、エックスにからエックス下付きエヌまで'], + [String.raw`k = 1, \dots, d`, 'ケー イコール いちからディーまで'], + // かっこ shows where a compound radicand ends, so the two formulas below sound different. + [String.raw`\sqrt{x+1}`, 'ルートかっこエックスプラスいちかっことじ'], + [String.raw`\sqrt{x}+1`, 'ルートエックスプラスいち'], + [String.raw`\{x \mid x > 0\}`, 'エックス 大なり ゼロ を満たすエックスの集合'], + ])('reads %s', (tex, expected) => { + expect(speech.speak(tex, 'ja')).toBe(expected) + }) +}) + +describe('readings of the second review', () => { + it.each([ + [String.raw`f(x)=\begin{cases}1 & x>0\\0 & \text{otherwise}\end{cases}`, 'エフエックス イコール エックス 大なり ゼロのときいち、それ以外のときゼロ'], + [String.raw`\frac{d^2 y}{dx^2}`, 'ワイをエックスでに回微分したもの'], + [String.raw`f^{-1}(x)`, 'エフの逆関数かっこエックスかっことじ'], + // A speed is 毎時. 時間 is a duration. + [String.raw`100\,\mathrm{km/h}`, 'ひゃくキロメートル毎時'], + // 一千万 is いっせんまん. + [String.raw`10000000`, 'いっせんまん'], + ])('reads %s', (tex, expected) => { + expect(speech.speak(tex, 'ja')).toBe(expected) + }) +}) + +describe('mappings of the review', () => { + it.each([ + [String.raw`\mathbb{E}[X \mid Y]`, 'ワイのもとでのエックスの期待値'], + // 合同 is a congruence modulo n. Alone, `\equiv` is an identity. + [String.raw`a \equiv b \pmod{n}`, 'エーとビーはエヌを法として合同'], + [String.raw`f(x) \equiv 0`, 'エフエックスはゼロと恒等的に等しい'], + [String.raw`A \subset B`, 'エーはビーに含まれる'], + ])('reads %s', (tex, expected) => { + expect(speech.speak(tex, 'ja')).toBe(expected) + }) +}) + +describe('readings of the eval review', () => { + it.each([ + // 以下同様 inside a chain was heard as 以下, which means less than or equal. + [String.raw`x_1 \leq \cdots \leq x_n`, 'エックスいちからエックス下付きエヌまで、前の値 小なりイコール 次の値'], + [String.raw`a < b < c`, 'エー 小なり ビー 小なり シー'], + ])('reads %s', (tex, expected) => { + expect(speech.speak(tex, 'ja')).toBe(expected) + }) +}) diff --git a/packages/speech-transformer-math/src/locales/ja.ts b/packages/speech-transformer-math/src/locales/ja.ts new file mode 100644 index 0000000000..628d622734 --- /dev/null +++ b/packages/speech-transformer-math/src/locales/ja.ts @@ -0,0 +1,214 @@ +import type { MathLocale, Relation } from '../locale' + +import { aliases, greek, joinCompact, letterName, unitPower } from '../locale' +import { spellNumber } from '../numbers' + +/** Latin letter names, A to Z. A Japanese listener names a Latin letter this way. */ +const jaLetterNames = ['エー', 'ビー', 'シー', 'ディー', 'イー', 'エフ', 'ジー', 'エイチ', 'アイ', 'ジェー', 'ケー', 'エル', 'エム', 'エヌ', 'オー', 'ピー', 'キュー', 'アール', 'エス', 'ティー', 'ユー', 'ブイ', 'ダブリュー', 'エックス', 'ワイ', 'ゼット'] + +export const ja: MathLocale = { + language: 'ja', + join: joinCompact, + number: digits => spellNumber(digits, 'ja'), + letter: letter => letterName(jaLetterNames, letter), + letterCase: (spoken, upper) => `${upper ? '大文字の' : '小文字の'}${spoken}`, + clauses: parts => parts.join('、'), + cases: items => items.map(({ value, condition }) => condition === undefined ? `それ以外のとき${value}` : `${condition}のとき${value}`).join('、'), + matrix: (rows, determinant) => `${determinant ? '行列式' : '行列'}、${rows.map((row, index) => `${spellNumber(String(index + 1), 'ja')}行目 ${row.join('、')}`).join('、')}`, + list: items => items.join('、'), + words: { + ...greek(['アルファ', 'ベータ', 'ガンマ', 'デルタ', 'イプシロン', 'イプシロン', 'ゼータ', 'イータ', 'シータ', 'シータ', 'イオタ', 'カッパ', 'ラムダ', 'ミュー', 'ニュー', 'クサイ', 'オミクロン', 'パイ', 'ロー', 'シグマ', 'タウ', 'ウプシロン', 'ファイ', 'ファイ', 'カイ', 'プサイ', 'オメガ']), + '\\nabla': 'ナブラ', + '∇': 'ナブラ', + '\\varsigma': 'シグマ', + 'M_\\odot': '太陽質量', + 'L_\\odot': '太陽光度', + 'R_\\odot': '太陽半径', + '\\checkmark': 'チェックマーク', + '\\frown': 'キャップ積', + '\\ast': 'アスタリスク', + '\\infty': '無限大', + '∞': '無限大', + '\\angle': '角', + '∠': '角', + '\\triangle': '三角形', + '△': '三角形', + '\\odot': '円', + '⊙': '円', + '\\circ': '度', + '°': '度', + '\\cdots': '以下同様', + '\\ldots': '以下同様', + '\\dots': '以下同様', + '…': '以下同様', + '\\emptyset': '空集合', + '\\varnothing': '空集合', + '∅': '空集合', + '\\forall': '任意の', + '\\exists': 'ある', + '\\nexists': '存在しない', + '∄': '存在しない', + '\\aleph': 'アレフ', + 'ℵ': 'アレフ', + '\\therefore': 'ゆえに', + '\\because': 'なぜならば', + '\\partial': 'ラウンド', + '\\sin': 'サイン', + '\\cos': 'コサイン', + '\\tan': 'タンジェント', + '\\cot': 'コタンジェント', + '\\sec': 'セカント', + '\\csc': 'コセカント', + '\\log': 'ログ', + '\\lg': 'ログ', + '\\ln': 'エルエヌ', + '\\lim': '極限', + '\\max': '最大値', + '\\min': '最小値', + '\\mid': 'ただし', + '\\%': 'パーセント', + '%': 'パーセント', + }, + operators: aliases({ '+': 'プラス', '-': 'マイナス', '\\times': 'かける', '\\cdot': 'かける', '\\div': 'わる', '\\pm': 'プラスマイナス', '\\mp': 'マイナスプラス', '\\cup': 'カップ', '\\cap': 'キャップ', '\\setminus': 'マイナス', '\\sqcup': '直和', '\\triangle': '対称差', '\\circ': '合成', '!': 'の階乗' }), + negative: 'マイナス', + positive: 'プラス', + relations: aliases({ + '=': (l, r) => `${l} イコール ${r}`, + '\\neq': (l, r) => `${l} ノットイコール ${r}`, + '<': (l, r) => `${l} 小なり ${r}`, + '>': (l, r) => `${l} 大なり ${r}`, + '\\ll': (l, r) => `${l}は${r}よりずっと小さい`, + '\\gg': (l, r) => `${l}は${r}よりずっと大きい`, + '\\leq': (l, r) => `${l} 小なりイコール ${r}`, + '\\geq': (l, r) => `${l} 大なりイコール ${r}`, + '\\approx': (l, r) => `${l} ニアリーイコール ${r}`, + // 合同 is a congruence modulo n, see `congruent`. Alone, `\\equiv` is an identity. + '\\equiv': (l, r) => `${l}は${r}と恒等的に等しい`, + '\\perp': (l, r) => `${l}と${r}は垂直`, + '\\parallel': (l, r) => `${l}と${r}は平行`, + '\\in': (l, r) => `${l}は${r}に属する`, + '\\notin': (l, r) => `${l}は${r}に属さない`, + '\\ni': (l, r) => `${l}は${r}を要素に持つ`, + '\\subseteq': (l, r) => `${l}は${r}に含まれる`, + // A Japanese textbook writes ⊂ for a subset, not a proper one. + '\\subset': (l, r) => `${l}は${r}に含まれる`, + '\\supseteq': (l, r) => `${l}は${r}を含む`, + '\\supset': (l, r) => `${l}は${r}を真に含む`, + '\\nsubseteq': (l, r) => `${l}は${r}に含まれない`, + '\\nsupseteq': (l, r) => `${l}は${r}を含まない`, + '\\sim': (l, r) => `${l}と${r}は相似`, + '\\cong': (l, r) => `${l}と${r}は合同`, + '\\to': (l, r) => `${l}を${r}に近づける`, + '\\mapsto': (l, r) => `${l}を${r}に写す`, + '\\Rightarrow': (l, r) => `${l} ならば ${r}`, + '\\Leftrightarrow': (l, r) => `${l} と ${r} は同値`, + '\\propto': (l, r) => `${l}は${r}に比例する`, + '\\gtrsim': (l, r) => `${l} 大なりニアリーイコール ${r}`, + '\\lesssim': (l, r) => `${l} 小なりニアリーイコール ${r}`, + '\\mid': (l, r) => `${l}は${r}を割り切る`, + '\\nmid': (l, r) => `${l}は${r}を割り切らない`, + '\\vdash': (l, r) => `${l}から${r}が導かれる`, + '\\coloneqq': (l, r) => `${l}を${r}と定義する`, + }), + // The denominator comes first, so a power on the numerator can bind to the whole fraction: 6分のパイの二乗. + fraction: (n, d, shape) => shape.numerator === 'compound' || shape.denominator === 'compound' || shape.numerator === 'scripted' || shape.followsOperator + ? `分子が${n}、分母が${d}の分数` + : `${d}分の${n}`, + derivative: (v, body, order) => `${body}を${v}で${order ? `${spellNumber(order, 'ja')}回` : ''}微分したもの`, + power: (b, e) => `${b}の${e}乗`, + squared: b => `${b}の二乗`, + cubed: b => `${b}の三乗`, + degrees: b => `${b}度`, + prime: b => `${b}プライム`, + superscript: (b, m) => `${b}上付き${m}`, + marks: { '\\ast': 'スター', '\\dagger': 'ダガー', '\\vee': '双対' }, + // A simple call keeps the reading of a product, as `f(x)` reads エフエックス. A grouped call says its parentheses. + apply: (f, args, grouped, decorated) => grouped || decorated ? `${f}かっこ${args.join('、')}かっことじ` : joinCompact([f, ...args]), + upTo: x => `${x}まで`, + range: (from, to) => `${from}から${to}まで`, + // 以下同様 inside a chain is heard as 以下, which means less than or equal. A chain through an ellipsis reads its range + // and the rule for neighbours instead. Other chains read as written. + chain: (operands, relations) => { + if (operands.includes(undefined) && relations.every(relation => relation === relations[0])) + return `${operands[0]}から${operands.at(-1)}まで、${relations[0]!('前の値', '次の値')}` + return relations.reduce((left, relation, index) => relation(left, operands[index + 1] ?? ''), operands[0] ?? '') + }, + placeholder: 'ドット', + numberIndex: (b, i) => `${b}${i}`, + index: (b, i) => `${b}下付き${i}`, + subscript: (b, s) => `${b}下付き${s}`, + // A compound radicand takes かっこ, as a compound group does, so ルートエックスプラスいち is only `\sqrt{x}+1`. + ordinal: value => `${value}番目`, + given: (event, condition) => `${condition}のもとでの${event}`, + modulo: m => `${m}を法として`, + congruent: (l, r, m) => `${l}と${r}は${m}を法として合同`, + inverse: name => `${name}の逆関数`, + root: (x, n, compound) => n === undefined ? (compound ? `ルートかっこ${x}かっことじ` : `ルート${x}`) : `${x}の${n}乗根`, + limit: (body, v, t, c, side) => { + if (v !== undefined && t !== undefined) + return `${v}を${side ? `${side === 'right' ? '右' : '左'}から` : ''}${t}に近づけたときの${body}の極限` + return c ? `${c}のときの${body}の極限` : `${body}の極限` + }, + norm: (x, index, squared) => `${x ? `${x}の` : ''}${index ? `${index}` : ''}ノルム${squared ? 'の二乗' : ''}`, + statistic: (kind, body, condition) => { + const noun = { Expectation: '期待値', Probability: '確率', Variance: '分散', Covariance: '共分散' }[kind] + return condition ? `${condition}としたときの${body}の${noun}` : `${body}の${noun}` + }, + about: x => `約${x}`, + distributed: (x, d) => `${x}が${d}に従う`, + arrow: (l, r) => `${l}から${r}へ`, + ceil: x => `${x}の切り上げ`, + floor: x => `${x}の切り捨て`, + tuple: items => `${items.join('、')}の組`, + bracket: x => `大かっこ${x}大かっことじ`, + mapping: (f, from, to) => `${f}は${from}から${to}への写像`, + bigOperator: (kind, body, { variable, from, to, condition, improper }) => { + const noun = { sum: '和', prod: '積', int: improper ? '広義積分' : '積分', union: '和集合', intersection: '共通部分' }[kind] + if (kind === 'int') + return from !== undefined || to !== undefined ? `${from ?? ''}から${to ?? ''}までの${body}の${noun}` : `${body}の${noun}` + if (variable !== undefined && from !== undefined) + return `${variable}が${from}から${to ?? '無限大'}までの${body}の${noun}` + return condition ? `${condition}についての${body}の${noun}` : `${body}の${noun}` + }, + absolute: x => `${x}の絶対値`, + quantity: x => `かっこ${x}かっことじ`, + // を満たす reads a condition that ends with a relation word, as 大なり ゼロ. ような after ゼロ is not a sentence. + set: (x, c) => c ? `${c} を満たす${x}の集合` : `集合${x}`, + interval: (a, b, { left, right }) => { + if (left === right) + return `${a}から${b}までの${left ? '閉' : '開'}区間` + return left ? `${a}以上${b}未満の区間` : `${a}より大きく${b}以下の区間` + }, + complement: (set, universe) => universe ? `${universe}における${set}の補集合` : `${set}の補集合`, + powerSet: set => `${set}の冪集合`, + cardinality: set => `${set}の要素の個数`, + percent: n => `${n}パーセント`, + accent: { + ket: b => `ケット${b}`, + bra: b => `ブラ${b}`, + overline: b => `${b}バー`, + bar: b => `${b}バー`, + vec: b => `ベクトル${b}`, + overrightarrow: b => `ベクトル${b}`, + hat: b => `${b}ハット`, + widehat: b => `${b}ハット`, + tilde: b => `${b}チルダ`, + dot: b => `${b}ドット`, + overarc: b => `弧${b}`, + wideparen: b => `弧${b}`, + }, + segment: name => `線分${name}`, + binomial: (n, k) => `${n}個から${k}個選ぶ組み合わせ`, + unitRatio: (n, d, power) => `${n}毎${unitPower(power, { 2: '平方', 3: '立方' })}${d}`, + celsius: value => `摂氏${value}度`, + numberSets: { 'R': '実数全体の集合', 'N': '自然数全体の集合', 'Z': '整数全体の集合', 'Q': '有理数全体の集合', 'C': '複素数全体の集合', 'N+': '正の整数全体の集合', 'N*': '正の整数全体の集合', 'Z+': '正の整数全体の集合', 'Z-': '負の整数全体の集合', 'Z*': 'ゼロでない整数全体の集合', 'R+': '正の実数全体の集合', 'R-': '負の実数全体の集合', 'R*': 'ゼロでない実数全体の集合', 'Q+': '正の有理数全体の集合' }, + units: { + prefixes: { Q: 'クエタ', R: 'ロナ', Y: 'ヨタ', Z: 'ゼタ', E: 'エクサ', P: 'ペタ', T: 'テラ', G: 'ギガ', M: 'メガ', k: 'キロ', h: 'ヘクト', da: 'デカ', d: 'デシ', c: 'センチ', m: 'ミリ', µ: 'マイクロ', n: 'ナノ', p: 'ピコ', f: 'フェムト', a: 'アト', z: 'ゼプト', y: 'ヨクト', r: 'ロント', q: 'クエクト' }, + units: { m: 'メートル', g: 'グラム', s: '秒', A: 'アンペア', K: 'ケルビン', mol: 'モル', cd: 'カンデラ', Hz: 'ヘルツ', N: 'ニュートン', Pa: 'パスカル', J: 'ジュール', W: 'ワット', C: 'クーロン', V: 'ボルト', F: 'ファラド', Ω: 'オーム', S: 'ジーメンス', Wb: 'ウェーバ', T: 'テスラ', H: 'ヘンリー', lm: 'ルーメン', lx: 'ルクス', Bq: 'ベクレル', Gy: 'グレイ', Sv: 'シーベルト', kat: 'カタール', L: 'リットル', l: 'リットル', t: 'トン', eV: '電子ボルト', Da: 'ダルトン', bar: 'バール', B: 'バイト', bit: 'ビット', cal: 'カロリー' }, + joinFactors: words => words.join(''), + plain: { min: '分', h: '時間', d: '日', ha: 'ヘクタール', au: '天文単位', Å: 'オングストローム', atm: '気圧', Rt: '直角', dB: 'デシベル', ft: 'フィート', mi: 'マイル' }, + // 時速 reads km/h as 毎時, and 時間 is a duration. + denominators: { h: '時' }, + power: (words, power) => `${power === '2' ? '平方' : '立方'}${words}`, + }, +} diff --git a/packages/speech-transformer-math/src/locales/ko.test.ts b/packages/speech-transformer-math/src/locales/ko.test.ts new file mode 100644 index 0000000000..0235190147 --- /dev/null +++ b/packages/speech-transformer-math/src/locales/ko.test.ts @@ -0,0 +1,161 @@ +import { describe, expect, it } from 'vitest' + +import { josa } from '../locale' +import { MathSpeech } from '../speech' + +const speech = new MathSpeech() +const basel = String.raw`\lim_{n \to \infty}\sum_{i=1}^{n}\frac{1}{i^2}=\frac{\pi^2}{6}` + +describe('speakMath', () => { + it('reads the Basel sum', () => { + expect(speech.speak(basel, 'ko')).toBe('엔이 무한대로 갈 때 아이가 일부터 엔까지 아이 제곱 분의 일의 합의 극한은 분자가 파이 제곱, 분모가 육인 분수') + }) + + it.each([ + [String.raw`AB \perp CD`, '에이 비는 씨 디에 수직'], + [String.raw`l_1 \parallel l_2`, '엘 아래첨자 일은 엘 아래첨자 이와 평행'], + [String.raw`a \leq b`, '에이는 비보다 작거나 같음'], + [String.raw`x \in A`, '엑스는 에이의 원소'], + [String.raw`A \subseteq B`, '에이는 비의 부분집합'], + [String.raw`\angle ABC = 90^\circ`, '각 에이 비 씨는 구십도'], + [String.raw`\triangle ABC \sim \triangle DEF`, '삼각형 에이 비 씨와 삼각형 디 E 에프는 닮음'], + [String.raw`(a+b)^2`, '괄호 에이 더하기 비 괄호 닫고 제곱'], + [String.raw`\sqrt[3]{8} = 2`, '팔의 세제곱근은 이'], + [String.raw`|x| \geq 0`, '엑스의 절댓값은 영보다 크거나 같음'], + [String.raw`\int_0^1 x^2 dx`, '영부터 일까지 엑스 제곱 디 엑스의 적분'], + [String.raw`-x^{-1}`, '마이너스 엑스의 마이너스 일 제곱'], + [String.raw`50\%`, '오십 퍼센트'], + ])('reads %s', (tex, expected) => { + expect(speech.speak(tex, 'ko')).toBe(expected) + }) + + it.each([ + [String.raw`t \in [0,1]`, '티는 영부터 일까지의 닫힌구간의 원소'], + [String.raw`x \in (0,1)`, '엑스는 영부터 일까지의 열린구간의 원소'], + [String.raw`x \in [0, 1)`, '엑스는 영 이상 일 미만인 구간의 원소'], + [String.raw`x \in \left(0, 1\right]`, '엑스는 영 초과 일 이하인 구간의 원소'], + ])('reads the interval %s', (tex, expected) => { + expect(speech.speak(tex, 'ko')).toBe(expected) + }) + + it('reads an integral with an infinite bound as an improper integral', () => { + const tex = String.raw`\int_0^{+\infty} e^{-x}\,dx` + // e keeps its Latin form in Korean, because its name 이 is also the number 2. + expect(speech.speak(tex, 'ko')).toBe('영부터 양의 무한대까지 e의 마이너스 엑스 제곱 디 엑스의 이상적분') + }) + + it.each([ + [String.raw`A \setminus B`, '에이 차집합 비'], + [String.raw`A \triangle B`, '에이 대칭차 비'], + [String.raw`A \sqcup B`, '에이 서로소 합집합 비'], + [String.raw`A^c`, '에이의 여집합'], + [String.raw`A^{\complement}`, '에이의 여집합'], + [String.raw`\complement_U A`, '유에 대한 에이의 여집합'], + [String.raw`\mathcal{P}(A)`, '에이의 멱집합'], + [String.raw`\#A`, '에이의 원소의 개수'], + [String.raw`\operatorname{card}(A)`, '에이의 원소의 개수'], + [String.raw`A \ni x`, '에이는 엑스를 원소로 가짐'], + [String.raw`A \supsetneq B`, '에이는 비를 진포함'], + [String.raw`A \nsubseteq B`, '에이는 비의 부분집합이 아님'], + [String.raw`\nexists x`, '존재하지 않는 엑스'], + [String.raw`a \ll b`, '에이는 비보다 훨씬 작음'], + [String.raw`a \gg b`, '에이는 비보다 훨씬 큼'], + ])('reads the set notation %s', (tex, expected) => { + expect(speech.speak(tex, 'ko')).toBe(expected) + }) +}) + +// The cases below come from the formula speech eval, where a StepFun voice read the rule output. +// Each one failed to reach the listener before, because the text left a choice to the voice or to the listener. +describe('speakMath readings that a voice cannot get wrong', () => { + it('spells numbers in English, Japanese, and Korean, and keeps Chinese digits', () => { + const tex = String.raw`\frac{1}{2}+\frac{1}{3}=\frac{5}{6}` + expect(speech.speak(tex, 'ko')).toBe('이 분의 일 더하기 삼 분의 일은 육 분의 오') + }) + + it.each([ + [String.raw`a \in A`, '소문자 에이는 대문자 에이의 원소'], + [String.raw`\Omega \neq \omega`, '대문자 오메가는 소문자 오메가와 같지 않음'], + ])('says the case of a letter that %s uses in both cases', (tex, expected) => { + expect(speech.speak(tex, 'ko')).toBe(expected) + }) + + it('keeps the case unsaid when each letter has one case', () => { + expect(speech.speak('F=ma', 'ko')).toBe('에프는 엠 에이') + }) + + it.each([ + [String.raw`9.8\,\mathrm{m/s^2}`, '구 점 팔 미터 매 제곱초'], + [String.raw`3\times10^{8}\,\mathrm{m/s}`, '삼 곱하기 십의 팔 제곱 미터 매 초'], + [String.raw`25^\circ\mathrm{C}`, '섭씨 이십오도'], + [String.raw`5\,\mathrm{kg}`, '오 킬로그램'], + ])('reads the unit of %s as words', (tex, expected) => { + expect(speech.speak(tex, 'ko')).toBe(expected) + }) + + it.each([ + // A compound part names the numerator and the denominator, so the listener knows where each one ends. + [String.raw`\frac{1}{1+\frac{1}{x}}`, '분자가 일, 분모가 일 더하기 분자가 일, 분모가 엑스인 분수인 분수'], + [String.raw`\frac{f-1}{2}`, '분자가 에프 빼기 일, 분모가 이인 분수'], + // A power after the second-read part could bind to the whole fraction, so it names both parts too. + [String.raw`\frac{\pi^2}{6}`, '분자가 파이 제곱, 분모가 육인 분수'], + [String.raw`\frac{6}{\pi^2}`, '파이 제곱 분의 육'], + ])('reads the fraction %s with clear bounds', (tex, expected) => { + expect(speech.speak(tex, 'ko')).toBe(expected) + }) +}) + +describe('josa', () => { + it('adds Korean particles by the final sound', () => { + expect(josa('극한', '은', '는')).toBe('극한은') + expect(josa('파이', '은', '는')).toBe('파이는') + expect(josa('n', '이', '가')).toBe('n이') + expect(josa('i', '이', '가')).toBe('i가') + expect(josa('6', '은', '는')).toBe('6은') + expect(josa('2', '은', '는')).toBe('2는') + expect(josa('무한대', '으로', '로')).toBe('무한대로') + expect(josa('l', '으로', '로')).toBe('l로') + expect(josa('n', '으로', '로')).toBe('n으로') + }) +}) + +describe('readings of the review', () => { + it.each([ + // The subscript of a name runs into its argument without 괄호: 큐 아래첨자 엠 티. + [String.raw`q_m(t)`, '큐 아래첨자 엠 괄호 티 괄호 닫고'], + [String.raw`x_1, x_2, \dots, x_n`, '엑스 아래첨자 일, 엑스 아래첨자 이부터 엑스 아래첨자 엔까지'], + [String.raw`k = 1, \dots, d`, '케이는 일부터 디까지'], + [String.raw`\sqrt{x+1}`, '루트 괄호 엑스 더하기 일 괄호 닫고'], + [String.raw`\sqrt{x}+1`, '루트 엑스 더하기 일'], + // A Korean relation ends its sentence, so a chain reads one relation for each pair, not 비보다 작음은. + [String.raw`a < b < c`, '에이는 비보다 작음, 그리고 비는 씨보다 작음'], + [String.raw`0 < x \leq 1`, '영은 엑스보다 작음, 그리고 엑스는 일보다 작거나 같음'], + [String.raw`x_1 \leq \cdots \leq x_n`, '엑스 아래첨자 일부터 엑스 아래첨자 엔까지, 앞의 값은 다음 값보다 작거나 같음'], + ])('reads %s', (tex, expected) => { + expect(speech.speak(tex, 'ko')).toBe(expected) + }) +}) + +describe('readings of the second review', () => { + it.each([ + [String.raw`f(x)=\begin{cases}1 & x>0\\0 & \text{otherwise}\end{cases}`, '에프 엑스는 일, 단 엑스는 영보다 큼; 그 외의 경우 영'], + // 일행 sounds like the number one, so the rows are 첫째 and 둘째. + [String.raw`\begin{pmatrix}1&2\\3&4\end{pmatrix}`, '행렬, 첫째 행 일, 이, 둘째 행 삼, 사'], + [String.raw`\frac{d^2 y}{dx^2}`, '와이를 엑스에 대해 이계 미분한 것'], + [String.raw`\begin{aligned}a&=b\\&=c\end{aligned}`, '에이는 비, 그리고 비는 씨'], + ])('reads %s', (tex, expected) => { + expect(speech.speak(tex, 'ko')).toBe(expected) + }) +}) + +describe('mappings of the review', () => { + it.each([ + [String.raw`\mathbb{E}[X \mid Y]`, '와이가 주어졌을 때 엑스의 기댓값'], + [String.raw`a \equiv b \pmod{n}`, '에이와 비는 법 엔에 대해 합동'], + // φ is 피, so it is not heard as π, 파이. + [String.raw`\phi + \pi`, '피 더하기 파이'], + [String.raw`A \subset B`, '에이는 비의 부분집합'], + ])('reads %s', (tex, expected) => { + expect(speech.speak(tex, 'ko')).toBe(expected) + }) +}) diff --git a/packages/speech-transformer-math/src/locales/ko.ts b/packages/speech-transformer-math/src/locales/ko.ts new file mode 100644 index 0000000000..c479805b70 --- /dev/null +++ b/packages/speech-transformer-math/src/locales/ko.ts @@ -0,0 +1,217 @@ +import type { MathLocale, Relation } from '../locale' + +import { aliases, greek, joinSpaced, josa, letterName, unitPower } from '../locale' +import { spellNumber } from '../numbers' + +/** Latin letter names, A to Z. A Korean listener names a Latin letter this way. */ +const koLetterNames = ['에이', '비', '씨', '디', '이', '에프', '지', '에이치', '아이', '제이', '케이', '엘', '엠', '엔', '오', '피', '큐', '알', '에스', '티', '유', '브이', '더블유', '엑스', '와이', '제트'] + +/** Native Korean ordinals. 일행 for row one sounds like the number one, so a row is 첫째 행. */ +const koreanOrdinals = ['첫째', '둘째', '셋째', '넷째', '다섯째', '여섯째', '일곱째', '여덟째', '아홉째', '열째'] + +export const ko: MathLocale = { + language: 'ko', + join: joinSpaced, + number: digits => spellNumber(digits, 'ko'), + // E keeps its Latin form: its name 이 is also the number 2, so `E=mc^2` would read as 2 equals m c squared. + letter: letter => letter.toUpperCase() === 'E' ? letter : letterName(koLetterNames, letter), + letterCase: (spoken, upper) => `${upper ? '대문자' : '소문자'} ${spoken}`, + clauses: parts => parts.join(', '), + // A Korean relation ends its sentence, so the condition follows its value with 단, as the set builder does. + cases: items => items.map(({ value, condition }) => condition === undefined ? `그 외의 경우 ${value}` : `${value}, 단 ${condition}`).join('; '), + matrix: (rows, determinant) => `${determinant ? '행렬식' : '행렬'}, ${rows.map((row, index) => `${koreanOrdinals[index] ?? `${spellNumber(String(index + 1), 'ko')}번째`} 행 ${row.join(', ')}`).join(', ')}`, + list: items => items.join(', '), + words: { + ...greek(['알파', '베타', '감마', '델타', '엡실론', '엡실론', '제타', '에타', '세타', '세타', '요타', '카파', '람다', '뮤', '뉴', '크사이', '오미크론', '파이', '로', '시그마', '타우', '입실론', '피', '피', '카이', '프사이', '오메가']), + '\\nabla': '나블라', + '∇': '나블라', + '\\varsigma': '시그마', + 'M_\\odot': '태양 질량', + 'L_\\odot': '태양 광도', + 'R_\\odot': '태양 반지름', + '\\checkmark': '체크 표시', + '\\frown': '캡곱', + '\\ast': '별표', + '\\infty': '무한대', + '∞': '무한대', + '\\angle': '각', + '∠': '각', + '\\triangle': '삼각형', + '△': '삼각형', + '\\odot': '원', + '⊙': '원', + '\\circ': '도', + '°': '도', + '\\cdots': '계속해서', + '\\ldots': '계속해서', + '\\dots': '계속해서', + '…': '계속해서', + '\\emptyset': '공집합', + '\\varnothing': '공집합', + '∅': '공집합', + '\\forall': '모든', + '\\exists': '어떤', + '\\nexists': '존재하지 않는', + '∄': '존재하지 않는', + '\\aleph': '알레프', + 'ℵ': '알레프', + '\\therefore': '따라서', + '\\because': '왜냐하면', + '\\partial': '라운드', + '\\sin': '사인', + '\\cos': '코사인', + '\\tan': '탄젠트', + '\\cot': '코탄젠트', + '\\sec': '시컨트', + '\\csc': '코시컨트', + '\\log': '로그', + '\\lg': '로그', + '\\ln': '자연로그', + '\\lim': '극한', + '\\max': '최댓값', + '\\min': '최솟값', + '\\mid': '단', + '\\%': '퍼센트', + '%': '퍼센트', + }, + operators: aliases({ '+': '더하기', '-': '빼기', '\\times': '곱하기', '\\cdot': '곱하기', '\\div': '나누기', '\\pm': '플러스 마이너스', '\\mp': '마이너스 플러스', '\\cup': '합집합', '\\cap': '교집합', '\\setminus': '차집합', '\\sqcup': '서로소 합집합', '\\triangle': '대칭차', '\\circ': '합성', '!': '팩토리얼' }), + negative: '마이너스', + positive: '양의', + relations: aliases({ + '=': (l, r) => `${josa(l, '은', '는')} ${r}`, + '\\neq': (l, r) => `${josa(l, '은', '는')} ${josa(r, '과', '와')} 같지 않음`, + '<': (l, r) => `${josa(l, '은', '는')} ${r}보다 작음`, + '>': (l, r) => `${josa(l, '은', '는')} ${r}보다 큼`, + '\\ll': (l, r) => `${josa(l, '은', '는')} ${r}보다 훨씬 작음`, + '\\gg': (l, r) => `${josa(l, '은', '는')} ${r}보다 훨씬 큼`, + '\\leq': (l, r) => `${josa(l, '은', '는')} ${r}보다 작거나 같음`, + '\\geq': (l, r) => `${josa(l, '은', '는')} ${r}보다 크거나 같음`, + '\\approx': (l, r) => `${josa(l, '은', '는')} 약 ${r}`, + // 합동 is a congruence modulo n, see `congruent`. Alone, `\\equiv` is an identity. + '\\equiv': (l, r) => `${josa(l, '은', '는')} ${josa(r, '과', '와')} 항등적으로 같음`, + '\\perp': (l, r) => `${josa(l, '은', '는')} ${r}에 수직`, + '\\parallel': (l, r) => `${josa(l, '은', '는')} ${josa(r, '과', '와')} 평행`, + '\\in': (l, r) => `${josa(l, '은', '는')} ${r}의 원소`, + '\\notin': (l, r) => `${josa(l, '은', '는')} ${r}의 원소가 아님`, + '\\ni': (l, r) => `${josa(l, '은', '는')} ${josa(r, '을', '를')} 원소로 가짐`, + '\\subseteq': (l, r) => `${josa(l, '은', '는')} ${r}의 부분집합`, + // A Korean textbook writes ⊂ for a subset, not a proper one. + '\\subset': (l, r) => `${josa(l, '은', '는')} ${r}의 부분집합`, + '\\supseteq': (l, r) => `${josa(l, '은', '는')} ${josa(r, '을', '를')} 포함`, + '\\supset': (l, r) => `${josa(l, '은', '는')} ${josa(r, '을', '를')} 진포함`, + '\\nsubseteq': (l, r) => `${josa(l, '은', '는')} ${r}의 부분집합이 아님`, + '\\nsupseteq': (l, r) => `${josa(l, '은', '는')} ${josa(r, '을', '를')} 포함하지 않음`, + '\\sim': (l, r) => `${josa(l, '과', '와')} ${josa(r, '은', '는')} 닮음`, + '\\cong': (l, r) => `${josa(l, '과', '와')} ${josa(r, '은', '는')} 합동`, + '\\to': (l, r) => `${josa(l, '이', '가')} ${josa(r, '으로', '로')} 감`, + '\\mapsto': (l, r) => `${josa(l, '을', '를')} ${josa(r, '으로', '로')} 대응`, + '\\Rightarrow': (l, r) => `${l} 이면 ${r}`, + '\\Leftrightarrow': (l, r) => `${josa(l, '은', '는')} ${josa(r, '과', '와')} 동치`, + '\\propto': (l, r) => `${josa(l, '은', '는')} ${r}에 비례`, + '\\gtrsim': (l, r) => `${josa(l, '은', '는')} ${r}보다 크거나 비슷함`, + '\\lesssim': (l, r) => `${josa(l, '은', '는')} ${r}보다 작거나 비슷함`, + '\\mid': (l, r) => `${josa(l, '은', '는')} ${r}의 약수`, + '\\nmid': (l, r) => `${josa(l, '은', '는')} ${r}의 약수가 아님`, + '\\vdash': (l, r) => `${l}에서 ${josa(r, '이', '가')} 유도됨`, + '\\coloneqq': (l, r) => `${josa(l, '은', '는')} ${josa(r, '으로', '로')} 정의됨`, + }), + // The denominator comes first, so a power on the numerator can bind to the whole fraction: 6 분의 파이 제곱. + fraction: (n, d, shape) => shape.numerator === 'compound' || shape.denominator === 'compound' || shape.numerator === 'scripted' || shape.followsOperator + ? `분자가 ${n}, 분모가 ${d}인 분수` + : `${d} 분의 ${n}`, + // 이계 is the Korean word for the second order, as in 이계 도함수. + derivative: (v, body, order) => `${josa(body, '을', '를')} ${v}에 대해 ${order ? `${spellNumber(order, 'ko')}계 ` : ''}미분한 것`, + power: (b, e) => `${b}의 ${e} 제곱`, + squared: b => `${b} 제곱`, + cubed: b => `${b} 세제곱`, + degrees: b => `${b}도`, + prime: b => `${b} 프라임`, + superscript: (b, m) => `${b} 위첨자 ${m}`, + marks: { '\\ast': '스타', '\\dagger': '대거', '\\vee': '쌍대' }, + // A simple call keeps the reading of a product, as `f(x)` reads 에프 엑스. A grouped call says its parentheses. + apply: (f, args, grouped, decorated) => grouped || decorated ? `${f} 괄호 ${args.join(', ')} 괄호 닫고` : joinSpaced([f, ...args]), + upTo: x => `${x}까지`, + range: (from, to) => `${from}부터 ${to}까지`, + // Each Korean relation ends its sentence, as 에이는 비보다 작음, so a chain reads one relation for each pair. A chain + // through an ellipsis reads its range and then the rule for neighbours. + chain: (operands, relations) => { + if (operands.includes(undefined) && relations.every(relation => relation === relations[0])) + return `${operands[0]}부터 ${operands.at(-1)}까지, ${relations[0]!('앞의 값', '다음 값')}` + return relations.map((relation, index) => relation(operands[index] ?? '', operands[index + 1] ?? '')).join(', 그리고 ') + }, + placeholder: '점', + // Korean keeps 아래첨자 also for a number: in 엘 이와, the number 이 is heard as part of the particle, as 엘과. + numberIndex: (b, i) => `${b} 아래첨자 ${i}`, + index: (b, i) => `${b} 아래첨자 ${i}`, + subscript: (b, s) => `${b} 아래첨자 ${s}`, + // A compound radicand takes 괄호, as a compound group does, so 루트 엑스 더하기 일 is only `\sqrt{x}+1`. + ordinal: value => `${value}번째`, + given: (event, condition) => `${josa(condition, '이', '가')} 주어졌을 때 ${event}`, + modulo: m => `법 ${m}`, + congruent: (l, r, m) => `${josa(l, '과', '와')} ${josa(r, '은', '는')} 법 ${m}에 대해 합동`, + inverse: name => `${name}의 역함수`, + root: (x, n, compound) => n === undefined ? (compound ? `루트 괄호 ${x} 괄호 닫고` : `루트 ${x}`) : n === '삼' ? `${x}의 세제곱근` : `${x}의 ${n} 제곱근`, + limit: (body, v, t, c, side) => { + if (v !== undefined && t !== undefined) + return `${josa(v, '이', '가')} ${side ? `${side === 'right' ? '오른쪽' : '왼쪽'}에서 ` : ''}${josa(t, '으로', '로')} 갈 때 ${body}의 극한` + return c ? `${c}일 때 ${body}의 극한` : `${body}의 극한` + }, + norm: (x, index, squared) => `${x ? `${x}의 ` : ''}${index ? `${index} ` : ''}노름${squared ? '의 제곱' : ''}`, + statistic: (kind, body, condition) => { + const noun = { Expectation: '기댓값', Probability: '확률', Variance: '분산', Covariance: '공분산' }[kind] + return condition ? `${condition}에 대한 ${body}의 ${noun}` : `${body}의 ${noun}` + }, + about: x => `약 ${x}`, + distributed: (x, d) => `${josa(x, '이', '가')} ${josa(d, '을', '를')} 따름`, + arrow: (l, r) => `${l}에서 ${josa(r, '으로', '로')}`, + ceil: x => `${x}의 올림`, + floor: x => `${x}의 내림`, + tuple: items => `순서쌍 ${items.join(', ')}`, + bracket: x => `대괄호 ${x} 대괄호 닫고`, + mapping: (f, from, to) => `${josa(f, '은', '는')} ${from}에서 ${josa(to, '으로', '로')}의 함수`, + bigOperator: (kind, body, { variable, from, to, condition, improper }) => { + const noun = { sum: '합', prod: '곱', int: improper ? '이상적분' : '적분', union: '합집합', intersection: '교집합' }[kind] + if (kind === 'int') + return from !== undefined || to !== undefined ? `${from ?? ''}부터 ${to ?? ''}까지 ${body}의 ${noun}` : `${body}의 ${noun}` + if (variable !== undefined && from !== undefined) + return `${josa(variable, '이', '가')} ${from}부터 ${to ?? '무한대'}까지 ${body}의 ${noun}` + return condition ? `${condition}에 대한 ${body}의 ${noun}` : `${body}의 ${noun}` + }, + absolute: x => `${x}의 절댓값`, + quantity: x => `괄호 ${x} 괄호 닫고`, + set: (x, c) => c ? `${x}의 집합 단 ${c}` : `집합 ${x}`, + interval: (a, b, { left, right }) => { + if (left === right) + return `${a}부터 ${b}까지의 ${left ? '닫힌구간' : '열린구간'}` + return left ? `${a} 이상 ${b} 미만인 구간` : `${a} 초과 ${b} 이하인 구간` + }, + complement: (set, universe) => universe ? `${universe}에 대한 ${set}의 여집합` : `${set}의 여집합`, + powerSet: set => `${set}의 멱집합`, + cardinality: set => `${set}의 원소의 개수`, + percent: n => `${n} 퍼센트`, + accent: { + ket: b => `켓 ${b}`, + bra: b => `브라 ${b}`, + overline: b => `${b} 바`, + bar: b => `${b} 바`, + vec: b => `벡터 ${b}`, + overrightarrow: b => `벡터 ${b}`, + hat: b => `${b} 햇`, + widehat: b => `${b} 햇`, + tilde: b => `${b} 틸데`, + dot: b => `${b} 닷`, + overarc: b => `호 ${b}`, + wideparen: b => `호 ${b}`, + }, + segment: name => `선분 ${name}`, + binomial: (n, k) => `${n} 개 중 ${k} 개를 고르는 조합`, + unitRatio: (n, d, power) => `${n} 매 ${unitPower(power, { 2: '제곱', 3: '세제곱' })}${d}`, + celsius: value => `섭씨 ${value}도`, + numberSets: { 'R': '실수 전체의 집합', 'N': '자연수 전체의 집합', 'Z': '정수 전체의 집합', 'Q': '유리수 전체의 집합', 'C': '복소수 전체의 집합', 'N+': '양의 정수 전체의 집합', 'N*': '양의 정수 전체의 집합', 'Z+': '양의 정수 전체의 집합', 'Z-': '음의 정수 전체의 집합', 'Z*': '영이 아닌 정수 전체의 집합', 'R+': '양의 실수 전체의 집합', 'R-': '음의 실수 전체의 집합', 'R*': '영이 아닌 실수 전체의 집합', 'Q+': '양의 유리수 전체의 집합' }, + units: { + prefixes: { Q: '퀘타', R: '론나', Y: '요타', Z: '제타', E: '엑사', P: '페타', T: '테라', G: '기가', M: '메가', k: '킬로', h: '헥토', da: '데카', d: '데시', c: '센티', m: '밀리', µ: '마이크로', n: '나노', p: '피코', f: '펨토', a: '아토', z: '젭토', y: '욕토', r: '론토', q: '퀙토' }, + units: { m: '미터', g: '그램', s: '초', A: '암페어', K: '켈빈', mol: '몰', cd: '칸델라', Hz: '헤르츠', N: '뉴턴', Pa: '파스칼', J: '줄', W: '와트', C: '쿨롬', V: '볼트', F: '패럿', Ω: '옴', S: '지멘스', Wb: '웨버', T: '테슬라', H: '헨리', lm: '루멘', lx: '럭스', Bq: '베크렐', Gy: '그레이', Sv: '시버트', kat: '카탈', L: '리터', l: '리터', t: '톤', eV: '전자볼트', Da: '돌턴', bar: '바', B: '바이트', bit: '비트', cal: '칼로리' }, + plain: { min: '분', h: '시간', d: '일', ha: '헥타르', au: '천문단위', Å: '옹스트롬', atm: '기압', Rt: '직각', dB: '데시벨', ft: '피트', mi: '마일' }, + power: (words, power) => `${power === '2' ? '제곱' : '세제곱'}${words}`, + }, +} diff --git a/packages/speech-transformer-math/src/locales/zh.test.ts b/packages/speech-transformer-math/src/locales/zh.test.ts new file mode 100644 index 0000000000..75e6e55923 --- /dev/null +++ b/packages/speech-transformer-math/src/locales/zh.test.ts @@ -0,0 +1,190 @@ +import { describe, expect, it } from 'vitest' + +import { MathSpeech } from '../speech' + +const speech = new MathSpeech() +const basel = String.raw`\lim_{n \to \infty}\sum_{i=1}^{n}\frac{1}{i^2}=\frac{\pi^2}{6}` + +describe('speakMath', () => { + it('reads the Basel sum', () => { + expect(speech.speak(basel, 'zh')).toBe('当n趋于无穷大时 i从1到n对i的平方分之1求和的极限 等于 分子是派的平方 分母是6的分数') + }) + + it.each([ + [String.raw`AB \perp CD`, 'A B 垂直于 C D'], + [String.raw`l_1 \parallel l_2`, 'l1 平行于 l2'], + [String.raw`a \leq b`, 'a 小于等于 b'], + [String.raw`x \in A`, 'x 属于 A'], + [String.raw`A \subseteq B`, 'A 包含于 B'], + [String.raw`\angle ABC = 90^\circ`, '角A B C 等于 90度'], + [String.raw`\triangle ABC \sim \triangle DEF`, '三角形A B C 相似于 三角形D E F'], + [String.raw`(a+b)^2`, 'a与b的和的平方'], + [String.raw`\sqrt[3]{8} = 2`, '8的立方根 等于 2'], + [String.raw`|x| \geq 0`, 'x的绝对值 大于等于 0'], + [String.raw`\int_0^1 x^2 dx`, '从0到1对x的平方d x积分'], + [String.raw`-x^{-1}`, '负x的负1次方'], + [String.raw`50\%`, '百分之50'], + ])('reads %s', (tex, expected) => { + expect(speech.speak(tex, 'zh')).toBe(expected) + }) + + it.each([ + [String.raw`t \in [0,1]`, 't 属于 闭区间0到1'], + [String.raw`x \in (0,1)`, 'x 属于 开区间0到1'], + [String.raw`x \in [0, 1)`, 'x 属于 左闭右开区间0到1'], + [String.raw`x \in \left(0, 1\right]`, 'x 属于 左开右闭区间0到1'], + ])('reads the interval %s', (tex, expected) => { + expect(speech.speak(tex, 'zh')).toBe(expected) + }) + + it('reads a pair as an interval only when a set symbol or an infinite bound shows it', () => { + expect(speech.speak(String.raw`(-\infty, 0) \cup (1, +\infty)`, 'zh')).toBe('开区间负无穷大到0并开区间1到正无穷大') + // A point, or the arguments of a function, keep the plain reading. + expect(speech.speak(String.raw`P(1, 2)`, 'zh')).toBe('P括号1，2括号') + }) + + it('reads parentheses that a neighbor binds to, so the grouping is not lost', () => { + // Silent brackets turned v - (x - epsilon) into v - x - epsilon, and (1-t) epsilon into 1 - t epsilon. + expect(speech.speak(String.raw`x_t=(1-t)\epsilon+tx`, 'zh')).toBe('x下标t 等于 1与t的差乘以艾普西隆加t x') + expect(speech.speak(String.raw`v-(x-\epsilon)`, 'zh')).toBe('v减x与艾普西隆的差') + expect(speech.speak(String.raw`(x+1)/2`, 'zh')).toBe('x与1的和除以2') + }) + + it('reads like a Chinese teacher', () => { + expect(speech.speak(String.raw`\sqrt{a^2+b^2}`, 'zh')).toBe('根号下a的平方加b的平方') + expect(speech.speak(String.raw`\sqrt{2}`, 'zh')).toBe('根号2') + expect(speech.speak(String.raw`\left(1+\frac{1}{n}\right)^n`, 'zh')).toBe('1与n分之1的和的n次方') + expect(speech.speak(String.raw`(a, b, c)`, 'zh')).toBe('3元组a、b、c') + expect(speech.speak(String.raw`\|x\|_2`, 'zh')).toBe('x的2范数') + expect(speech.speak(String.raw`K^\dagger`, 'zh')).toBe('K共轭转置') + expect(speech.speak(String.raw`\theta^*`, 'zh')).toBe('西塔星') + // A call with a decorated name says its brackets in Chinese, so it is not heard as a product. + expect(speech.speak(String.raw`\hat{H}(t)`, 'zh')).toBe('H帽括号t括号') + expect(speech.speak(String.raw`f(x)`, 'zh')).toBe('f x') + // Scientific notation is a power of ten, not Euler's number. + expect(speech.speak(String.raw`1e-3`, 'zh')).toBe('1乘10的负3次方') + }) + + it('reads an integral with an infinite bound as an improper integral', () => { + const tex = String.raw`\int_0^{+\infty} e^{-x}\,dx` + expect(speech.speak(tex, 'zh')).toBe('从0到正无穷大对e的负x次方d x的反常积分') + }) + + it.each([ + [String.raw`A \setminus B`, 'A减B'], + [String.raw`A \triangle B`, 'A对称差B'], + [String.raw`A \sqcup B`, 'A不交并B'], + [String.raw`A^c`, 'A的补集'], + [String.raw`A^{\complement}`, 'A的补集'], + [String.raw`\complement_U A`, 'A在U中的补集'], + [String.raw`\mathcal{P}(A)`, 'A的幂集'], + [String.raw`\#A`, 'A的元素个数'], + [String.raw`\operatorname{card}(A)`, 'A的元素个数'], + [String.raw`A \ni x`, 'A 包含元素 x'], + [String.raw`A \supsetneq B`, 'A 真包含 B'], + [String.raw`A \nsubseteq B`, 'A 不包含于 B'], + [String.raw`\nexists x`, '不存在x'], + [String.raw`a \ll b`, 'a 远小于 b'], + [String.raw`a \gg b`, 'a 远大于 b'], + ])('reads the set notation %s', (tex, expected) => { + expect(speech.speak(tex, 'zh')).toBe(expected) + }) + + it('keeps a power with a lowercase base, and a triangle that starts an operand', () => { + expect(speech.speak(String.raw`x^c`, 'zh')).toBe('x的c次方') + expect(speech.speak(String.raw`\triangle ABC`, 'zh')).toBe('三角形A B C') + }) + + it('reads an unknown command by its name and does not fail on unbalanced input', () => { + expect(speech.speak('}{', 'zh')).toBe('') + }) +}) + +// The cases below come from the formula speech eval, where a StepFun voice read the rule output. +// Each one failed to reach the listener before, because the text left a choice to the voice or to the listener. +describe('speakMath readings that a voice cannot get wrong', () => { + it('spells numbers in English, Japanese, and Korean, and keeps Chinese digits', () => { + const tex = String.raw`\frac{1}{2}+\frac{1}{3}=\frac{5}{6}` + expect(speech.speak(tex, 'zh')).toBe('2分之1加3分之1 等于 6分之5') + }) + + it.each([ + [String.raw`a \in A`, '小a 属于 大A'], + [String.raw`\Omega \neq \omega`, '大写欧米伽 不等于 小写欧米伽'], + ])('says the case of a letter that %s uses in both cases', (tex, expected) => { + expect(speech.speak(tex, 'zh')).toBe(expected) + }) + + it.each([ + [String.raw`9.8\,\mathrm{m/s^2}`, '9.8米每平方秒'], + [String.raw`3\times10^{8}\,\mathrm{m/s}`, '3乘10的8次方米每秒'], + [String.raw`25^\circ\mathrm{C}`, '25摄氏度'], + [String.raw`5\,\mathrm{kg}`, '5公斤'], + ])('reads the unit of %s as words', (tex, expected) => { + expect(speech.speak(tex, 'zh')).toBe(expected) + }) + + it.each([ + // A compound part names the numerator and the denominator, so the listener knows where each one ends. + [String.raw`\frac{1}{1+\frac{1}{x}}`, '分子是1 分母是1加分子是1 分母是x的分数的分数'], + [String.raw`\frac{f-1}{2}`, '分子是f减1 分母是2的分数'], + // A power after the second-read part could bind to the whole fraction, so it names both parts too. + [String.raw`\frac{\pi^2}{6}`, '分子是派的平方 分母是6的分数'], + [String.raw`\frac{6}{\pi^2}`, '派的平方分之6'], + ])('reads the fraction %s with clear bounds', (tex, expected) => { + expect(speech.speak(tex, 'zh')).toBe(expected) + }) + + it('reads a derivative, a one-sided limit, a subscript, and a function signature', () => { + expect(speech.speak(String.raw`\frac{d}{dx}x^{n}=nx^{n-1}`, 'zh')).toBe('x的n次方对x求导 等于 n x的n减1次方') + expect(speech.speak(String.raw`\lim_{x \to 0^{+}}\ln x=-\infty`, 'zh')).toBe('当x从右侧趋于0时 ln x的极限 等于 负无穷大') + }) +}) + +describe('readings of the review', () => { + it('says the brackets of a call whose name has a subscript', () => { + expect(speech.speak(String.raw`q_m(t)`, 'zh')).toBe('q下标m括号t括号') + }) +}) + +describe('readings of the second review', () => { + it.each([ + // `\ln` before parentheses calls them. It does not multiply the sum. + [String.raw`\ln(x+1)`, 'ln括号x加1括号'], + [String.raw`f^{-1}(x)`, 'f的反函数括号x括号'], + [String.raw`\frac{d^2 y}{dx^2}`, 'y对x求二阶导'], + [String.raw`f(x)=\begin{cases}1 & x>0\\0 & \text{otherwise}\end{cases}`, 'f x 等于 当x 大于 0时为1，其他情况下为0'], + [String.raw`\begin{pmatrix}1&2\\3&4\end{pmatrix}`, '矩阵，第1行是1、2，第2行是3、4'], + [String.raw`\mathrm{Rt}\triangle ABC`, '直角三角形A B C'], + [String.raw`5\,\mathrm{kg\cdot m^2}`, '5公斤平方米'], + [String.raw`100\,\mathrm{hPa}`, '100百帕'], + ])('reads %s', (tex, expected) => { + expect(speech.speak(tex, 'zh')).toBe(expected) + }) +}) + +describe('mappings of the review', () => { + it.each([ + [String.raw`P(A \mid B)`, 'P B条件下的A'], + [String.raw`a \mid b`, 'a 整除 b'], + [String.raw`R = 10\,\Omega`, 'R 等于 10欧姆'], + [String.raw`\mathbb{N}^{+}`, '正整数集'], + // 恒等于 is an identity. A congruence modulo n is 同余. + [String.raw`a \equiv b \pmod{n}`, 'a 与 b 模 n 同余'], + ])('reads %s', (tex, expected) => { + expect(speech.speak(tex, 'zh')).toBe(expected) + }) +}) + +describe('readings of the streaming review', () => { + it.each([ + // A number under a letter is said after it, as a teacher says x0. A letter keeps 下标, so x下标t is not x times t. + // 乘以 after a group: an ASR heard 差乘 as 叉乘, the cross product. + [String.raw`x_t=(1-t)x_0+tx_1`, 'x下标t 等于 1与t的差乘以x0加t x1'], + // Several arguments take 关于, without bracket words or a comma that the TTS chunker could cut at. + [String.raw`\frac{dx_t}{dt}=v_\theta(x_t,t)`, 'x下标t对t求导 等于 v下标西塔关于x下标t和t'], + [String.raw`P(1, 2)`, 'P括号1，2括号'], + ])('reads %s', (tex, expected) => { + expect(speech.speak(tex, 'zh')).toBe(expected) + }) +}) diff --git a/packages/speech-transformer-math/src/locales/zh.ts b/packages/speech-transformer-math/src/locales/zh.ts new file mode 100644 index 0000000000..f8740412fd --- /dev/null +++ b/packages/speech-transformer-math/src/locales/zh.ts @@ -0,0 +1,218 @@ +import type { MathLocale, Relation } from '../locale' + +import { aliases, greek, joinCompact, unitPower } from '../locale' + +const chineseOrders: Readonly> = { 2: '二', 3: '三', 4: '四' } + +export const zh: MathLocale = { + language: 'zh', + join: joinCompact, + // Chinese voices read digits in Chinese, so the digits stay. + number: digits => digits, + letter: letter => letter, + // A Latin letter takes the short form 大A and 小a, as a teacher says it. A Greek name takes 大写 and 小写. + letterCase: (spoken, upper) => /^[a-z]$/i.test(spoken) ? `${upper ? '大' : '小'}${spoken}` : `${upper ? '大写' : '小写'}${spoken}`, + clauses: parts => parts.join('，'), + cases: items => items.map(({ value, condition }) => condition === undefined ? `其他情况下为${value}` : `当${condition}时为${value}`).join('，'), + matrix: (rows, determinant) => `${determinant ? '行列式' : '矩阵'}，${rows.map((row, index) => `第${index + 1}行是${row.join('、')}`).join('，')}`, + list: items => items.join('、'), + words: { + ...greek(['阿尔法', '贝塔', '伽马', '德尔塔', '艾普西隆', '艾普西隆', '泽塔', '伊塔', '西塔', '西塔', '约塔', '卡帕', '兰姆达', '缪', '纽', '克西', '欧米克戎', '派', '柔', '西格玛', '陶', '宇普西隆', '斐', '斐', '希', '普西', '欧米伽']), + '\\nabla': '那勃勒', + '∇': '那勃勒', + '\\varsigma': '西格玛', + 'M_\\odot': '太阳质量', + 'L_\\odot': '太阳光度', + 'R_\\odot': '太阳半径', + '\\checkmark': '对勾', + '\\frown': '卡积', + '\\ast': '星号', + '\\infty': '无穷大', + '∞': '无穷大', + '\\angle': '角', + '∠': '角', + '\\triangle': '三角形', + '△': '三角形', + '\\odot': '圆', + '⊙': '圆', + '\\circ': '度', + '°': '度', + '\\cdots': '依此类推', + '\\ldots': '依此类推', + '\\dots': '依此类推', + '…': '依此类推', + '\\emptyset': '空集', + '\\varnothing': '空集', + '∅': '空集', + '\\forall': '对任意', + '\\exists': '存在', + '\\nexists': '不存在', + '∄': '不存在', + '\\aleph': '阿列夫', + 'ℵ': '阿列夫', + '\\therefore': '所以', + '\\because': '因为', + '\\partial': '偏', + '\\sin': 'sine', + '\\cos': 'cosine', + '\\tan': 'tangent', + '\\cot': 'cotangent', + '\\sec': 'secant', + '\\csc': 'cosecant', + '\\log': 'log', + '\\lg': 'lg', + '\\ln': 'ln', + '\\lim': '极限', + '\\max': '最大值', + '\\min': '最小值', + '\\mid': '满足', + '\\%': '百分号', + '%': '百分号', + }, + operators: aliases({ '+': '加', '-': '减', '\\times': '乘', '\\cdot': '乘', '\\div': '除以', '\\pm': '正负', '\\mp': '负正', '\\cup': '并', '\\cap': '交', '\\setminus': '减', '\\sqcup': '不交并', '\\triangle': '对称差', '\\circ': '复合', '!': '的阶乘' }), + negative: '负', + positive: '正', + relations: aliases({ + '=': (l, r) => `${l} 等于 ${r}`, + '\\neq': (l, r) => `${l} 不等于 ${r}`, + '<': (l, r) => `${l} 小于 ${r}`, + '>': (l, r) => `${l} 大于 ${r}`, + '\\ll': (l, r) => `${l} 远小于 ${r}`, + '\\gg': (l, r) => `${l} 远大于 ${r}`, + '\\leq': (l, r) => `${l} 小于等于 ${r}`, + '\\geq': (l, r) => `${l} 大于等于 ${r}`, + '\\approx': (l, r) => `${l} 约等于 ${r}`, + '\\equiv': (l, r) => `${l} 恒等于 ${r}`, + '\\perp': (l, r) => `${l} 垂直于 ${r}`, + '\\parallel': (l, r) => `${l} 平行于 ${r}`, + '\\in': (l, r) => `${l} 属于 ${r}`, + '\\notin': (l, r) => `${l} 不属于 ${r}`, + '\\ni': (l, r) => `${l} 包含元素 ${r}`, + '\\subseteq': (l, r) => `${l} 包含于 ${r}`, + '\\subset': (l, r) => `${l} 真包含于 ${r}`, + '\\supseteq': (l, r) => `${l} 包含 ${r}`, + '\\supset': (l, r) => `${l} 真包含 ${r}`, + '\\nsubseteq': (l, r) => `${l} 不包含于 ${r}`, + '\\nsupseteq': (l, r) => `${l} 不包含 ${r}`, + '\\sim': (l, r) => `${l} 相似于 ${r}`, + '\\cong': (l, r) => `${l} 全等于 ${r}`, + '\\to': (l, r) => `${l} 趋于 ${r}`, + '\\mapsto': (l, r) => `${l} 映射到 ${r}`, + '\\Rightarrow': (l, r) => `${l} 推出 ${r}`, + '\\Leftrightarrow': (l, r) => `${l} 等价于 ${r}`, + '\\propto': (l, r) => `${l} 正比于 ${r}`, + '\\gtrsim': (l, r) => `${l} 大于或约等于 ${r}`, + '\\lesssim': (l, r) => `${l} 小于或约等于 ${r}`, + '\\mid': (l, r) => `${l} 整除 ${r}`, + '\\nmid': (l, r) => `${l} 不整除 ${r}`, + '\\vdash': (l, r) => `由 ${l} 可推出 ${r}`, + '\\coloneqq': (l, r) => `${l} 定义为 ${r}`, + }), + // The denominator comes first, so a power on the numerator can bind to the whole fraction: 6分之派的平方. + fraction: (n, d, shape) => shape.numerator === 'compound' || shape.denominator === 'compound' || shape.numerator === 'scripted' || shape.followsOperator + ? `分子是${n} 分母是${d}的分数` + : `${d}分之${n}`, + // A textbook says 二阶导 with a Chinese numeral, not 2阶导. + derivative: (v, body, order) => `${body}对${v}求${order ? `${chineseOrders[order] ?? order}阶` : ''}导`, + power: (b, e) => `${b}的${e}次方`, + squared: b => `${b}的平方`, + cubed: b => `${b}的立方`, + degrees: b => `${b}度`, + prime: b => `${b}撇`, + superscript: (b, m) => `${b}上标${m}`, + marks: { '\\ast': '星', '\\dagger': '共轭转置', '\\vee': '对偶' }, + // A simple call keeps the reading of a product, as Chinese teachers read `f(x)`. A grouped call says its parentheses. + // A function with several arguments says 关于 between its name and the arguments, as in v下标西塔关于x下标t和t. + // Bracket words are heavy, and a comma between the arguments let the TTS chunker cut the formula in two requests. + // A point, as `P(1, 2)`, keeps its brackets: its name is one capital letter. A decorated name or an argument with an + // operator keeps them too, so `\hat H(t)` is not heard as a product and `f(x+1)` is not heard as `f(x) + 1`. + apply: (f, args, grouped, decorated) => { + if (args.length > 1 && !/^[A-Z]$/.test(f)) + return `${f}关于${args.join('和')}` + return grouped || decorated ? `${f}括号${args.join('，')}括号` : joinCompact([f, ...args]) + }, + upTo: x => `直到${x}`, + placeholder: '点', + groupTimes: '乘以', + numberIndex: (b, i) => `${b}${i}`, + index: (b, i) => `${b}下标${i}`, + subscript: (b, s) => `${b}下标${s}`, + // 根号下 marks a compound radicand, so 根号下a的平方加b的平方 is not heard as the square of 根号a. + ordinal: value => `第${value}`, + given: (event, condition) => `${condition}条件下的${event}`, + modulo: m => `模${m}`, + congruent: (l, r, m) => `${l} 与 ${r} 模 ${m} 同余`, + inverse: name => `${name}的反函数`, + root: (x, n, compound) => n === undefined ? `根号${compound ? '下' : ''}${x}` : n === '3' ? `${x}的立方根` : `${x}的${n}次方根`, + limit: (body, v, t, c, side) => { + if (v !== undefined && t !== undefined) + return side ? `当${v}从${side === 'right' ? '右' : '左'}侧趋于${t}时 ${body}的极限` : `当${v}趋于${t}时 ${body}的极限` + return c ? `${c}时 ${body}的极限` : `${body}的极限` + }, + norm: (x, index, squared) => `${x ? `${x}的` : ''}${index ?? ''}范数${squared ? '的平方' : ''}`, + statistic: (kind, body, condition) => { + const noun = { Expectation: '期望', Probability: '概率', Variance: '方差', Covariance: '协方差' }[kind] + return condition ? `在${condition}下 ${body}的${noun}` : `${body}的${noun}` + }, + about: x => `约${x}`, + distributed: (x, d) => `${x}服从${d}`, + arrow: (l, r) => `${l} 到 ${r}`, + ceil: x => `${x}向上取整`, + floor: x => `${x}向下取整`, + tuple: items => `${items.length === 2 ? '有序对' : `${items.length}元组`}${items.join('、')}`, + bracket: x => `方括号${x}方括号`, + mapping: (f, from, to) => `${f} 是从${from}到${to}的映射`, + bigOperator: (kind, body, { variable, from, to, condition, improper }) => { + const verb = { sum: '求和', prod: '求积', int: '积分', union: '求并集', intersection: '求交集' }[kind] + if (kind === 'int' && improper) + return `从${from ?? ''}到${to ?? ''}对${body}的反常积分` + if (kind === 'int') + return from !== undefined || to !== undefined ? `从${from ?? ''}到${to ?? ''}对${body}积分` : `对${body}积分` + if (variable !== undefined && from !== undefined) + return `${variable}从${from}到${to ?? '无穷大'}对${body}${verb}` + return condition ? `${condition}时对${body}${verb}` : `对${body}${verb}` + }, + absolute: x => `${x}的绝对值`, + quantity: x => `括号${x}括号`, + // A Chinese teacher reads `(a+b)^2` as a 与 b 的和的平方, without the brackets. + groupOf: { sum: (a, b) => `${a}与${b}的和`, difference: (a, b) => `${a}与${b}的差` }, + set: (x, c) => c ? `满足${c}的${x}的集合` : `集合${x}`, + interval: (a, b, { left, right }) => { + if (left === right) + return `${left ? '闭' : '开'}区间${a}到${b}` + return `${left ? '左闭右开' : '左开右闭'}区间${a}到${b}` + }, + complement: (set, universe) => universe ? `${set}在${universe}中的补集` : `${set}的补集`, + powerSet: set => `${set}的幂集`, + cardinality: set => `${set}的元素个数`, + percent: n => `百分之${n}`, + accent: { + ket: b => `ket ${b}`, + bra: b => `bra ${b}`, + overline: b => `${b}拔`, + bar: b => `${b}拔`, + vec: b => `向量${b}`, + overrightarrow: b => `向量${b}`, + hat: b => `${b}帽`, + widehat: b => `${b}帽`, + tilde: b => `${b}波浪`, + dot: b => `${b}点`, + overarc: b => `弧${b}`, + wideparen: b => `弧${b}`, + }, + segment: name => `线段${name}`, + binomial: (n, k) => `从${n}中取${k}的组合数`, + unitRatio: (n, d, power) => `${n}每${unitPower(power, { 2: '平方', 3: '立方' })}${d}`, + celsius: value => `${value}摄氏度`, + numberSets: { 'R': '实数集', 'N': '自然数集', 'Z': '整数集', 'Q': '有理数集', 'C': '复数集', 'N+': '正整数集', 'N*': '正整数集', 'Z+': '正整数集', 'Z-': '负整数集', 'Z*': '非零整数集', 'R+': '正实数集', 'R-': '负实数集', 'R*': '非零实数集', 'Q+': '正有理数集' }, + units: { + prefixes: { Q: '昆它', R: '容那', Y: '尧它', Z: '泽它', E: '艾可萨', P: '拍它', T: '太', G: '吉', M: '兆', k: '千', h: '百', da: '十', d: '分', c: '厘', m: '毫', µ: '微', n: '纳', p: '皮', f: '飞', a: '阿', z: '仄', y: '幺', r: '柔', q: '亏' }, + units: { m: '米', g: '克', s: '秒', A: '安培', K: '开尔文', mol: '摩尔', cd: '坎德拉', Hz: '赫兹', N: '牛顿', Pa: '帕斯卡', J: '焦耳', W: '瓦特', C: '库仑', V: '伏特', F: '法拉', Ω: '欧姆', S: '西门子', Wb: '韦伯', T: '特斯拉', H: '亨利', lm: '流明', lx: '勒克斯', Bq: '贝可勒尔', Gy: '戈瑞', Sv: '希沃特', kat: '开特', L: '升', l: '升', t: '吨', eV: '电子伏特', Da: '道尔顿', bar: '巴', B: '字节', bit: '比特', cal: '卡' }, + joinFactors: words => words.join(''), + plain: { min: '分钟', h: '小时', d: '天', ha: '公顷', au: '天文单位', Å: '埃', atm: '标准大气压', Rt: '直角', dB: '分贝', ft: '英尺', mi: '英里' }, + // 公斤, not 千克: an ASR model writes 5千克 as 5000克. + // 百帕 is what a weather report says. The composed 百帕斯卡 after 100 sounds like 一百百帕斯卡. + overrides: { kg: '公斤', hPa: '百帕' }, + power: (words, power) => `${power === '2' ? '平方' : '立方'}${words}`, + }, +} diff --git a/packages/speech-transformer-math/src/lower.test.ts b/packages/speech-transformer-math/src/lower.test.ts new file mode 100644 index 0000000000..f444d34e10 --- /dev/null +++ b/packages/speech-transformer-math/src/lower.test.ts @@ -0,0 +1,112 @@ +import { headOf } from '@proj-airi/mathjson' +import { describe, expect, it } from 'vitest' + +import { lowerMath } from './lower' +import { parseMath } from './parse' + +const lower = (tex: string) => lowerMath(parseMath(tex)) + +describe('lowerMath', () => { + it('lowers relations, operators, and juxtaposition, with multiplication first', () => { + expect(lower('a+b c=d')).toEqual(['Equal', ['Add', 'a', ['InvisibleOperator', 'b', 'c']], 'd']) + expect(lower(String.raw`a+b\times c`)).toEqual(['Add', 'a', ['Multiply', 'b', 'c']]) + expect(lower('-x')).toEqual(['Negate', 'x']) + expect(lower(String.raw`x=\frac{-b\pm 1}{2a}`)).toEqual(['Equal', 'x', ['Divide', ['PlusMinus', ['Negate', 'b'], { num: '1' }], ['InvisibleOperator', { num: '2' }, 'a']]]) + }) + + it('keeps what the writer wrote: percent, scientific notation, and names of geometry', () => { + expect(lower(String.raw`50\%`)).toEqual(['Percent', { num: '50' }]) + expect(lower(String.raw`3\times10^{8}`)).toEqual(['Multiply', { num: '3' }, ['Power', { num: '10' }, { num: '8' }]]) + expect(lower(String.raw`AB \perp CD`)).toEqual(['Perpendicular', ['Name', 'A', 'B'], ['Name', 'C', 'D']]) + expect(lower(String.raw`\overline{AB}`)).toEqual(['Segment', 'A', 'B']) + }) + + it('lowers the meaning of derivatives, one-sided limits, and big operators', () => { + expect(lower(String.raw`\frac{d}{dx}x^n`)).toEqual(['D', ['Power', 'x', 'n'], 'x']) + expect(lower(String.raw`\lim_{x \to 0^{+}} f`)).toEqual(['Limit', 'f', ['Bounds', 'x', { num: '0' }, { num: '1' }]]) + expect(lower(String.raw`\sum_{k=1}^{n} k`)).toEqual(['Sum', 'k', ['Bounds', 'k', { num: '1' }, 'n']]) + expect(lower(String.raw`\int_0^1 x`)).toEqual(['Integrate', 'x', ['Bounds', 'Nothing', { num: '0' }, { num: '1' }]]) + }) + + it('lowers delimiters by their role', () => { + expect(lower(String.raw`x \in (0, 1]`)).toEqual(['Element', 'x', ['Interval', ['Open', { num: '0' }], { num: '1' }]]) + // A pair after a name is the arguments of a call, not an interval and not a product. + expect(lower('P(1, 2)')).toEqual(['Apply', 'P', { num: '1' }, { num: '2' }]) + expect(lower('n(n+1)')).toEqual(['InvisibleOperator', 'n', ['Group', ['Add', 'n', { num: '1' }]]]) + expect(lower(String.raw`\{x \mid x > 0\}`)).toEqual(['Set', 'x', ['Condition', ['Greater', 'x', { num: '0' }]]]) + }) + + it('lowers a letter before parentheses as a call or as a product', () => { + // No operator inside: a call. A writer multiplies two letters as `xt`, not `x(t)`. + expect(lower('x(t)')).toEqual(['Apply', 'x', 't']) + // A comma: the arguments of a call, because a product has no comma. + expect(lower('H_k(M, M-N)')).toEqual(['Apply', ['Subscript', 'H', 'k'], 'M', ['Subtract', 'M', 'N']]) + // An operator inside: a call after a decorated name or a function letter, a product after other letters. + expect(lower('H_1(S-P)')).toEqual(['Apply', ['Subscript', 'H', { num: '1' }], ['Subtract', 'S', 'P']]) + expect(lower('f(x+1)')).toEqual(['Apply', 'f', ['Add', 'x', { num: '1' }]]) + expect(lower('a(b+c)')).toEqual(['InvisibleOperator', 'a', ['Group', ['Add', 'b', 'c']]]) + // A number power is a power, so the parentheses multiply. + expect(lower('x^2(x+1)')).toEqual(['InvisibleOperator', ['Power', 'x', { num: '2' }], ['Group', ['Add', 'x', { num: '1' }]]]) + // e is a constant, never a function. + expect(lower(String.raw`e^{x}(1+x)`)).toEqual(['InvisibleOperator', ['Power', 'e', 'x'], ['Group', ['Add', { num: '1' }, 'x']]]) + // The last capital of a geometry name is not a function. + expect(lower('AB(x)')).toEqual(['InvisibleOperator', ['Name', 'A', 'B'], ['Delimiter', 'x']]) + // A centered dot alone is an empty slot, and scripts after the parentheses apply to the call. + expect(lower(String.raw`p(\cdot)`)).toEqual(['Apply', 'p', ['Placeholder']]) + expect(lower('f(x)^2')).toEqual(['Power', ['Apply', 'f', 'x'], { num: '2' }]) + expect(lower(String.raw`\bar{x}(t)`)).toEqual(['Apply', ['Accent', '\'bar\'', 'x'], 't']) + }) + + it('reads an arrow between structures as a map and an arrow toward a value as a limit', () => { + expect(lower(String.raw`n \to \infty`)).toEqual(['To', 'n', '\\infty']) + expect(lower(String.raw`f(x) \to L`)).toEqual(['To', ['Apply', 'f', 'x'], 'L']) + expect(lower(String.raw`X \to Y`)).toEqual(['Arrow', 'X', 'Y']) + expect(lower(String.raw`H_k(M) \to H_{k-1}(M)`)).toEqual(['Arrow', ['Apply', ['Subscript', 'H', 'k'], 'M'], ['Apply', ['Subscript', 'H', ['Subtract', 'k', { num: '1' }]], 'M']]) + }) + + it('keeps the colon of \\colon, so a signature reads as one', () => { + // KaTeX expands `\colon` to `{:}` between kerns. + expect(lower(String.raw`p \colon X \to Y`)).toEqual(['Mapping', 'p', 'X', 'Y']) + }) + + it('reads square brackets around one item', () => { + expect(lower('[M]_x')).toEqual(['Subscript', ['Bracket', 'M'], 'x']) + expect(lower('p([M])')).toEqual(['Apply', 'p', ['Bracket', 'M']]) + }) + + it('lowers expectations, norms, and distributions', () => { + // The brackets after the expectation hold its argument, and its subscript holds the distributions. + expect(lower(String.raw`\mathbb{E}_{x \sim p}[f(x)]`)).toEqual(['Expectation', ['Apply', 'f', 'x'], ['Distributed', 'x', 'p']]) + // Without brackets, the expectation takes the rest of the operand, as a sum does. + expect(lower(String.raw`\mathbb{E}_{t} \|v\|^2`)).toEqual(['Expectation', ['Power', ['Norm', 'v'], { num: '2' }], 't']) + expect(lower(String.raw`\operatorname{Var}(X)`)).toEqual(['Variance', 'X']) + expect(lower(String.raw`\lVert x \rVert_2`)).toEqual(['Norm', 'x', { num: '2' }]) + // A calligraphic name before brackets is a distribution on an interval. + expect(lower(String.raw`t \sim \mathcal{U}[0,1]`)).toEqual(['Distributed', 't', ['Apply', ['Accent', '\'mathcal\'', 'U'], ['Interval', { num: '0' }, { num: '1' }]]]) + // Between two figures, `\sim` keeps its meaning of similarity. + expect(headOf(lower(String.raw`\triangle ABC \sim \triangle DEF`))).toBe('Similar') + }) + + it('lowers a comma list as a sequence', () => { + expect(lower(String.raw`r, s \in \Sigma`)).toEqual(['Element', ['Sequence', 'r', 's'], '\\Sigma']) + // The split comes first, so the minus is the sign of the second item. + expect(lower('(0, -1)')).toEqual(['Tuple', { num: '0' }, ['Negate', { num: '1' }]]) + expect(lower(String.raw`\{a, b\}`)).toEqual(['Set', ['Sequence', 'a', 'b']]) + }) + + it('lowers a sign or a label in a superscript as a mark, not a power', () => { + // A sign on a number set names a set of its own, as the positive integers. + expect(lower(String.raw`\mathbb{N}^{+}`)).toEqual(['NumberSet', 'N+']) + expect(lower('x^*')).toEqual(['Superscript', 'x', '\\ast']) + expect(lower('f^{(n)}')).toEqual(['Superscript', 'f', ['Delimiter', 'n']]) + }) + + it('lowers a function signature and comma-separated statements', () => { + expect(lower(String.raw`f: X \to Y, x \mapsto y`)).toEqual(['Clauses', ['Mapping', 'f', 'X', 'Y'], ['MapsTo', 'x', 'y']]) + }) + + it('lowers units and temperatures', () => { + expect(lower(String.raw`9.8\,\mathrm{m/s^2}`)).toEqual(['InvisibleOperator', { num: '9.8' }, ['Unit', '\'m\'', '\'s\'', { num: '2' }]]) + expect(lower(String.raw`25^\circ\mathrm{C}`)).toEqual(['Celsius', { num: '25' }]) + }) +}) diff --git a/packages/speech-transformer-math/src/lower.ts b/packages/speech-transformer-math/src/lower.ts new file mode 100644 index 0000000000..4851263bab --- /dev/null +++ b/packages/speech-transformer-math/src/lower.ts @@ -0,0 +1,1104 @@ +import type { MathJson } from '@proj-airi/mathjson' + +import type { TexNode } from './tex' + +import { argsOf, headOf, isFunction, isNumber, nothing, text } from '@proj-airi/mathjson' + +import { operatorHeads, rawHead, relationHeads } from './heads' +import { canonicalSymbol, greekNames } from './locale' + +const multiplicative = new Set(['Multiply', 'InlineDivide']) +/** Signs that can start an operand, as in `-1` or `\pm b`. */ +const unaryHeads: Readonly> = { '+': 'Positive', '-': 'Negate', '\\pm': 'PlusMinus', '\\mp': 'MinusPlus' } + +const smallSetOperators: Readonly> = { '\\cap': 'BigIntersection', '∩': 'BigIntersection', '\\cup': 'BigUnion', '∪': 'BigUnion' } + +const bigOperators: Readonly> = { + '\\lim': 'Limit', + '\\sum': 'Sum', + '∑': 'Sum', + '\\prod': 'Product', + '∏': 'Product', + '\\int': 'Integrate', + '∫': 'Integrate', + '\\iint': 'Integrate', + '\\oint': 'Integrate', + '\\bigcup': 'BigUnion', + '⋃': 'BigUnion', + '\\bigcap': 'BigIntersection', + '⋂': 'BigIntersection', +} + +/** The heads of operators that take the rest of the operand, with bounds below and above. */ +type BigHead = 'Limit' | 'Sum' | 'Product' | 'Integrate' | 'BigUnion' | 'BigIntersection' + +/** Symbols that only group or separate. Unpaired delimiters land here too. */ +const silent = new Set(['(', ')', '[', ']', '|', '\\{', '\\}', '\\lbrace', '\\rbrace', '\\langle', '\\rangle', '\\vert', '\\lvert', '\\rvert', '&', ',', ';', ':', '.', '\\colon']) +/** A separator starts a new operand, so a sign after it is unary, for example `-1` in `(0, -1)`. */ +const separators = new Set([',', ';', '&']) +/** + * Separators that split operands into a list, as in `r, s \in \Sigma` or `f(x; \theta)`. The items of a list are read + * with a pause between them. Without the split, `r, s` would join as the product `r s`. + * `&` is not here: it aligns columns, and an aligned equation is not a list. + */ +const listSeparators = new Set([',', ';']) +/** Between two operands, a triangle is the symmetric difference of two sets. Before an operand, it is the shape. */ +const triangles = new Set(['\\triangle', '△', '\\bigtriangleup']) +const infinities = new Set(['\\infty', '∞']) +const setRelations = new Set(['\\in', '\\notin', '\\subseteq', '\\subset', '\\supseteq', '\\supset']) +const setOperators = new Set(['\\cup', '\\cap', '\\setminus']) +const complements = new Set(['\\complement', '∁']) +const degreeMarks = new Set(['\\circ', '°']) +const derivativeMarks = new Set(['d', '\\partial', '∂']) +const fractionCommands = new Set(['frac', 'dfrac', 'tfrac', 'cfrac']) +const binomialCommands = new Set(['binom', 'dbinom', 'tbinom']) +const arrows = new Set(['\\to', '\\rightarrow', '→', '\\longrightarrow']) +/** `A^c` is a complement only on a set name. On other bases, for example `x^c`, it is a power. */ +const number = /^\d+(?:\.\d+)?$/ +const upperLetter = /^[A-Z]$/ +const latinLetter = /^[a-z]$/i +const digits = /^\d+$/ +/** Greek letter commands in both cases, for example `\phi` and `\Phi`. */ +const greekCommands = new Set(greekNames.flatMap(name => [`\\${name}`, `\\${name[0]!.toUpperCase()}${name.slice(1)}`])) +/** + * Letters that name a function by convention, so `f(x+1)` is a call although its parentheses hold a sum. + * Other letters before a sum in parentheses multiply it, as `n` in `n(n+1)`. + */ +const functionLetters = new Set(['f', 'g', 'h', 'p', 'q', 'u', 'v', 'w', 'F', 'G', 'H', 'L', 'P', 'Q', '\\phi', '\\varphi', '\\psi', '\\Phi', '\\Psi', '\\Gamma']) +/** + * Function commands of TeX, as `\ln` or `\gcd`. Parentheses after them hold arguments, also with a sum inside: + * `\ln(x+1)` is the log of x + 1, never `\ln` times x + 1. + */ +const functionCommands = new Set(['\\sin', '\\cos', '\\tan', '\\cot', '\\sec', '\\csc', '\\arcsin', '\\arccos', '\\arctan', '\\sinh', '\\cosh', '\\tanh', '\\coth', '\\ln', '\\log', '\\lg', '\\exp', '\\det', '\\gcd', '\\max', '\\min', '\\sup', '\\inf', '\\deg', '\\dim', '\\ker', '\\arg', '\\hom']) +/** Letters that upright text writes for a constant or a differential: `\mathrm{d}x`, `\mathrm{e}^x`, `\mathrm{i}`. */ +const uprightLetters = new Set(['d', 'e', 'i']) +/** Superscripts that restrict a number set by sign, keyed to the suffix of the set name in `numberSets`. */ +const signedSets: Readonly> = { '+': '+', '-': '-', '*': '*', '\\ast': '*' } +/** Constants that are never called: `e^{i\theta}(\cos\theta + i\sin\theta)` multiplies. */ +const constants = new Set(['e', 'i', '\\pi']) +/** Accents that keep a letter a name, for example `\bar{x}(t)` or `\hat{f}(\xi)`. */ +const accentCommands = new Set(['bar', 'hat', 'tilde', 'vec', 'dot', 'ddot', 'check', 'breve', 'overline', 'widehat', 'widetilde']) +/** A centered dot alone in an argument marks an empty slot, as in `p(\cdot)` or `\langle \cdot, \cdot \rangle`. */ +const placeholderMarks = new Set(['\\cdot', '\\bullet', '·', '∙']) +/** Delimiters of a norm, as in `\|x\|`. */ +const normDelimiters = new Set(['\\|', '\\Vert', '\\lVert', '‖']) +/** Font commands that keep a letter a name, so `\mathcal{L}(\theta)` is a call. `\mathbb` is not here: it names sets. */ +const nameFonts = new Set(['mathcal', 'mathscr', 'mathfrak', 'mathbf', 'boldsymbol', 'bm']) +/** Calligraphic fonts. A distribution is often named with one, for example `\mathcal{N}` or `\mathcal{U}`. */ +const calligraphicFonts = new Set(['mathcal', 'mathscr']) +/** Fonts that make a letter bold. A bold letter is a variable. */ +const boldFonts = new Set(['bf', 'mathbf', 'textbf', 'boldsymbol', 'bm']) +const solarBases = new Set(['M', 'L', 'R']) +const solarMarks = new Set(['\\odot', '⊙']) +/** + * Letters that name a probability density or a distribution, so `x \sim p_t` means x is drawn from it. + * Without one of these on the right, `\sim` keeps its meaning of similarity, as in `\triangle ABC \sim \triangle DEF`. + */ +const densityLetters = new Set(['p', 'q', 'P', 'Q', '\\pi']) + +/** The operators of probability, keyed by the letter or the name that writes them. */ +const statistics: Readonly> = { + E: 'Expectation', + P: 'Probability', + Pr: 'Probability', + Var: 'Variance', + Cov: 'Covariance', +} + +/** Superscripts that mark a symbol instead of raising it to a power: `\mathbb{R}^{+}`, `x^{*}`. */ +const superscriptMarks: Readonly> = { '+': '+', '-': '-', '*': '\\ast', '\\ast': '\\ast', '\\dagger': '\\dagger', '\\dag': '\\dagger', '†': '\\dagger', '\\top': 'T', '\\intercal': 'T', '\\vee': '\\vee', '\\wedge': '\\wedge' } + +function relationOf(node: TexNode): string | undefined { + if (node.type !== 'symbol') + return undefined + const key = canonicalSymbol(node.value) + return relationHeads[key] === undefined ? undefined : key +} + +function operatorOf(value: string): string | undefined { + // `/` and `*` have their own spellings, but they read as `\div` and `\times`. + const key = canonicalSymbol(value) + return operatorHeads[key] === undefined ? undefined : key +} + +function containsSymbol(nodes: readonly TexNode[] | undefined, values: ReadonlySet): boolean { + return nodes?.some((node) => { + switch (node.type) { + case 'symbol': + return values.has(node.value) + case 'group': + case 'fence': + return containsSymbol(node.children, values) + case 'scripts': + return containsSymbol([node.base], values) || containsSymbol(node.sub, values) || containsSymbol(node.sup, values) + case 'command': + return node.args.some(argument => containsSymbol(argument, values)) + default: + return false + } + }) ?? false +} + +function containsOperator(nodes: readonly TexNode[]): boolean { + return nodes.some(node => (node.type === 'symbol' && operatorOf(node.value) !== undefined) || ((node.type === 'group' || node.type === 'fence') && containsOperator(node.children))) +} + +function isOnlySymbol(nodes: readonly TexNode[] | undefined, value: string): boolean { + return nodes?.length === 1 && nodes[0]!.type === 'symbol' && nodes[0]!.value === value +} + +/** Splits `a, b` at its only comma. A list with more or fewer items is not a pair. */ +function splitPair(nodes: readonly TexNode[]): [TexNode[], TexNode[]] | undefined { + const commas = nodes.flatMap((node, index) => node.type === 'symbol' && node.value === ',' ? [index] : []) + if (commas.length !== 1) + return undefined + const left = nodes.slice(0, commas[0]) + const right = nodes.slice(commas[0]! + 1) + return left.length > 0 && right.length > 0 ? [left, right] : undefined +} + +/** Splits a list at its top-level commas. Commas inside groups and delimiters stay. */ +function splitClauses(nodes: readonly TexNode[]): TexNode[][] { + const clauses: TexNode[][] = [[]] + for (const node of nodes) { + if (node.type === 'symbol' && node.value === ',') + clauses.push([]) + else + clauses.at(-1)!.push(node) + } + return clauses +} + +/** A number keeps its digits. Other symbols keep their TeX token. */ +function atom(value: string): MathJson { + // `.5` omits the zero before the point. It reads as 0.5. + if (/^\.\d+$/.test(value)) + return { num: `0${value}` } + return number.test(value) ? { num: value } : value +} + +/** + * One operand or operator of a term list, before the operators are nested. + * `prefix` marks a unary sign that applies to the next operand. + */ +type Item + = | { kind: 'operand', value: MathJson } + | { kind: 'operator', key: string } + | { kind: 'prefix', key: string } + +/** An item of {@link lowerTerms}: a term item, or a separator between the items of a list. */ +type ListItem = Item | { kind: 'separator' } + +/** + * Lowers the syntax tree of a formula into MathJSON. + * + * This is the only place that decides what notation means, for example that `\frac{d}{dx}` is a derivative or that a + * pair after `\in` is an interval. It keeps what the writer wrote and never simplifies. + */ +export function lowerMath(nodes: readonly TexNode[]): MathJson { + return lowerList(nodes) +} + +/** Lowers a list that may hold relations and comma-separated statements. */ +function lowerList(nodes: readonly TexNode[]): MathJson { + const clauses = splitClauses(nodes) + if (clauses.filter(clause => clause.some(node => relationOf(node) !== undefined)).length >= 2) + return ['Clauses', ...clauses.map(lowerClause)] + return lowerClause(nodes) +} + +/** Lowers one statement. Each relation joins the expression on its left with the operand on its right. */ +function lowerClause(nodes: readonly TexNode[]): MathJson { + const operands: TexNode[][] = [[]] + const relations: string[] = [] + for (const node of nodes) { + let relation = relationOf(node) + // `:=` is two tokens in TeX. A colon right before `=` makes the definition, as `\coloneqq` does. + const last = operands.at(-1)!.at(-1) + if (relation === '=' && last?.type === 'symbol' && last.value === ':') { + operands.at(-1)!.pop() + relation = '\\coloneqq' + } + if (relation !== undefined) { + relations.push(relation) + operands.push([]) + } + else { + operands.at(-1)!.push(node) + } + } + + // `f: X \to Y` is a function signature: the arrow maps the set before it, not the name. + const first = operands[0]! + const colon = first.findIndex(node => node.type === 'symbol' && (node.value === ':' || node.value === '\\colon')) + let start = 0 + let left: MathJson + if (colon > 0 && relations[0] === '\\to') { + left = ['Mapping', lowerTerms(first.slice(0, colon)), lowerTerms(first.slice(colon + 1)), lowerTerms(operands[1]!)] + start = 1 + } + else { + left = lowerTerms(first) + } + + for (let index = start; index < relations.length; index++) { + const right = lowerTerms(operands[index + 1]!, relations[index]) + let head = relationHeads[relations[index]!]! + if (relations[index] === '\\to' && isMapArrow(left, right)) + head = 'Arrow' + // `x \sim p_t` draws x from a distribution. Between two figures, `\sim` keeps its meaning of similarity. + else if (relations[index] === '\\sim' && isDistribution(right)) + head = 'Distributed' + // In measurements `\sim` means about: `\sim 50` alone, or `z \sim 1` before a quantity. + else if (relations[index] === '\\sim' && isQuantity(right)) + head = left === nothing ? 'About' : 'Approx' + // `\approx 7` with nothing on its left reads "about seven", as a leading `\sim` does. + else if (relations[index] === '\\approx' && left === nothing && isQuantity(right)) + head = 'About' + left = head === 'About' ? ['About', right] : [head, left, right] + } + return left +} + +/** Heads of objects that are sets, spaces, or other structures, not numbers, so nothing approaches them. */ +const structureHeads = new Set(['Apply', 'NumberSet', 'Name', 'Set', 'Sequence', 'Bracket']) + +/** + * True when an expression names a structure: a capital letter, a number set, or one of these with scripts. + * A call is a structure when its name is one, so `H_k(M)` is a group of homology but `f(x)` is a value. + */ +function isStructure(value: MathJson): boolean { + if (typeof value === 'string') + return upperLetter.test(value) || (greekCommands.has(value) && value[1] !== value[1]!.toLowerCase()) + const head = headOf(value) + if (head === 'Subscript' || head === 'Superscript' || head === 'Apply') + return isStructure(argsOf(value)[0] ?? nothing) + if (head === 'Accent') + return isStructure(argsOf(value)[1] ?? nothing) + return head !== undefined && structureHeads.has(head) +} + +/** + * True when `left \to right` maps one structure to another, as in `H_k(M) \to H_k(M, N)` or `X \to Y`, so the arrow + * reads "to". Otherwise it is a limit, as in `n \to \infty` or `f(x) \to L`, and reads "approaches". + * + * A call on the right is a structure: nothing approaches `H_k(M, N)`. A capital letter on the right is a structure + * only when the left is one too, because `f(x) \to L` approaches the limit L. + */ +function isMapArrow(left: MathJson, right: MathJson): boolean { + const head = headOf(right) + if (head !== undefined && structureHeads.has(head)) + return true + return isStructure(right) && isStructure(left) +} + +/** Nests a flat list of operands and operators, with multiplicative operators first. Each level is left-associative. */ +function nest(items: readonly Item[]): MathJson { + // Apply unary signs to the operand after them. + const resolved: Item[] = [] + for (let index = 0; index < items.length; index++) { + const item = items[index]! + if (item.kind !== 'prefix') { + resolved.push(item) + continue + } + const next = items[index + 1] + if (next?.kind === 'operand') { + resolved.push({ kind: 'operand', value: [unaryHeads[item.key]!, next.value] }) + index++ + } + else { + resolved.push({ kind: 'operand', value: [unaryHeads[item.key]!] }) + } + } + + // Adjacent operands multiply without a sign, as in `2ab`. + const joined: Item[] = [] + for (const item of resolved) { + const previous = joined.at(-1) + if (item.kind === 'operand' && previous?.kind === 'operand') { + const value = previous.value + previous.value = isFunction(value, 'InvisibleOperator') ? ['InvisibleOperator', ...argsOf(value), item.value] : ['InvisibleOperator', value, item.value] + continue + } + joined.push({ ...item }) + } + + function level(list: readonly Item[], heads: (head: string) => boolean): Item[] { + const result: Item[] = [] + for (let index = 0; index < list.length; index++) { + const item = list[index]! + const left = result.at(-1) + const right = list[index + 1] + if (item.kind === 'operator' && heads(operatorHeads[item.key]!) && left?.kind === 'operand' && right?.kind === 'operand') { + left.value = [operatorHeads[item.key]!, left.value, right.value] + index++ + continue + } + result.push({ ...item }) + } + return result + } + + const result = level(level(joined, head => multiplicative.has(head)), () => true) + // A dangling operator, as in `1 +`, keeps its place as a symbol. + const values = result.map(item => item.kind === 'operand' ? item.value : item.key) + return values.length === 1 ? values[0]! : values.length === 0 ? nothing : ['InvisibleOperator', ...values] +} + +/** + * Nests each part of a list of terms. One part is an ordinary expression. More parts make `["Sequence", ...]`. + * + * The split comes before the nesting, so an operator never joins two items of a list: in `a, -b` the minus is the + * sign of `b`, not a subtraction from `a`. Empty parts, as in a trailing comma, are dropped. + */ +function nestList(items: readonly ListItem[]): MathJson { + const parts: Item[][] = [[]] + for (const item of items) { + if (item.kind === 'separator') + parts.push([]) + else + parts.at(-1)!.push(item) + } + const filled = parts.filter(part => part.length > 0) + return filled.length > 1 ? ['Sequence', ...filled.map(nest)] : nest(filled[0] ?? []) +} + +/** True when a node is a calligraphic letter, with or without scripts, for example `\mathcal{N}` or `\mathcal{L}_{FM}`. */ +function isCalligraphic(node: TexNode): boolean { + if (node.type === 'scripts') + return isCalligraphic(node.base) + return node.type === 'command' && calligraphicFonts.has(node.name) +} + +/** + * Returns the operator of probability that a node writes, with its subscript, for example `\mathbb{E}_{x \sim p}`, + * `\operatorname{Var}`, or `\Pr`. The subscript holds the variables or the distributions that the operator averages + * over. + */ +function statisticOf(node: TexNode): { head: string, sub?: TexNode[] } | undefined { + if (node.type === 'scripts' && !node.sup?.length) { + const inner = statisticOf(node.base) + return inner && { head: inner.head, sub: node.sub } + } + let name: string | undefined + if (node.type === 'command' && (node.name === 'mathbb' || node.name === 'mathbf') && node.args[0]?.length === 1 && node.args[0][0]!.type === 'symbol') + name = node.args[0][0]!.value + else if (node.type === 'text' && (node.name === 'operatorname' || node.name === 'mathrm')) + name = node.value.trim() + else if (node.type === 'symbol' && node.value === '\\Pr') + name = 'Pr' + const head = name === undefined ? undefined : statistics[name] + return head && { head } +} + +/** Number sets of school that read as sets with powers, as `\mathbb{R}^n`. Other blackboard letters name spaces. */ +const schoolNumberSets = new Set(['R', 'Z', 'Q', 'C', 'N']) + +/** True when an expression is a blackboard letter that names a space, as `\mathbb{P}`, not a number set of school. */ +function isSpaceName(value: MathJson): boolean { + if (headOf(value) !== 'NumberSet') + return false + const letter = argsOf(value)[0] + return typeof letter === 'string' && !schoolNumberSets.has(letter) +} + +/** Letters that name function spaces, as `L^p(\Omega)` or `H^1(\Omega)`. */ +const spaceLetters = new Set(['L', 'H', 'W', 'C', '\\ell']) + +/** + * The name of a called function space keeps its exponent as a label: `L^p(\Omega)` is L p of omega. A power of a + * letter is not called, so `L^p` before parentheses is a space. + */ +function spaceCallee(callee: MathJson): MathJson { + const [base = nothing, exponent = nothing] = argsOf(callee) + // Upright text before a call names the function, as `\mathrm{GL}(n)`. It is not the unit gigaliter. + if (headOf(callee) === 'Unit' && exponent === nothing) + return ['Text', base] + return headOf(callee) === 'Power' && typeof base === 'string' && spaceLetters.has(base) ? ['Superscript', base, exponent] : callee +} + +/** True when an expression is a measured quantity: a number, possibly with a power, a percent, or a unit. */ +function isQuantity(value: MathJson): boolean { + if (isNumber(value)) + return true + const head = headOf(value) + if (head === 'Power' || head === 'Percent' || head === 'Negate') + return isQuantity(argsOf(value)[0] ?? nothing) + if (head === 'InvisibleOperator' || head === 'Multiply') + return isQuantity(argsOf(value)[0] ?? nothing) && argsOf(value).slice(1).every(item => headOf(item) === 'Unit' || isQuantity(item)) + return false +} + +/** True when the right side of `\sim` names a distribution or a density, see {@link densityLetters}. */ +function isDistribution(value: MathJson): boolean { + if (typeof value === 'string') + return densityLetters.has(value) + const head = headOf(value) + const [first = nothing, second = nothing] = argsOf(value) + if (head === 'Accent') + return first === '\'mathcal\'' || first === '\'mathscr\'' + if (head === 'Subscript' || head === 'Superscript' || head === 'Apply') + return isDistribution(first) + // `\text{Bernoulli}(p)` and `\operatorname{Unif}`: a named function on the right of `\sim` is a distribution. + return head === 'Text' && second === nothing +} + +/** Operators that bind tighter than a sum, so a sum in parentheses after them keeps its brackets: `a - (b - c)`. */ +const tightOperators = new Set(['-', '\\times', '\\cdot', '\\div']) + +/** + * True when parentheses with an operator inside must be read, because a neighbor binds to them. + * + * Silent brackets lose the grouping that they write. `v - (x - \epsilon)` read as "v minus x minus epsilon" means + * `v - x - \epsilon`, and `(1-t)\epsilon` read as "one minus t epsilon" means `1 - t\epsilon`. So the brackets are read + * when an operand comes right before them, as in `n(n+1)`, when an operand comes right after them, as in `(1-t)\epsilon`, + * when a minus, a times, or a division sign comes before them, or when a times or a division sign comes after them, as + * in `(x+1)/2`. After a plus or a relation, and before a plus, a minus, or a relation, they can stay silent. + */ +function isBound(afterOperand: boolean, previous: TexNode | undefined, next: TexNode | undefined): boolean { + if (afterOperand) + return true + if (previous?.type === 'symbol' && tightOperators.has(operatorOf(previous.value) ?? previous.value)) + return true + if (next === undefined) + return false + if (next.type !== 'symbol') + return true + const nextOperator = operatorOf(next.value) + if (nextOperator !== undefined && nextOperator !== '-' && tightOperators.has(nextOperator)) + return true + // A symbol after the brackets starts an operand unless it is an operator, a relation, a separator, or a postfix mark. + return operatorOf(next.value) === undefined && relationOf(next) === undefined && !silent.has(next.value) && !['!', '\'', '\\prime', '%', '\\%'].includes(next.value) +} + +/** True when the nodes are one centered dot, the mark of an empty argument slot. */ +function isPlaceholder(nodes: readonly TexNode[]): boolean { + return nodes.length === 1 && nodes[0]!.type === 'symbol' && placeholderMarks.has(nodes[0]!.value) +} + +/** + * True when a node can name a function: a letter, a Greek letter, or one of them with scripts or an accent, for + * example `f`, `\phi`, `H_k`, `f^{-1}`, or `\bar{x}`. + * + * `before` is the node before `name`. A capital letter after another capital letter is the end of a name of geometry, + * as `B` in `AB`, so it does not name a function. + */ +function isCallee(name: TexNode, before: TexNode | undefined): boolean { + switch (name.type) { + case 'symbol': + if (functionCommands.has(name.value)) + return true + if (constants.has(name.value) || (!latinLetter.test(name.value) && !greekCommands.has(name.value))) + return false + // False for the last capital of a name such as `AB`. + return !upperLetter.test(name.value) || before?.type !== 'symbol' || !upperLetter.test(before.value) + case 'scripts': + // A power of a letter before parentheses multiplies, as `x^2(t+1)`. A function command keeps its call, as + // `\sin^2(x)`. + if (name.base.type === 'symbol' && functionCommands.has(name.base.value)) + return true + if (!name.sub?.length && name.sup?.length && name.sup.every(item => item.type === 'symbol' && digits.test(item.value))) + return false + return isCallee(name.base, undefined) + // `\operatorname{tr}(A)` and `\mathrm{GL}(n)` name a function in upright text. + case 'text': + return (name.name === 'operatorname' || name.name === 'mathrm') && /^[a-z]+$/i.test(name.value.trim()) + case 'command': + // A blackboard letter that names a space can be called, as `\mathbb{L}^\infty(X)`. School number sets cannot. + if (name.name === 'mathbb' && name.args[0]?.length === 1 && name.args[0][0]!.type === 'symbol' && !schoolNumberSets.has(name.args[0][0]!.value)) + return true + // A font name can have a few letters, as `\mathcal{CN}` of the complex normal distribution. + if (nameFonts.has(name.name) && name.args[0] && name.args[0].length <= 3 && name.args[0].every(item => item.type === 'symbol' && latinLetter.test(item.value))) + return true + return (accentCommands.has(name.name) || nameFonts.has(name.name)) && name.args[0]?.length === 1 && isCallee(name.args[0][0]!, undefined) + default: + return false + } +} + +/** + * True when a name carries more than a letter: a subscript, a superscript that is not a number, or an accent. Such a + * name is usually a function or an operator of its field, for example `H_k` of homology or `f^{-1}`. A number + * superscript is a power, so `x^2(x+1)` stays a product. + */ +function isDecorated(name: TexNode): boolean { + if (name.type === 'command') + return true + if (name.type !== 'scripts') + return false + const numberPower = name.sup !== undefined && name.sup.every(node => node.type === 'symbol' && digits.test(node.value)) + return Boolean(name.sub?.length) || (Boolean(name.sup?.length) && !numberPower) +} + +/** + * Returns the arguments of `name(...)` when the parentheses are a function call, or undefined when they group a + * factor of a product. + * + * TeX writes `f(x)` and `n(n+1)` the same way: a letter, then parentheses. Speech must choose, because "f of x" and + * "n times the quantity n plus one" are different formulas for the listener. The rules, in order: + * + * 1. Only parentheses after a name can be a call, see {@link isCallee}. + * 2. A top-level comma means arguments, because a product has no comma: `H_k(M, M-N)`, `A_1(\Sigma, f)`. A pair with + * an infinite bound is an interval instead, as in `(-\infty, 0)`. + * 3. One argument without an operator is a call: `x(t)`, `A_1(f)`, `p(\cdot)`. A writer multiplies letters as `xt`. + * 4. An argument with an operator is a call only after a decorated name or a function letter: `H_1(S-P)` and + * `f(x+1)` are calls, but `n(n+1)` and `a(b+c)` are products. + * + * In Chinese, Japanese, and Korean a simple call reads as the product did, so these rules change only English words + * and the grouped readings, see `MathLocale.apply`. + */ +function callArguments(name: TexNode, before: TexNode | undefined, fence: Extract): MathJson[] | undefined { + // A calligraphic name before brackets is a distribution on an interval, as in `\mathcal{U}[0,1]`. The interval is + // its one argument. + if (fence.open === '[' && isCalligraphic(name)) + return [lowerFence({ ...fence, sub: undefined, sup: undefined }, true, false)] + // A name of capitals with an argument list is a function, as `vMF(\mu, \kappa)`. A name of geometry has no commas. + const namedList = name.type === 'symbol' && upperLetter.test(name.value) && splitClauses(fence.children).length > 1 + if (fence.open !== '(' || (!isCallee(name, before) && !namedList)) + return undefined + const parts = splitClauses(fence.children) + if (parts.length > 1 && containsSymbol(fence.children, infinities)) + return undefined + const call = parts.length > 1 + || isPlaceholder(fence.children) + || !containsOperator(fence.children) + || isDecorated(name) + || name.type === 'text' + || (name.type === 'symbol' && (functionLetters.has(name.value) || functionCommands.has(name.value))) + || (name.type === 'scripts' && name.base.type === 'symbol' && functionCommands.has(name.base.value)) + if (!call) + return undefined + if (fence.children.length === 0) + return [] + return parts.map(part => isPlaceholder(part) ? ['Placeholder'] : lowerArgument(part)) +} + +/** + * Lowers an argument of a call or of a statistic. A bar inside it is a condition, as `P(A \mid B)` or + * `p(x \mid \theta)`: the first part given the second. Outside arguments and sets, `a \mid b` is divides. + */ +function lowerArgument(nodes: readonly TexNode[]): MathJson { + const bar = nodes.findIndex(node => node.type === 'symbol' && (node.value === '\\mid' || node.value === '|')) + if (bar <= 0 || bar === nodes.length - 1) + return lowerList(nodes) + return ['Given', lowerList(nodes.slice(0, bar)), lowerList(nodes.slice(bar + 1))] +} + +/** + * Lowers `\pmod{n}`. KaTeX writes it as parentheses around an upright mod and the modulus, which would read as a call. + */ +function modulusOf(node: TexNode): MathJson | undefined { + if (node.type !== 'fence' || node.open !== '(') + return undefined + const [wrapped, ...rest] = node.children + // KaTeX wraps the upright mod in a group. + const first = wrapped?.type === 'group' && wrapped.children.length === 1 ? wrapped.children[0] : wrapped + return first?.type === 'text' && first.value.trim() === 'mod' && rest.length > 0 ? ['Modulo', lowerList(rest)] : undefined +} + +/** `relation` is the relation before the terms, so `\in` can mark the interval after it. */ +function lowerTerms(nodes: readonly TexNode[], relation?: string): MathJson { + const items: ListItem[] = [] + let afterOperand = false + + const push = (value: MathJson) => { + items.push({ kind: 'operand', value }) + afterOperand = true + } + const replaceLast = (wrap: (value: MathJson) => MathJson) => { + const last = items.at(-1) + if (last?.kind === 'operand') + last.value = wrap(last.value) + else + push(wrap(nothing)) + } + + for (let index = 0; index < nodes.length; index++) { + const node = nodes[index]! + const next = nodes[index + 1] + const previous = nodes[index - 1] + + // `\mathbb{E}` is an expectation only with an argument after it, or with a subscript and more terms after it. + // Alone, as in `B(\mathbb{E})` or `(\Omega, \mathcal F, \mathbb P)`, it names a space or a measure. + const statistic = statisticOf(node) + const hasArgument = next?.type === 'fence' || (statistic?.sub !== undefined && next !== undefined) + if (statistic && hasArgument) { + const condition = statistic.sub ? [lowerList(statistic.sub)] : [] + // `\mathbb{E}[X]` and `\Pr(A)`: the brackets after the operator hold its argument. They are not read. + if (next?.type === 'fence' && (next.open === '[' || next.open === '(' || next.open === '\\{')) { + push(lowerScripts([statistic.head, lowerArgument(next.children), ...condition], next.sub, next.sup)) + index++ + continue + } + // `\mathbb{E}_{x} \|f(x)\|^2` has no brackets, so the operator takes the rest of the operand, as a sum does. + push([statistic.head, lowerTerms(nodes.slice(index + 1)), ...condition]) + break + } + + const big = bigOperator(node) + if (big) { + // A big operator takes the rest of the operand, up to the next relation. + push(lowerBigOperator(big, lowerTerms(nodes.slice(index + 1)))) + break + } + + const derivative = derivativeOf(node) + if (derivative) { + // `\frac{d}{dx}` alone is an operator, so it takes the rest of the operand, as a big operator does. + if (derivative.body !== undefined) { + push(derivative.order ? ['D', derivative.body, derivative.variable, { num: derivative.order }] : ['D', derivative.body, derivative.variable]) + continue + } + const body = lowerTerms(nodes.slice(index + 1)) + push(derivative.order ? ['D', body, derivative.variable, { num: derivative.order }] : ['D', body, derivative.variable]) + break + } + + const prefixed = next === undefined ? undefined : lowerSetPrefix(node, next) + if (prefixed !== undefined) { + push(prefixed) + index++ + continue + } + + const modulus = modulusOf(node) + if (modulus !== undefined) { + push(modulus) + continue + } + + if (node.type === 'fence') { + // `f(x)` wraps the name that was pushed last into a call, with the scripts of the parentheses, as in `f(x)^2`. + const args = afterOperand && previous !== undefined ? callArguments(previous, nodes[index - 2], node) : undefined + if (args !== undefined) { + replaceLast(callee => lowerScripts(['Apply', spaceCallee(callee), ...args], node.sub, node.sup)) + continue + } + const afterSetSymbol = index === 0 + ? relation !== undefined && setRelations.has(relation) + : previous?.type === 'symbol' && setOperators.has(canonicalSymbol(previous.value)) + push(lowerFence(node, afterSetSymbol, isBound(afterOperand, previous, next))) + continue + } + + // `25^\circ\mathrm{C}` is a temperature, not 25 degrees followed by the letter C. + if (node.type === 'text' && node.value.trim() === 'C' && previous?.type === 'scripts' && previous.sup?.length === 1 && previous.sup[0]!.type === 'symbol' && degreeMarks.has(previous.sup[0]!.value)) { + replaceLast(() => ['Celsius', lowerNode(previous.base)]) + continue + } + + // `AB` of geometry is one name, not a product of points. + if (node.type === 'symbol' && upperLetter.test(node.value)) { + let end = index + while (nodes[end + 1]?.type === 'symbol' && upperLetter.test((nodes[end + 1] as { value: string }).value)) + end++ + if (end > index) { + push(['Name', ...nodes.slice(index, end + 1).map(item => (item as { value: string }).value)]) + index = end + continue + } + } + + if (node.type !== 'symbol') { + push(lowerNode(node)) + continue + } + + const value = node.value + if (silent.has(value)) { + if (separators.has(value)) + afterOperand = false + if (listSeparators.has(value)) + items.push({ kind: 'separator' }) + continue + } + + // `\mathrm{Rt}\triangle ABC` is a right triangle. The triangle after Rt is the shape, not symmetric difference. + const rightTriangle = previous?.type === 'text' && previous.value.trim() === 'Rt' + if (afterOperand && triangles.has(value) && !rightTriangle) { + items.push({ kind: 'operator', key: '\\triangle' }) + afterOperand = false + continue + } + + const operator = operatorOf(value) + if (operator !== undefined && !triangles.has(value)) { + items.push(afterOperand || unaryHeads[operator] === undefined ? { kind: 'operator', key: operator } : { kind: 'prefix', key: operator }) + afterOperand = false + continue + } + + if (value === '!') { + replaceLast(operand => ['Factorial', operand]) + continue + } + + if ((value === '%' || value === '\\%') && previous?.type === 'symbol' && number.test(previous.value)) { + replaceLast(operand => ['Percent', operand]) + continue + } + + if (value === '\'' || value === '\\prime') { + replaceLast(operand => ['Prime', operand]) + continue + } + + // `10\,\Omega` is a resistance: the capital omega after a number is the ohm. + if ((value === '\\Omega' || value === 'Ω') && afterOperand && isNumber(items.at(-1)?.kind === 'operand' ? (items.at(-1) as { value: MathJson }).value : undefined)) { + push(['Unit', text('Ω')]) + continue + } + + push(atom(value)) + } + + return nestList(items) +} + +/** Lowers a prefix that takes the operand after it, for example `\complement_U A`, `\mathcal{P}(A)`, or `\#A`. */ +function lowerSetPrefix(node: TexNode, next: TexNode): MathJson | undefined { + if (node.type === 'symbol' && complements.has(node.value)) + return ['Complement', lowerNode(next)] + if (node.type === 'scripts' && node.base.type === 'symbol' && complements.has(node.base.value) && !node.sup?.length) + return node.sub ? ['Complement', lowerNode(next), lowerList(node.sub)] : ['Complement', lowerNode(next)] + if (node.type === 'symbol' && node.value === '#') + return ['Cardinality', lowerNode(next)] + + if (next.type !== 'fence' || next.open !== '(') + return undefined + if (node.type === 'command' && node.name === 'mathcal' && isOnlySymbol(node.args[0], 'P')) + return ['PowerSet', lowerList(next.children)] + if (node.type === 'text' && node.value.trim() === 'card') + return ['Cardinality', lowerList(next.children)] + return undefined +} + +interface BigOperator { head: BigHead, sub?: TexNode[], sup?: TexNode[] } + +function bigOperator(node: TexNode): BigOperator | undefined { + if (node.type === 'symbol' && bigOperators[node.value]) + return { head: bigOperators[node.value]! } + if (node.type === 'scripts' && node.base.type === 'symbol' && bigOperators[node.base.value]) + return { head: bigOperators[node.base.value]!, sub: node.sub, sup: node.sup } + // A small `\cap` or `\cup` with bounds is written for the big one, as in `\cap_{i=1}^r E_i`. + if (node.type === 'scripts' && node.base.type === 'symbol' && (node.sub?.length || node.sup?.length) && smallSetOperators[node.base.value]) + return { head: smallSetOperators[node.base.value]!, sub: node.sub, sup: node.sup } + return undefined +} + +/** + * Lowers a big operator with the body that it takes. + * + * - `["Limit", body, ["Bounds", variable, target, side]]`, where side is `1` from the right and `-1` from the left. + * - `["Sum" | "Product" | "Integrate", body, ["Bounds", variable, from, to]]`. + * - Any of them with `["Condition", condition, to]` when the subscript is not `variable = from`. + * + * `Nothing` fills a bound that the writer left out. + */ +function lowerBigOperator({ head, sub, sup }: BigOperator, body: MathJson): MathJson { + if (head === 'Limit') { + const arrow = sub?.findIndex(node => node.type === 'symbol' && arrows.has(node.value)) ?? -1 + if (sub && arrow > 0) { + // `x \to 0^{+}` approaches from the right. The sign is a side, not a power. + const target = sub.slice(arrow + 1) + const only = target.length === 1 ? target[0]! : undefined + const sign = only?.type === 'scripts' && !only.sub?.length && only.sup?.length === 1 && only.sup[0]!.type === 'symbol' ? only.sup[0]!.value : undefined + if (only?.type === 'scripts' && (sign === '+' || sign === '-')) + return ['Limit', body, ['Bounds', lowerList(sub.slice(0, arrow)), lowerNode(only.base), { num: sign === '+' ? '1' : '-1' }]] + return ['Limit', body, ['Bounds', lowerList(sub.slice(0, arrow)), lowerList(target)]] + } + return sub ? ['Limit', body, ['Condition', lowerList(sub)]] : ['Limit', body] + } + + const to = sup ? lowerList(sup) : nothing + if (head === 'Integrate') + return sub || sup ? ['Integrate', body, ['Bounds', nothing, sub ? lowerList(sub) : nothing, to]] : ['Integrate', body] + + const equals = sub?.findIndex(node => node.type === 'symbol' && node.value === '=') ?? -1 + if (sub && equals > 0) + return [head, body, ['Bounds', lowerList(sub.slice(0, equals)), lowerList(sub.slice(equals + 1)), to]] + if (sub) + return [head, body, ['Condition', lowerList(sub), to]] + return sup ? [head, body, ['Condition', nothing, to]] : [head, body] +} + +/** + * Returns the variable and the body of `\frac{d}{dx}`, `\frac{dy}{dx}`, or `\frac{\partial f}{\partial x}`. + * `body` is undefined when the numerator holds only the mark, so the derivative applies to what follows. + */ +function derivativeOf(node: TexNode): { variable: MathJson, body?: MathJson, order?: string } | undefined { + if (node.type !== 'command' || !fractionCommands.has(node.name)) + return undefined + const [numerator = [], denominator = []] = node.args + const isMark = (item: TexNode | undefined) => (item?.type === 'symbol' && derivativeMarks.has(item.value)) || (item?.type === 'text' && item.value.trim() === 'd') + // `d^2` in the numerator of `\frac{d^2 y}{dx^2}` gives the order. The power of the variable repeats it. + const first = numerator[0] + const order = first?.type === 'scripts' && isMark(first.base) && !first.sub?.length && first.sup?.length === 1 && first.sup[0]!.type === 'symbol' && digits.test(first.sup[0]!.value) ? first.sup[0]!.value : undefined + if ((!isMark(first) && order === undefined) || !isMark(denominator[0]) || denominator.length < 2) + return undefined + const last = denominator.at(-1)! + const variable = order !== undefined && last.type === 'scripts' && !last.sub?.length ? [...denominator.slice(1, -1), last.base] : denominator.slice(1) + return { variable: lowerList(variable), body: numerator.length > 1 ? lowerList(numerator.slice(1)) : undefined, order } +} + +/** + * Lowers a delimited group. + * + * - A bracket or an infinite bound makes an interval. `(a, b)` alone is also a point or the arguments of a function, + * so it needs a set symbol before it. + * - `|x|` is `Abs`, and `\{...\}` is a set, with a condition after `\mid`. + * - Other parentheses are `Delimiter`, which is silent, or `Group`, which is read as a quantity. A group is read when a + * power applies to it, or when it holds an operator and `bound` is true: an operand or a tight operator touches it, + * see {@link isBound}. Its brackets then decide the meaning, so they cannot go silent. + */ +function lowerFence(node: Extract, afterSetSymbol: boolean, bound: boolean): MathJson { + let base: MathJson + const pair = (node.open === '(' || node.open === '[') ? splitPair(node.children) : undefined + const bracketed = node.open === '[' || node.close === ']' + if (pair && (bracketed || afterSetSymbol || containsSymbol(node.children, infinities))) { + const from = lowerList(pair[0]) + const to = lowerList(pair[1]) + base = ['Interval', node.open === '[' ? from : ['Open', from], node.close === ']' ? to : ['Open', to]] + } + else if (node.open === '\\lceil' || node.open === '⌈' || node.open === '\\lfloor' || node.open === '⌊') { + base = [node.open === '\\lceil' || node.open === '⌈' ? 'Ceil' : 'Floor', lowerList(node.children)] + } + else if (normDelimiters.has(node.open)) { + // `\|x\|_2` is the 2-norm. The subscript names the norm, so it stays inside `Norm` and only the power applies outside. + // `\|\cdot\|` names the norm itself. Its dot is an empty slot, not a product. + const inner: MathJson = isPlaceholder(node.children) ? ['Placeholder'] : lowerList(node.children) + return lowerScripts(node.sub?.length ? ['Norm', inner, lowerList(node.sub)] : ['Norm', inner], undefined, node.sup) + } + else if (node.open === '|' || node.open === '\\vert' || node.open === '\\lvert') { + base = ['Abs', lowerList(node.children)] + } + else if (node.open === '\\{' || node.open === '\\lbrace') { + const condition = node.children.findIndex(child => child.type === 'symbol' && (child.value === '\\mid' || child.value === '|' || child.value === ':')) + base = condition > 0 + ? ['Set', lowerList(node.children.slice(0, condition)), ['Condition', lowerList(node.children.slice(condition + 1))]] + : ['Set', lowerList(node.children)] + } + else { + const inner = lowerList(node.children) + const product = bound && node.open === '(' && containsOperator(node.children) + // Square brackets around one item mean something, for example the class `[M]` or a concentration, so they are read. + if (node.open === '[') + base = ['Bracket', inner] + // `(V, E)` is a pair and `(a, b, c)` a tuple. Silent brackets would read them as a list of loose items. + else if (node.open === '(' && headOf(inner) === 'Sequence') + base = ['Tuple', ...argsOf(inner)] + else + base = node.sup?.length || product ? ['Group', inner] : ['Delimiter', inner] + } + return lowerScripts(base, node.sub, node.sup) +} + +/** Lowers the scripts of a base. `baseNode` is the TeX base, so `A^c` checks the letter, not a lowered expression. */ +function lowerScripts(base: MathJson, sub?: TexNode[], sup?: TexNode[], baseNode?: TexNode): MathJson { + let result = base + // A word in a subscript is a label, as `t` in `I_{\mathrm t}` or `max` in `x_{\text{max}}`, never a unit. + const label = sub?.length === 1 && sub[0]!.type === 'text' ? sub[0]!.value.replace(/[{}\\]/g, '').trim() : undefined + // `45.1_{\pm 0.5}` writes the uncertainty of a measurement: plus or minus 0.5. + const uncertainty = isNumber(result) && sub?.[0]?.type === 'symbol' && (sub[0].value === '\\pm' || sub[0].value === '±') && sub.length > 1 + if (uncertainty) + result = ['PlusMinus', result, lowerList(sub!.slice(1))] + else if (label !== undefined) + result = ['Subscript', result, ['Text', text(label)]] + else if (sub?.length) + result = ['Subscript', result, lowerList(sub)] + if (sup?.length) { + // `\mathrm{T}` and `\mathsf{T}` write the transpose mark in upright text. + const uprightMark = sup.length === 1 && sup[0]!.type === 'text' && sup[0]!.name !== 'text' ? sup[0]!.value.replace(/[{}\\]/g, '').trim() : undefined + const only = sup.length === 1 && sup[0]!.type === 'symbol' ? sup[0]!.value : uprightMark === 'T' ? 'T' : undefined + const primes = sup.every(node => node.type === 'symbol' && (node.value === '\\prime' || node.value === '\'')) + // `f^{-1}` of a function is its inverse, not 1/f. Other bases keep the power, as `x^{-1}`. + const inverse = sup.length === 2 && sup[0]!.type === 'symbol' && sup[0]!.value === '-' && sup[1]!.type === 'symbol' && sup[1]!.value === '1' + && baseNode?.type === 'symbol' && (functionLetters.has(baseNode.value) || functionCommands.has(baseNode.value)) + // `n^{\text{th}}` writes an ordinal, not a power of the word th. + const ordinal = sup.length === 1 && sup[0]!.type === 'text' && /^(?:th|st|nd|rd)$/.test(sup[0]!.value.trim()) + if (inverse) + result = ['Inverse', result] + else if (ordinal) + result = ['Ordinal', result] + // `f''` is the second derivative: a prime for each mark, never a power. + else if (primes && sup.length > 1) + result = sup.reduce(value => ['Prime', value], result) + else if (only !== undefined && degreeMarks.has(only)) + result = ['Degrees', result] + else if (only === '\\prime' || only === '\'') + result = ['Prime', result] + else if (only !== undefined && (complements.has(only) || (only === 'c' && baseNode?.type === 'symbol' && upperLetter.test(baseNode.value)))) + result = ['Complement', result] + // `\mathbb{N}^{+}` and `\mathbb{N}^*` name the positive integers. The packs name the whole set, not the mark. + else if (only !== undefined && headOf(result) === 'NumberSet' && signedSets[only] !== undefined && typeof argsOf(result)[0] === 'string') + result = ['NumberSet', `${argsOf(result)[0] as string}${signedSets[only]}`] + else if (only !== undefined && superscriptMarks[only] !== undefined) + result = ['Superscript', result, superscriptMarks[only]!] + // Some superscripts are labels, not powers: + // - `x^1` is never written as a power, so its 1 is an index, as in `D^1_{\mu}`. A number base keeps the power: TeX + // reads `2^10` as `2^1` and 0. + // - `A^T` is a transpose, and a capital T after a letter is a label more often than a power. + // - A space named by a blackboard letter, as `\mathbb{P}^n` or `\mathbb{L}^\infty`, takes its dimension or its + // exponent as a name. The number sets of school, such as `\mathbb{R}^n`, keep the power reading. + // - A superscript before its symbol, as `^2\Pi_{3/2}` of spectroscopy, has no base to raise. + else if ((only === '1' && baseNode?.type !== 'symbol') || (only === '1' && baseNode?.type === 'symbol' && !number.test(baseNode.value)) || (only === 'T' && baseNode?.type !== 'symbol') || (only === 'T' && baseNode?.type === 'symbol' && latinLetter.test(baseNode.value)) || isSpaceName(base) || (result === nothing && only !== undefined && number.test(only))) + result = ['Superscript', result, uprightMark === 'T' ? 'T' : lowerList(sup)] + // `f^{(n)}` names an order or a label in parentheses. It is not a power of n. + else if (sup.length === 1 && sup[0]!.type === 'fence' && sup[0]!.open === '(' && !sup[0]!.sub?.length && !sup[0]!.sup?.length) + result = ['Superscript', result, ['Delimiter', lowerList(sup[0]!.children)]] + else + result = ['Power', result, lowerList(sup)] + } + return result +} + +/** Splits a row of an array at its `&` cells. KaTeX wraps each cell in a group, which is unwrapped. */ +function cellsOf(row: readonly TexNode[]): TexNode[][] { + const cells: TexNode[][] = [[]] + for (const node of row) { + if (node.type === 'symbol' && node.value === '&') + cells.push([]) + // `aligned` puts an empty group before the relation of a cell, as in `&= b`. It has no sound. + else if (node.type === 'group') + cells.at(-1)!.push(...node.children.filter(child => child.type !== 'group' || child.children.length > 0)) + else + cells.at(-1)!.push(node) + } + return cells +} + +/** Words that start the condition of a case, as `\text{if } x > 0`. The packs say their own word. */ +const caseWords = /^\s*(?:if|for|when|,)\s*$/i +/** Words that write the last case, as `\text{otherwise}`. */ +const otherwiseWords = /^\s*(?:otherwise|else|其他|其它|それ以外|그 ?외)\s*$/i + +/** + * Lowers an array. + * + * - `cases` becomes `["Cases", ["Case", value, condition?], ...]`. A case without a condition, or with "otherwise", + * is the last case. + * - A matrix or a determinant becomes `["Matrix", ["Row", ...cells], ...]` or `Determinant`, so each cell is heard. + * - `aligned` and other arrays are `Rows` of statements. Their `&` only aligns the columns. + */ +function lowerRows(node: Extract): MathJson { + if (node.name === 'cases') { + return ['Cases', ...node.rows.filter(row => row.length > 0).map((row): MathJson => { + const [value = [], ...rest] = cellsOf(row) + const condition = rest.flat() + while (condition[0]?.type === 'text' && caseWords.test(condition[0].value)) + condition.shift() + const otherwise = condition.length === 0 || (condition.length === 1 && condition[0]!.type === 'text' && otherwiseWords.test(condition[0].value)) + return otherwise ? ['Case', lowerList(value)] : ['Case', lowerList(value), lowerList(condition)] + })] + } + if (node.name === 'matrix' || node.name === 'determinant') + return [node.name === 'matrix' ? 'Matrix' : 'Determinant', ...node.rows.filter(row => row.length > 0).map((row): MathJson => ['Row', ...cellsOf(row).map(lowerList)])] + // A row that starts with a relation continues the row before it, as `&= c` in a derivation, so they read as one chain. + const statements: TexNode[][] = [] + for (const row of node.rows) { + const tokens = cellsOf(row).flat() + if (tokens.length === 0) + continue + if (statements.length > 0 && relationOf(tokens[0]!) !== undefined) + statements.at(-1)!.push(...tokens) + else + statements.push(tokens) + } + return statements.length === 1 ? lowerList(statements[0]!) : ['Rows', ...statements.map(lowerList)] +} + +/** Lowers a unit in text, for example `m`, `km2`, `m/s`, or `m/s^2`. Other text stays text. */ +function lowerText(name: string, value: string): MathJson { + // Unit commands become their symbols before the braces and backslashes go, so `\mu m` stays micrometers. + const content = value.replace(/\\mu\s*/g, 'µ').replace(/\\Omega\s*/g, 'Ω').replace(/\\cdot|·/g, ' ').replace(/[{}\\]/g, '').replace(/\s+/g, ' ').trim() + if (name === 'text' || name === 'operatorname') + return ['Text', text(value.trim())] + if (uprightLetters.has(content)) + return content + // A bold letter is a vector or a matrix, as `{\bf B}`, never the unit byte. + if (boldFonts.has(name) && /^[a-z]$/i.test(content)) + return content + const ratio = /^([a-z]+\d?)\/([a-z]+)(?:\^(\d))?$/i.exec(content) + if (ratio) + return ratio[3] ? ['Unit', text(ratio[1]!), text(ratio[2]!), { num: ratio[3] }] : ['Unit', text(ratio[1]!), text(ratio[2]!)] + return lowerUnitFactors(content) ?? ['Unit', text(content.replace(/\^?(\d)$/, '$1'))] +} + +/** + * Lowers a unit written as factors with powers, as `kg m^2`, `mol L^-1`, or `J mol^-1 K^-1`. Factors with a negative + * power go after "per". Returns undefined for one factor with a positive power, which ./units reads alone. + */ +function lowerUnitFactors(content: string): MathJson | undefined { + const factors = content.split(' ').map(factor => /^([^\d^-]+)\^?(-?\d)?$/.exec(factor)) + if (factors.includes(null)) + return undefined + const parsed = factors as RegExpExecArray[] + const up = parsed.filter(factor => !factor[2]?.startsWith('-')) + const down = parsed.filter(factor => factor[2]?.startsWith('-')) + if (parsed.length === 1 && down.length === 0) + return undefined + const numerator = text(up.map(factor => factor[1]! + (factor[2] ?? '')).join(' ')) + if (down.length === 0) + return ['Unit', numerator] + // One power after "per" reads as squared or cubed. Several factors after "per" each have the power one. + if (down.length === 1 && down[0]![2] !== '-1') + return ['Unit', numerator, text(down[0]![1]!), { num: down[0]![2]!.slice(1) }] + if (down.some(factor => factor[2] !== '-1')) + return undefined + return ['Unit', numerator, text(down.map(factor => factor[1]!).join(' '))] +} + +function lowerNode(node: TexNode): MathJson { + switch (node.type) { + case 'symbol': + return silent.has(node.value) ? nothing : atom(node.value) + case 'group': + return lowerList(node.children) + case 'text': + return lowerText(node.name, node.value) + // An extension reads the source of its command and replaces this node, see ./speech. + case 'raw': + return [rawHead, text(node.command), text(node.value)] + case 'rows': + return lowerRows(node) + case 'scripts': { + // `M_\odot` is the solar mass of astronomy, also `L_\odot` and `R_\odot`. The packs name them as words. + if (node.base.type === 'symbol' && solarBases.has(node.base.value) && node.sub?.length === 1 && node.sub[0]!.type === 'symbol' && solarMarks.has(node.sub[0]!.value) && !node.sup?.length) + return `${node.base.value}_\\odot` + const base = node.base.type === 'symbol' && node.base.value === '' ? nothing : lowerNode(node.base) + return lowerScripts(base, node.sub, node.sup, node.base) + } + case 'fence': + return lowerFence(node, false, false) + case 'command': + return lowerCommand(node) + } +} + +function lowerCommand(node: Extract): MathJson { + const [first = [], second = []] = node.args + if (fractionCommands.has(node.name)) + return ['Divide', lowerList(first), lowerList(second)] + if (binomialCommands.has(node.name)) + return ['Binomial', lowerList(first), lowerList(second)] + switch (node.name) { + case 'sqrt': + return node.option?.length ? ['Root', lowerList(first), lowerList(node.option)] : ['Sqrt', lowerList(first)] + case 'overset': + case 'stackrel': + // `\overset{\frown}{AB}` is the arc AB. Other marks over a symbol are not read. + return isOnlySymbol(first, '\\frown') ? ['Arc', lowerList(second)] : lowerList(second) + case 'underset': + return lowerList(second) + case 'mathbb': + return ['NumberSet', lowerList(first)] + case 'overline': + // A line over capital letters is a segment of geometry. Over other symbols it is a bar, as in a mean. + return first.length >= 2 && first.every(item => item.type === 'symbol' && upperLetter.test(item.value)) + ? ['Segment', ...first.map(item => (item as { value: string }).value)] + : ['Accent', text('overline'), lowerList(first)] + default: + return ['Accent', text(node.name), lowerList(first)] + } +} diff --git a/packages/speech-transformer-math/src/numbers.test.ts b/packages/speech-transformer-math/src/numbers.test.ts new file mode 100644 index 0000000000..3df061fb78 --- /dev/null +++ b/packages/speech-transformer-math/src/numbers.test.ts @@ -0,0 +1,25 @@ +import { describe, expect, it } from 'vitest' + +import { spellNumber } from './numbers' + +describe('spellNumber', () => { + it.each([ + ['0', 'zero', 'ゼロ', '영'], + ['14', 'fourteen', 'じゅうよん', '십사'], + ['25', 'twenty-five', 'にじゅうご', '이십오'], + ['300', 'three hundred', 'さんびゃく', '삼백'], + ['800', 'eight hundred', 'はっぴゃく', '팔백'], + ['1,200', 'one thousand two hundred', 'せんにひゃく', '천이백'], + ['3000', 'three thousand', 'さんぜん', '삼천'], + ['10000', 'ten thousand', 'いちまん', '만'], + ['3.14', 'three point one four', 'さんてんいちよん', '삼 점 일사'], + ])('spells %s', (digits, en, ja, ko) => { + expect(spellNumber(digits, 'en')).toBe(en) + expect(spellNumber(digits, 'ja')).toBe(ja) + expect(spellNumber(digits, 'ko')).toBe(ko) + }) + + it('reads a very long integer digit by digit, as an identifier', () => { + expect(spellNumber('1234567890123', 'en')).toBe('one two three four five six seven eight nine zero one two three') + }) +}) diff --git a/packages/speech-transformer-math/src/numbers.ts b/packages/speech-transformer-math/src/numbers.ts new file mode 100644 index 0000000000..702910a370 --- /dev/null +++ b/packages/speech-transformer-math/src/numbers.ts @@ -0,0 +1,132 @@ +/** + * Spells numbers as words, so a TTS voice cannot read a digit in another language. + * + * A voice of one language can read a bare digit in its own language. For example, a Chinese voice read the `1` of + * `1 over 2` as `yī`, which an English listener hears as "E". Words of the target language leave no such choice. + * Japanese uses kana, because a kanji numeral such as 五十 can still be read in Chinese. + */ + +/** Integers with more digits than this are read digit by digit, as a code or an identifier is. */ +const maxGroupedDigits = 12 + +const enOnes = ['zero', 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine', 'ten', 'eleven', 'twelve', 'thirteen', 'fourteen', 'fifteen', 'sixteen', 'seventeen', 'eighteen', 'nineteen'] +const enTens = ['', '', 'twenty', 'thirty', 'forty', 'fifty', 'sixty', 'seventy', 'eighty', 'ninety'] +const enScales = ['', 'thousand', 'million', 'billion'] + +function enBelowThousand(value: number): string { + const words: string[] = [] + const hundreds = Math.floor(value / 100) + const rest = value % 100 + if (hundreds > 0) + words.push(`${enOnes[hundreds]} hundred`) + if (rest >= 20) + words.push(rest % 10 === 0 ? enTens[Math.floor(rest / 10)]! : `${enTens[Math.floor(rest / 10)]}-${enOnes[rest % 10]}`) + else if (rest > 0) + words.push(enOnes[rest]!) + return words.join(' ') +} + +function enInteger(digits: string): string { + const value = Number(digits) + if (value === 0) + return 'zero' + const words: string[] = [] + for (let scale = enScales.length - 1; scale >= 0; scale--) { + const group = Math.floor(value / 1000 ** scale) % 1000 + if (group > 0) + words.push([enBelowThousand(group), enScales[scale]].filter(Boolean).join(' ')) + } + return words.join(' ') +} + +const jaDigits = ['ゼロ', 'いち', 'に', 'さん', 'よん', 'ご', 'ろく', 'なな', 'はち', 'きゅう'] +/** Rendaku and gemination of 百 and 千, for example さんびゃく and はっせん. */ +const jaHundreds = ['', 'ひゃく', 'にひゃく', 'さんびゃく', 'よんひゃく', 'ごひゃく', 'ろっぴゃく', 'ななひゃく', 'はっぴゃく', 'きゅうひゃく'] +const jaThousands = ['', 'せん', 'にせん', 'さんぜん', 'よんせん', 'ごせん', 'ろくせん', 'ななせん', 'はっせん', 'きゅうせん'] +const jaScales = ['', 'まん', 'おく'] + +function jaBelowTenThousand(value: number): string { + const thousands = Math.floor(value / 1000) + const hundreds = Math.floor(value / 100) % 10 + const tens = Math.floor(value / 10) % 10 + const ones = value % 10 + return [ + jaThousands[thousands], + jaHundreds[hundreds], + tens === 0 ? '' : tens === 1 ? 'じゅう' : `${jaDigits[tens]}じゅう`, + ones === 0 ? '' : jaDigits[ones], + ].join('') +} + +function jaInteger(digits: string): string { + const value = Number(digits) + if (value === 0) + return jaDigits[0]! + let result = '' + for (let scale = jaScales.length - 1; scale >= 0; scale--) { + const group = Math.floor(value / 10000 ** scale) % 10000 + if (group === 0) + continue + // 一万 is いちまん, but 千 is せん without いち. Before 万 and 億, 一千 is いっせん: 一千万 is いっせんまん. + const thousand = scale > 0 && Math.floor(group / 1000) === 1 ? 'いっ' : '' + result += (scale > 0 && group === 1 ? 'いち' : thousand + jaBelowTenThousand(group)) + jaScales[scale] + } + return result +} + +const koDigits = ['영', '일', '이', '삼', '사', '오', '육', '칠', '팔', '구'] +const koScales = ['', '만', '억'] + +function koBelowTenThousand(value: number): string { + const parts = [[Math.floor(value / 1000), '천'], [Math.floor(value / 100) % 10, '백'], [Math.floor(value / 10) % 10, '십']] as const + // Sino-Korean drops 일 before 십, 백, and 천: 십이, not 일십이. + const prefix = parts.map(([count, unit]) => count === 0 ? '' : `${count === 1 ? '' : koDigits[count]}${unit}`).join('') + const ones = value % 10 + return prefix + (ones === 0 ? '' : koDigits[ones]) +} + +function koInteger(digits: string): string { + const value = Number(digits) + if (value === 0) + return koDigits[0]! + let result = '' + for (let scale = koScales.length - 1; scale >= 0; scale--) { + const group = Math.floor(value / 10000 ** scale) % 10000 + if (group === 0) + continue + // 만 alone is 10000, without 일. + result += (scale === 1 && group === 1 ? '' : koBelowTenThousand(group)) + koScales[scale] + } + return result +} + +/** The languages that spell numbers. Chinese keeps digits, because Chinese voices read them as Chinese already. */ +export type NumberLanguage = 'en' | 'ja' | 'ko' + +const spellers: Readonly string, digit: readonly string[], point: (integer: string, fraction: string) => string }>> = { + en: { integer: enInteger, digit: enOnes.slice(0, 10), point: (i, f) => `${i} point ${f}` }, + ja: { integer: jaInteger, digit: jaDigits, point: (i, f) => `${i}てん${f}` }, + ko: { integer: koInteger, digit: koDigits, point: (i, f) => `${i} 점 ${f}` }, +} + +/** + * Spells a decimal number as words. Digits after the point are read one by one, as people read them. + * A number with grouping commas is read as its value. A very long integer is read digit by digit. + * + * @example + * spellNumber('1,200', 'en') + * // => 'one thousand two hundred' + * spellNumber('3.14', 'ja') + * // => 'さんてんいちよん' + * spellNumber('14', 'ko') + * // => '십사' + */ +export function spellNumber(source: string, language: NumberLanguage): string { + const speller = spellers[language] + const [integer = '', fraction] = source.replaceAll(',', '').split('.') + // English digits are words, so they need spaces. Kana and hangul digits are one syllable each and join, as 일사. + const byDigit = (digits: string) => [...digits].map(digit => speller.digit[Number(digit)]).join(language === 'en' ? ' ' : '') + // A leading zero, as `007`, is a code or a label, so its digits are read one by one and the zeros stay. + const whole = integer.length > maxGroupedDigits || /^0\d/.test(integer) ? byDigit(integer) : speller.integer(integer || '0') + return fraction === undefined ? whole : speller.point(whole, byDigit(fraction)) +} diff --git a/packages/speech-transformer-math/src/parse.test.ts b/packages/speech-transformer-math/src/parse.test.ts new file mode 100644 index 0000000000..c86900c104 --- /dev/null +++ b/packages/speech-transformer-math/src/parse.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, it } from 'vitest' + +import { parseMath } from './parse' + +describe('parseMath', () => { + it('joins the digits that KaTeX splits, also when the last digit has a script', () => { + expect(parseMath('3.14')).toEqual([{ type: 'symbol', value: '3.14' }]) + expect(parseMath('10^{8}')).toEqual([{ type: 'scripts', base: { type: 'symbol', value: '10' }, sup: [{ type: 'symbol', value: '8' }] }]) + // A point that does not sit between two digits stays a point. + expect(parseMath('3.')).toEqual([{ type: 'symbol', value: '3' }, { type: 'symbol', value: '.' }]) + }) + + it('unwraps the groups that KaTeX adds around scripts and arguments', () => { + expect(parseMath(String.raw`\sum_{i=1}^{n}`)).toEqual([{ + type: 'scripts', + base: { type: 'symbol', value: '\\sum' }, + sub: [{ type: 'symbol', value: 'i' }, { type: 'symbol', value: '=' }, { type: 'symbol', value: '1' }], + sup: [{ type: 'symbol', value: 'n' }], + }]) + expect(parseMath(String.raw`\frac{1}{x}`)).toEqual([{ type: 'command', name: 'frac', args: [[{ type: 'symbol', value: '1' }], [{ type: 'symbol', value: 'x' }]] }]) + }) + + it('pairs delimiters into a fence, and moves a script of the closing delimiter onto it', () => { + const [fence] = parseMath(String.raw`\left(a+b\right)^2`) + expect(fence).toMatchObject({ type: 'fence', open: '(', close: ')', sup: [{ type: 'symbol', value: '2' }] }) + expect(parseMath('|x|')).toMatchObject([{ type: 'fence', open: '|', close: '|' }]) + }) + + it('reads negations from the MathML branch of KaTeX', () => { + expect(parseMath(String.raw`a \neq b`)[1]).toEqual({ type: 'symbol', value: '≠' }) + expect(parseMath(String.raw`a \notin A`)[1]).toEqual({ type: 'symbol', value: '∉' }) + expect(parseMath(String.raw`a \not\subseteq B`)[1]).toEqual({ type: 'symbol', value: '\\nsubseteq' }) + }) + + it('keeps units as text, and lifts the raw commands of extensions out of KaTeX', () => { + expect(parseMath(String.raw`9.8\,\mathrm{m/s^2}`)).toEqual([{ type: 'symbol', value: '9.8' }, { type: 'text', name: 'mathrm', value: 'm/s^2' }]) + expect(parseMath(String.raw`\ce{2H2 + O2 -> 2H2O}`, ['ce'])).toEqual([{ type: 'raw', command: 'ce', value: '2H2 + O2 -> 2H2O' }]) + }) + + it('balances braces, and keeps the source of a formula that KaTeX cannot parse', () => { + expect(parseMath(String.raw`\frac{1}{`)).toEqual([{ type: 'command', name: 'frac', args: [[{ type: 'symbol', value: '1' }], []] }]) + expect(parseMath('}{')).toEqual([{ type: 'group', children: [] }]) + expect(parseMath(String.raw`\left( x`)).toEqual([{ type: 'text', name: 'text', value: 'left( x' }]) + }) +}) diff --git a/packages/speech-transformer-math/src/parse.ts b/packages/speech-transformer-math/src/parse.ts new file mode 100644 index 0000000000..66d5ed76f8 --- /dev/null +++ b/packages/speech-transformer-math/src/parse.ts @@ -0,0 +1,431 @@ +import type { TexNode } from './tex' + +import katex from 'katex' + +import { fence } from './tex' + +/** + * The fields of a KaTeX parse node that the speech pipeline reads. + * + * KaTeX does not export the types of its parse tree, so this interface lists only the fields in use. `body` is a node, + * a list, or the rows of an array, depending on `type`. + */ +interface KatexNode { + type: string + text?: string + name?: string + font?: string + label?: string + mclass?: string + left?: string + right?: string + hasBarLine?: boolean + body?: KatexNode | KatexNode[] | KatexNode[][] + base?: KatexNode | null + sup?: KatexNode | null + sub?: KatexNode | null + numer?: KatexNode + denom?: KatexNode + index?: KatexNode | null + below?: KatexNode | null + /** `align` for the arrays of `aligned` and `align`, whose rows are statements. */ + colSeparationType?: string + /** Where the node starts and ends in the source that KaTeX parsed. */ + loc?: { start: number, end: number } | null + mathml?: KatexNode[] +} + +type KatexParse = (source: string, settings: { throwOnError: boolean, strict: boolean }) => KatexNode[] + +// NOTICE: +// `katex.__parse` returns the parse tree, but KaTeX marks it as an internal API and its types omit it. +// The stage apps render with the same KaTeX: pnpm-workspace.yaml overrides the katex of rehype-katex to the catalog +// version, so speech reads the formula that the user sees. +// Source: node_modules/katex/katex.js (`__parse`), KaTeX 0.19.0. The catalog pins the exact version. +// Remove when KaTeX exports a typed parse function. +const katexParse = Reflect.get(katex, '__parse') as KatexParse + +/** + * Marks a placeholder of a raw command inside `\text{}`, see {@link liftRaw}. KaTeX rejects private-use characters in + * text, so the marker is an ASCII word that a reply does not write. + */ +const rawMarker = 'airiraw' + +/** A command that an extension reads from its source, as `\ce{H2O}` of chemistry. */ +interface RawCommand { + command: string + value: string +} + +/** `\not` before a relation: KaTeX writes it as a combining long solidus in the MathML branch. */ +const notMark = '\u0338' +const negations: Readonly> = { '=': '\\neq', '\\in': '\\notin', '\\subseteq': '\\nsubseteq', '\\supseteq': '\\nsupseteq', '\\subset': '\\nsubseteq', '\\supset': '\\nsupseteq', '\\exists': '\\nexists' } + +/** Fonts that show plain text, so their content is read as written, for example a unit in `\mathrm{m/s}`. */ +const textFonts = new Set(['mathrm', 'textrm', 'mathit', 'textit', 'mathbf', 'textbf', 'mathsf', 'mathtt', 'rm', 'it', 'bf']) +/** Fonts that the reader handles as a command, for example `\mathbb{R}`. */ +const commandFonts = new Set(['mathbb', 'mathcal', 'mathfrak', 'boldsymbol', 'bm', 'pmb']) +/** Nodes that change only spacing, size, or color. Their content, if any, is read. */ +const transparent = new Set(['styling', 'sizing', 'color', 'smash', 'lap', 'internal']) +/** Nodes that have no sound. */ +const silent = new Set(['kern', 'spacing', 'vphantom', 'phantom', 'hphantom', 'raw', 'cr', 'tag', 'middle']) + +const digit = /^\d$/ +/** KaTeX keeps the escape of these characters, but the reader matches the character, as in `\#A`. */ +const escapedSpecials: Readonly> = { '\\#': '#', '\\$': '$', '\\&': '&', '\\_': '_' } + +/** + * Closes open braces and drops closing braces without a group, so KaTeX can parse a formula that a reply cut short. + * + * @example + * balanceBraces('\\frac{1}{') + * // => '\\frac{1}{}' + */ +function balanceBraces(source: string): string { + let depth = 0 + let result = '' + for (let index = 0; index < source.length; index++) { + const char = source[index]! + if (char === '\\') { + result += source.slice(index, index + 2) + index++ + continue + } + if (char === '}' && depth === 0) + continue + if (char === '{') + depth++ + if (char === '}') + depth-- + result += char + } + return result + '}'.repeat(depth) +} + +/** + * Replaces each raw command, as `\ce{...}`, with a text placeholder. KaTeX does not know the commands of an extension, + * and the extension reads their source itself, so KaTeX must not parse it. + */ +function liftRaw(source: string, commands: readonly string[]): { source: string, raw: RawCommand[] } { + const raw: RawCommand[] = [] + if (commands.length === 0) + return { source, raw } + const pattern = new RegExp(`\\\\(${commands.map(command => command.replace(/\W/g, '')).join('|')})\\s*\\{`, 'g') + let result = '' + let last = 0 + for (const match of source.matchAll(pattern)) { + if (match.index < last) + continue + const open = match.index + match[0].length + let depth = 1 + let end = open + for (; end < source.length && depth > 0; end++) { + if (source[end] === '\\') { + end++ + continue + } + if (source[end] === '{') + depth++ + if (source[end] === '}') + depth-- + } + raw.push({ command: match[1]!, value: source.slice(open, depth === 0 ? end - 1 : end) }) + result += `${source.slice(last, match.index)}\\text{${rawMarker}${raw.length - 1}}` + last = end + } + return { source: result + source.slice(last), raw } +} + +function asList(body: KatexNode['body']): KatexNode[] { + if (body === undefined) + return [] + if (!Array.isArray(body)) + return [body] + return body.flat() +} + +/** Writes the content of a text node back as source text, for example `m/s^2` from `\mathrm{m/s^2}`. */ +function plainText(nodes: readonly KatexNode[]): string { + return nodes.map((node) => { + if (node.type === 'supsub') + return plainText(node.base ? [node.base] : []) + (node.sub ? `_${plainText([node.sub])}` : '') + (node.sup ? `^${plainText([node.sup])}` : '') + if (node.type === 'spacing' || node.type === 'kern') + return ' ' + if (node.text !== undefined) + return node.text + return plainText(asList(node.body)) + }).join('') +} + +/** + * Converts the KaTeX parse tree into the syntax tree of the reader. + * + * It joins what KaTeX splits for display: digits and a decimal point become one number, also when the last digit + * carries a script, as in `10^{8}`. It reads negations from the MathML branch, lifts `\ce{}` content, and drops nodes + * without sound. Delimiters are paired afterwards by {@link fence}. + */ +/** + * The kind of an array by its delimiters: `cases` in a left brace, a matrix in parentheses or brackets, and a + * determinant in bars. + */ +const arrayKinds: Readonly> = { + '\\{.': 'cases', + '\\lbrace.': 'cases', + '()': 'matrix', + '[]': 'matrix', + '\\lbrack\\rbrack': 'matrix', + '||': 'determinant', + '\\vert\\vert': 'determinant', +} + +/** + * The rows of an array, with `&` between the cells. `aligned` and `align` hold statements, and other arrays hold + * cells. The kind of an array in delimiters is set by the caller. + */ +function rowsOf(node: KatexNode, context: NormalizeContext): Extract { + return { + type: 'rows', + name: node.colSeparationType === 'align' || node.colSeparationType === 'alignat' ? 'aligned' : 'array', + rows: (node.body as KatexNode[][] | undefined ?? []).map(row => row.flatMap((cell, column) => [ + ...(column > 0 ? [{ type: 'symbol', value: '&' } as TexNode] : []), + ...normalize([cell], context), + ])), + } +} + +/** What {@link normalize} reads besides the nodes: the lifted raw commands, and the source that KaTeX parsed. */ +interface NormalizeContext { + raw: readonly RawCommand[] + source: string +} + +/** Spacing that groups the digits of one number, as `1\,000`. Wider spacing, as `\quad`, separates two numbers. */ +const thinSpaces = new Set(['\\,', '\\!', '\\thinspace']) + +function normalize(nodes: readonly KatexNode[], context: NormalizeContext): TexNode[] { + const result: TexNode[] = [] + // True after spacing that separates numbers, so `2 \quad 3` stays two numbers. + let gap = false + + /** + * Joins a digit node to the number before it, so `9` and `0^\circ` become `90^\circ`. `join` is false when the source + * separates the digits, as `2 3` or `2 \quad 3`. KaTeX drops that space, so the reader checks the source. + */ + function pushDigits(digits: string, scripts?: { sub?: TexNode[], sup?: TexNode[] }, join = true) { + // Every digit and point before joins, not only the last one: in `45.1_{\pm 0.5}` KaTeX gives `45`, `.`, and the + // scripted `1`, and the number is 45.1. + let value = digits + for (let previous = join ? result.at(-1) : undefined; previous?.type === 'symbol' && /^[\d.]+$/.test(previous.value); previous = result.at(-1)) { + value = previous.value + value + result.pop() + } + result.push(scripts ? { type: 'scripts', base: { type: 'symbol', value }, ...scripts } : { type: 'symbol', value }) + } + + /** + * Normalizes an argument or a script. KaTeX wraps each one in a group for layout, but the reader needs the tokens, + * for example `i`, `=`, and `1` of `\sum_{i=1}`. + */ + function argument(node: KatexNode | null | undefined): TexNode[] { + if (!node) + return [] + return fence(normalize(node.type === 'ordgroup' ? asList(node.body) : [node], context)) + } + + for (let index = 0; index < nodes.length; index++) { + const node = nodes[index]! + switch (node.type) { + case 'mathord': + case 'textord': { + const value = node.text ?? '' + const previous = result.at(-1) + const afterNumber = previous?.type === 'symbol' && /^\d[\d.]*$/.test(previous.value) + // A point joins a number only between two digits, so a sentence end in `x = 3.` stays a point. + if (digit.test(value) || (value === '.' && afterNumber && digit.test(nodes[index + 1]?.text ?? ''))) { + const spaced = gap || (node.loc ? /\s/.test(context.source[node.loc.start - 1] ?? '') : false) + pushDigits(value, undefined, !spaced) + gap = false + break + } + if (value === notMark && nodes[index + 1]?.type === 'atom') { + const relation = nodes[index + 1]!.text ?? '' + result.push({ type: 'symbol', value: negations[relation] ?? relation }) + index++ + break + } + result.push({ type: 'symbol', value: value === '\\@cdots' ? '\\cdots' : escapedSpecials[value] ?? value }) + break + } + case 'atom': + result.push({ type: 'symbol', value: node.text === '\\@cdots' ? '\\cdots' : node.text ?? '' }) + break + case 'op': + if (node.name) + result.push({ type: 'symbol', value: node.name }) + else + result.push(...normalize(asList(node.body), context)) + break + case 'supsub': { + const sub = node.sub ? argument(node.sub) : undefined + const sup = node.sup ? argument(node.sup) : undefined + // Delimiters of the base pair here, so `\left(1+x\right)^n` is one fence with the power. + const base = argument(node.base) + const scripts = { ...(sub?.length ? { sub } : {}), ...(sup?.length ? { sup } : {}) } + if (base.length === 1 && base[0]!.type === 'symbol' && /^\d+$/.test(base[0]!.value)) { + pushDigits(base[0]!.value, scripts) + break + } + // A script on a delimited group belongs to the group, as in `(a+b)^2`, so the reader can say the quantity. + if (base.length === 1 && base[0]!.type === 'fence') { + result.push({ ...base[0]!, ...scripts }) + break + } + const only = base.length === 1 ? base[0]! : base.length === 0 ? { type: 'symbol' as const, value: '' } : { type: 'group' as const, children: base } + result.push({ type: 'scripts', base: only, ...scripts }) + break + } + case 'genfrac': + result.push({ type: 'command', name: node.hasBarLine ? 'frac' : 'binom', args: [argument(node.numer), argument(node.denom)] }) + break + case 'sqrt': + result.push({ type: 'command', name: 'sqrt', args: [argument(node.body as KatexNode)], ...(node.index ? { option: argument(node.index) } : {}) }) + break + case 'ordgroup': { + const children = normalize(asList(node.body), context) + // KaTeX expands `\colon` to kerns around `{:}`. The braces would hide the colon of a signature such as + // `p \colon A \to B`, so a group that holds only a colon gives the colon itself. See `defineMacro("\\colon"` in + // node_modules/katex/src/macros.ts. + if (children.length === 1 && children[0]!.type === 'symbol' && children[0]!.value === ':') + result.push(children[0]!) + else + result.push({ type: 'group', children }) + break + } + case 'leftright': { + // An array in braces or brackets is `cases` or a matrix. The delimiters only mark the kind, so they are not read. + const [only] = asList(node.body) + const kind = asList(node.body).length === 1 && only?.type === 'array' ? arrayKinds[`${node.left}${node.right}`] : undefined + if (kind) { + result.push({ ...rowsOf(only!, context), name: kind }) + break + } + if (node.left && node.left !== '.') + result.push({ type: 'symbol', value: node.left }) + result.push(...normalize(asList(node.body), context)) + if (node.right && node.right !== '.') + result.push({ type: 'symbol', value: node.right }) + break + } + case 'font': { + const font = node.font ?? '' + if (commandFonts.has(font)) + result.push({ type: 'command', name: font, args: [argument(node.body as KatexNode)] }) + else if (textFonts.has(font)) + result.push({ type: 'text', name: font, value: plainText(asList(node.body)) }) + else + result.push(...normalize(asList(node.body), context)) + break + } + case 'text': { + const value = plainText(asList(node.body)) + if (value.startsWith(rawMarker)) { + const lifted = context.raw[Number(value.slice(rawMarker.length))] + result.push({ type: 'raw', command: lifted?.command ?? '', value: lifted?.value ?? '' }) + break + } + result.push({ type: 'text', name: 'text', value }) + break + } + case 'operatorname': + result.push({ type: 'text', name: 'operatorname', value: plainText(asList(node.body)) }) + break + case 'accent': + result.push({ type: 'command', name: (node.label ?? '').replace(/^\\/, ''), args: [argument(node.base)] }) + break + case 'overline': + case 'underline': + result.push({ type: 'command', name: node.type, args: [argument(node.body as KatexNode)] }) + break + case 'mclass': { + // `\overset{\frown}{AB}` is an operator with the arc above it. + const inner = asList(node.body) + const scripts = inner.length === 1 && inner[0]!.type === 'supsub' ? inner[0]! : undefined + if (scripts?.base?.type === 'op' && scripts.sup) { + result.push({ type: 'command', name: 'overset', args: [argument(scripts.sup), fence(normalize(asList(scripts.base.body), context))] }) + break + } + result.push(...normalize(inner, context)) + break + } + case 'htmlmathml': { + // `\not` is a lone not mark in its own node, and the relation that it negates is the next sibling. + const mathml = node.mathml ?? [] + const next = nodes[index + 1] + if (mathml.length === 1 && mathml[0]!.text === notMark && next?.type === 'atom') { + result.push({ type: 'symbol', value: negations[next.text ?? ''] ?? next.text ?? '' }) + index++ + break + } + result.push(...normalize(mathml, context)) + break + } + case 'xArrow': + // A labeled arrow, as in mhchem `->[\Delta]`: the arrow, then its label over and under it. + result.push({ type: 'command', name: 'xarrow', args: [[{ type: 'symbol', value: node.label ?? '' }], argument(node.body as KatexNode), argument(node.below)] }) + break + case 'array': + result.push(rowsOf(node, context)) + break + default: + if (silent.has(node.type)) { + if (node.type === 'kern' || (node.type === 'spacing' && !thinSpaces.has(node.text ?? ''))) + gap = true + break + } + if (transparent.has(node.type) || node.body !== undefined) + result.push(...normalize(asList(node.body), context)) + } + } + return result +} + +/** + * Scientific notation as code writes it, `1e-3` or `2.5e10`, with nothing between the parts. `2e - 1` has spaces, and + * it is 2e minus 1 with Euler's number. KaTeX drops the spaces, so the rule reads the source. + */ +const scientific = /(? `${mantissa}\\times 10^{${exponent}}`) + .replace(groupedDigits, (_, head: string, groups: string) => head + groups.replaceAll(',', '')) + .replace(bracedGroups, '$1') +} + +/** + * Parses TeX math source into the syntax tree of the reader, with KaTeX. + * + * The parse is total. Unbalanced braces are balanced first. A formula that KaTeX still cannot parse becomes one text + * node with its source, so the speech keeps the writer's words instead of failing. + * + * `rawCommands` are the commands that extensions read from their source, as `ce` and `pu` of chemistry. They become + * `raw` nodes with their source, see ./extension. + */ +export function parseMath(source: string, rawCommands: readonly string[] = []): TexNode[] { + const lifted = liftRaw(rewriteNumbers(balanceBraces(source)), rawCommands) + let nodes: KatexNode[] + try { + nodes = katexParse(lifted.source, { throwOnError: false, strict: false }) + } + catch { + return [{ type: 'text', name: 'text', value: source.replace(/\\([a-z]+)/gi, '$1').replace(/[{}\\]/g, ' ').replace(/\s+/g, ' ').trim() }] + } + return fence(normalize(nodes, { raw: lifted.raw, source: lifted.source })) +} diff --git a/packages/speech-transformer-math/src/render.test.ts b/packages/speech-transformer-math/src/render.test.ts new file mode 100644 index 0000000000..f50036c3ad --- /dev/null +++ b/packages/speech-transformer-math/src/render.test.ts @@ -0,0 +1,77 @@ +import type { MathJson } from '@proj-airi/mathjson' + +import { describe, expect, it } from 'vitest' + +import { renderMath } from './render' +import { MathSpeech } from './speech' + +const speech = new MathSpeech() +const num = (value: string): MathJson => ({ num: value }) + +describe('renderMath', () => { + it('reads a call with "of" in English, and with parentheses when it is grouped', () => { + expect(renderMath(['Apply', 'f', 'x'], speech.locale('en'))).toBe('f of x') + expect(renderMath(['Apply', 'f', ['Add', 'x', num('1')]], speech.locale('en'))).toBe('f of the quantity x plus one') + expect(renderMath(['Apply', 'f', 'x', 'y'], speech.locale('en'))).toBe('f of x, y') + // A simple call keeps the reading of the product in Chinese, Japanese, and Korean. + expect(renderMath(['Apply', 'f', 'x'], speech.locale('zh'))).toBe('f x') + expect(renderMath(['Apply', 'f', 'x', 'y'], speech.locale('zh'))).toBe('f关于x和y') + expect(renderMath(['Apply', 'f', 'x', 'y'], speech.locale('ja'))).toBe('エフかっこエックス、ワイかっことじ') + expect(renderMath(['Apply', 'f', 'x', 'y'], speech.locale('ko'))).toBe('에프 괄호 엑스, 와이 괄호 닫고') + expect(renderMath(['Apply', 'p', ['Placeholder']], speech.locale('en'))).toBe('p of dot') + }) + + it('reads a sequence with pauses and a superscript mark', () => { + expect(renderMath(['Sequence', 'r', 's'], speech.locale('en'))).toBe('r, s') + expect(renderMath(['Superscript', ['NumberSet', 'N'], '+'], speech.locale('en'))).toBe('the natural numbers plus') + expect(renderMath(['Superscript', 'f', ['Delimiter', 'n']], speech.locale('en'))).toBe('f n') + }) + + it('reads an expectation with its conditions as one list', () => { + const loss: MathJson = ['Expectation', ['Power', ['Norm', ['Subtract', 'v', 'u']], num('2')], ['Clauses', ['Distributed', 't', 'q'], ['Distributed', 'x', 'p']]] + expect(renderMath(loss, speech.locale('en'))).toBe('the expected value, over t drawn from q, x drawn from p, of the squared norm of the difference between v and u') + expect(renderMath(loss, speech.locale('zh'))).toBe('在t服从q、x服从p下 v减u的范数的平方的期望') + expect(renderMath(loss, speech.locale('ja'))).toBe('ティーがキューに従う、エックスがピーに従うとしたときのブイマイナスユーのノルムの二乗の期待値') + }) + + it('says the case of a letter only when the formula has both cases of it', () => { + // A lone capital is clear, so `\Sigma` reads as sigma, as `S` reads as S. + expect(renderMath(['Element', 'r', '\\Sigma'], speech.locale('en'))).toBe('r is an element of sigma') + expect(renderMath(['Element', 'r', 'S'], speech.locale('en'))).toBe('r is an element of S') + // A Chinese teacher says 大A and 小a for a Latin letter, and keeps 大写 for a Greek name. + expect(renderMath(['Element', 'a', 'A'], speech.locale('zh'))).toBe('小a 属于 大A') + }) + + it('reads each head with the words of the language', () => { + const expression: MathJson = ['Equal', ['Add', 'a', num('1')], 'b'] + expect(renderMath(expression, speech.locale('zh'))).toBe('a加1 等于 b') + expect(renderMath(expression, speech.locale('en'))).toBe('a plus one equals b') + expect(renderMath(expression, speech.locale('ja'))).toBe('エープラスいち イコール ビー') + expect(renderMath(expression, speech.locale('ko'))).toBe('에이 더하기 일은 비') + }) + + it('chooses the fraction reading by the shape of its parts', () => { + expect(renderMath(['Divide', num('1'), num('2')], speech.locale('en'))).toBe('one over two') + expect(renderMath(['Divide', ['Subtract', 'f', num('1')], num('2')], speech.locale('en'))).toBe('the fraction with numerator f minus one and denominator two') + // After an operator, a non-numeric denominator names its parts in a language that reads the denominator first. + expect(renderMath(['Add', num('1'), ['Divide', num('1'), 'x']], speech.locale('ja'))).toBe('いちプラス分子がいち、分母がエックスの分数') + expect(renderMath(['Add', num('1'), ['Divide', num('1'), num('3')]], speech.locale('ja'))).toBe('いちプラスさん分のいち') + }) + + it('says a group only where lowering made it a group', () => { + expect(renderMath(['Delimiter', ['Add', 'a', 'b']], speech.locale('en'))).toBe('a plus b') + expect(renderMath(['Power', ['Group', ['Add', 'a', 'b']], num('2')], speech.locale('en'))).toBe('the square of the quantity a plus b') + }) + + it('says the case of a letter only when the expression has both cases of it', () => { + expect(renderMath(['Equal', 'N', ['Add', 'n', num('1')]], speech.locale('en'))).toBe('capital N equals n plus one') + expect(renderMath(['Equal', 'N', 'm'], speech.locale('en'))).toBe('N equals m') + }) + + it('reads limits, units, segments, and an operator without operands', () => { + expect(renderMath(['Limit', 'f', ['Bounds', 'x', num('0'), num('-1')]], speech.locale('en'))).toBe('the limit as x approaches zero from the left of f') + expect(renderMath(['Unit', '\'km\'', '\'h\''], speech.locale('zh'))).toBe('千米每小时') + expect(renderMath(['Segment', 'A', 'B'], speech.locale('zh'))).toBe('线段A B') + expect(renderMath(['InvisibleOperator', num('1'), '+'], speech.locale('en'))).toBe('one plus') + }) +}) diff --git a/packages/speech-transformer-math/src/render.ts b/packages/speech-transformer-math/src/render.ts new file mode 100644 index 0000000000..1b75a511f5 --- /dev/null +++ b/packages/speech-transformer-math/src/render.ts @@ -0,0 +1,462 @@ +import type { MathJson } from '@proj-airi/mathjson' + +import type { RenderHandlers } from './extension' +import type { FractionPart, FractionShape, MathLocale } from './locale' + +import { argsOf, headOf, isFunction, isNumber, isSymbol, isText, nothing, textOf } from '@proj-airi/mathjson' + +import { operatorHeads, rawHead, relationHeads } from './heads' +import { greekNames } from './locale' +import { speakUnit } from './units' + +const relationKeys: Readonly> = Object.fromEntries(Object.entries(relationHeads).map(([key, head]) => [head, key])) +const operatorKeys: Readonly> = Object.fromEntries(Object.entries(operatorHeads).map(([key, head]) => [head, key])) +const unarySigns: Readonly> = { Negate: '-', Positive: '+', PlusMinus: '\\pm', MinusPlus: '\\mp' } +const infinities = new Set(['\\infty', '∞']) +const latinLetter = /^[a-z]$/i +/** A Greek letter command, for example `\omega` or `\Omega`. Its lowercase name is the key of its case pair. */ +const greekCommand = /^\\([a-z]+)$/i +/** The names of Greek letters. A subscript number under them reads without a subscript word, as `\\theta_0`. */ +const greekLetters: ReadonlySet = new Set(greekNames) +const binaryHeads = new Set(Object.values(operatorHeads)) +const bigOperatorKinds: Readonly> = { Sum: 'sum', Product: 'prod', Integrate: 'int', BigUnion: 'union', BigIntersection: 'intersection' } +const ellipses = new Set(['\\cdots', '\\ldots', '\\dots', '…']) + +/** The case-pair key of a letter: `n` for `n` and `N`, `omega` for `\omega` and `\Omega`. Other symbols have none. */ +function caseKey(value: string): string | undefined { + if (latinLetter.test(value)) + return value.toLowerCase() + const greek = greekCommand.exec(value) + return greek ? greek[1]!.toLowerCase() : undefined +} + +function isUpperCase(value: string): boolean { + const letter = value.startsWith('\\') ? value[1]! : value + return letter !== letter.toLowerCase() +} + +/** Every symbol of an expression, also the letters of a name. */ +function symbolsOf(expression: MathJson): string[] { + if (isSymbol(expression)) + return [expression] + if (!isFunction(expression)) + return [] + return argsOf(expression).flatMap(symbolsOf) +} + +/** The letters that an expression uses in both cases, for example `n` in `N = n + 1`. Speech has no case, so only these say it. */ +function mixedCaseKeys(expression: MathJson): Set { + const seen = new Map>() + for (const value of symbolsOf(expression)) { + const key = caseKey(value) + if (key !== undefined) + seen.set(key, (seen.get(key) ?? new Set()).add(isUpperCase(value))) + } + return new Set([...seen].filter(([, cases]) => cases.size === 2).map(([key]) => key)) +} + +function contains(expression: MathJson, test: (value: MathJson) => boolean): boolean { + return test(expression) || (isFunction(expression) && argsOf(expression).some(argument => contains(argument, test))) +} + +/** True when an expression has a binary operator at any depth, as the sum in `n(n+1)`. */ +function hasOperator(expression: MathJson): boolean { + return contains(expression, value => binaryHeads.has(headOf(value) ?? '') && argsOf(value).length >= 2) +} + +/** How hard a part of a fraction is to hear, see {@link FractionPart}. */ +function fractionPart(expression: MathJson): FractionPart { + const head = headOf(expression) + if (head === undefined) + return 'simple' + if ((binaryHeads.has(head) && argsOf(expression).length >= 2) || relationKeys[head] !== undefined || head === 'Divide' || head === 'Clauses') + return 'compound' + if (head === 'Negate' || head === 'Positive') + return fractionPart(argsOf(expression)[0] ?? nothing) + if ((head === 'Group' || head === 'Delimiter') && hasOperator(expression)) + return 'compound' + if (head === 'Power' || head === 'Subscript' || head === 'Degrees' || head === 'Prime' || head === 'Complement') + return 'scripted' + if (head === 'InvisibleOperator') { + const parts = argsOf(expression).map(fractionPart) + if (parts.includes('compound')) + return 'compound' + return parts.includes('scripted') ? 'scripted' : 'simple' + } + return 'simple' +} + +/** Splits `b (mod n)` into `b` and `n`, or returns undefined when the expression does not end with a modulus. */ +function trailingModulus(value: MathJson): { rest: MathJson, modulus: MathJson } | undefined { + if (headOf(value) === 'Modulo') + return { rest: nothing, modulus: argsOf(value)[0] ?? nothing } + if (headOf(value) !== 'InvisibleOperator') + return undefined + const args = argsOf(value) + const last = args.at(-1) + if (headOf(last) !== 'Modulo') + return undefined + const rest = args.slice(0, -1) + return { rest: rest.length === 1 ? rest[0]! : ['InvisibleOperator', ...rest], modulus: argsOf(last!)[0] ?? nothing } +} + +/** + * Reads lowered MathJSON aloud with one locale pack. + * + * It decides only the words: each head maps to an entry of the pack. What the notation means was decided by + * lowering, see ./lower. + * + * `handlers` read the heads of extensions, for example `Chemistry`. A handler wins over a built-in head. + */ +export function renderMath(expression: MathJson, lexicon: MathLocale, handlers: RenderHandlers = {}): string { + const mixedCase = mixedCaseKeys(expression) + + function word(value: string): string { + const known = lexicon.words[value] + // An operator without operands, as in `1 +`, keeps its operator word. A word wins, so `\triangle` is the shape. + if (known === undefined && operatorHeads[value] !== undefined) + return lexicon.operators[value] ?? value + let spoken: string + if (known !== undefined) + spoken = known + else if (latinLetter.test(value)) + spoken = lexicon.letter(value) + else + spoken = value.startsWith('\\') ? value.slice(1) : value + const key = caseKey(value) + if (key === undefined) + return spoken + // A letter says its case only when the formula has both cases of it, as in `a \in A`. A lone `\Sigma` or `N` is + // clear, and its case words only make the reading longer. The grader folds case in that formula too. + return mixedCase.has(key) ? lexicon.letterCase(spoken, isUpperCase(value)) : spoken + } + + /** Reads text, with any TeX command inside it as a word. */ + function plain(value: string): string { + return value.replace(/\\[a-z]+/gi, match => word(match)).replace(/[{}\\]/g, '').trim() + } + + function optional(value: MathJson | undefined): string | undefined { + return value === undefined || value === nothing ? undefined : render(value) + } + + /** Reads a condition list, as in `t \sim U, x \sim p`, with the list joiner, even when its items are clauses. */ + function renderList(value: MathJson): string { + const head = headOf(value) + return head === 'Clauses' || head === 'Sequence' ? lexicon.list(argsOf(value).map(item => render(item))) : render(value) + } + + /** + * Reads a group of two simple terms with the words of the pack, for example a 与 b 的和, or returns undefined. A + * compound term inside, as in `(a+b+c)`, keeps the brackets: a 与 b 的和与 c 的和 is harder to hear than brackets. + */ + function groupWords(inner: MathJson): string | undefined { + const head = headOf(inner) + const words = head === 'Add' ? lexicon.groupOf?.sum : head === 'Subtract' ? lexicon.groupOf?.difference : undefined + const [left = nothing, right = nothing] = argsOf(inner) + // A fraction is one term, so `(1+\\frac{1}{n})` reads as 1 与 n 分之 1 的和. + const compound = (term: MathJson) => fractionPart(term) === 'compound' && headOf(term) !== 'Divide' + if (!words || argsOf(inner).length !== 2 || compound(left) || compound(right)) + return undefined + return words(render(left), render(right)) + } + + /** `\\|\\cdot\\|` names the norm itself, so its empty slot is not read: "the norm", not "the norm of dot". */ + function normArgument(inner: MathJson): string { + if (headOf(inner) === 'Subtract' && argsOf(inner).length === 2 && lexicon.normDifference) + return lexicon.normDifference(render(argsOf(inner)[0]!), render(argsOf(inner)[1]!, true)) + return headOf(inner) === 'Placeholder' ? '' : render(inner) + } + + /** Reads the items of a list or a tuple. `k = 1, \dots, d` ends at d: "one, and so on, up to d". */ + function renderItems(items: readonly MathJson[]): string[] { + const ellipsisBeforeLast = items.length >= 3 && ellipses.has(String(items.at(-2))) + if (ellipsisBeforeLast && lexicon.range) { + // `x_1, x_2, \dots, x_n` reads as x_1, then x_2 to x_n: the item before the ellipsis starts the range. + const spoken = items.slice(0, -3).map(item => render(item)) + return [...spoken, lexicon.range(render(items.at(-3)!), render(items.at(-1)!))] + } + return items.map((item, position) => ellipsisBeforeLast && position === items.length - 1 ? lexicon.upTo(render(item)) : render(item)) + } + + /** `afterOperator` is true when an operator comes right before the expression, so a fraction can name its parts. */ + function render(value: MathJson, afterOperator = false): string { + if (isNumber(value)) + return lexicon.number(value.num) + if (value === nothing) + return '' + if (isText(value)) + return plain(textOf(value)) + if (typeof value === 'string') + return word(value) + + const [head, ...args] = value + const [first = nothing, second = nothing, third = nothing] = args + + const handler = handlers[head] + if (handler) + return handler(value, { lexicon, render: argument => render(argument) }) + + const relation = relationKeys[head] + if (relation !== undefined && lexicon.chain && relationKeys[headOf(first) ?? ''] !== undefined) { + // Lowering nests a chain on the left: `a < b < c` is `Less(Less(a, b), c)`. A pack whose relations wrap their + // operands, as Korean 에이는 비보다 작음, cannot nest them, so it gets the operands and relations in order. + const operands: MathJson[] = [] + const relations: string[] = [] + let link: MathJson = value + while (relationKeys[headOf(link) ?? ''] !== undefined) { + const [left = nothing, right = nothing] = argsOf(link) + operands.unshift(right) + relations.unshift(relationKeys[headOf(link)!]!) + link = left + } + operands.unshift(link) + return lexicon.chain( + operands.map(operand => ellipses.has(String(operand)) ? undefined : render(operand)), + relations.map(key => lexicon.relations[key]!), + ).trim() + } + // `a \equiv b \pmod{n}` is a congruence. Its modulus ends the right side, after `b`. + const modulus = head === 'Equivalent' ? trailingModulus(second) : undefined + if (modulus !== undefined) + return lexicon.congruent(render(first), render(modulus.rest), render(modulus.modulus)).trim() + if (relation !== undefined) + return lexicon.relations[relation]!(render(first), render(second)).trim() + + const unary = unarySigns[head] + if (unary !== undefined && args.length === 1) { + const sign = unary === '-' ? lexicon.negative : unary === '+' ? lexicon.positive : lexicon.operators[unary]! + return lexicon.join([sign, render(first, true)]) + } + if (unary !== undefined && args.length === 0) + return unary === '-' ? lexicon.negative : unary === '+' ? lexicon.positive : lexicon.operators[unary]! + + const operator = operatorKeys[head] + if (operator !== undefined && args.length === 2) + return lexicon.join([render(first), lexicon.operators[operator]!, render(second, true)]) + + switch (head) { + case 'Clauses': + return lexicon.clauses(args.map(argument => render(argument))) + case 'Mapping': + return lexicon.mapping(render(first), render(second), render(third)) + case 'InvisibleOperator': { + const parts = args.map(argument => render(argument)) + // A spoken group has a start word but no end word, so "the quantity one minus t epsilon" for `(1-t)\epsilon` can + // be heard as `(1 - t\epsilon)`. The times word after a group ends it. In a pack with group words, as Chinese + // n 与 1 的和 for `n(n+1)`, the group has no start word either, so the times word also comes before it. + const isGroup = (value: MathJson) => headOf(value) === 'Group' + const wordGroup = (value: MathJson) => isGroup(value) && lexicon.groupOf !== undefined && groupWords(argsOf(value)[0] ?? nothing) !== undefined + return lexicon.join(parts.flatMap((part, position) => { + if (position === 0) + return [part] + const previous = args[position - 1]! + // "one millimeter", not "one millimeters". + if (headOf(args[position]!) === 'Unit' && isNumber(previous) && previous.num === '1' && lexicon.units.singular) + return [lexicon.units.singular(part)] + // A fraction has no end word either: "seven over sixty k" can be heard as 7/(60k). + return isGroup(previous) || headOf(previous) === 'Divide' || wordGroup(args[position]!) ? [lexicon.groupTimes ?? lexicon.operators['\\times']!, part] : [part] + })) + } + case 'Name': + return lexicon.join(args.map(argument => render(argument))) + // The rows of `aligned` are statements, so each one is a clause. + case 'Rows': + return lexicon.clauses(args.map(argument => render(argument))) + case 'Cases': + return lexicon.cases(args.map((item) => { + const [value = nothing, condition] = argsOf(item) + return { value: render(value), condition: condition === undefined ? undefined : render(condition) } + })) + case 'Matrix': + case 'Determinant': + return lexicon.matrix(args.map(row => argsOf(row).map(cell => render(cell))), head === 'Determinant') + case 'Sequence': + return lexicon.list(renderItems(args)) + case 'Apply': { + // `["Apply", f, ...arguments]`. A call with more than one argument, or with a sum or a relation in its only + // argument, says its parentheses, so `f(x+1)` is not heard as `f(x) + 1`. + const callArgs = args.slice(1) + const grouped = callArgs.length > 1 || (callArgs.length === 1 && fractionPart(callArgs[0]!) === 'compound') + // A name with a subscript is decorated too: Japanese キュー下付きエムティー for `q_m(t)` runs the subscript into the + // argument, so the listener cannot tell where the name ends. + const decorated = headOf(first) === 'Accent' || headOf(first) === 'Name' || headOf(first) === 'NumberSet' || headOf(first) === 'Subscript' || headOf(first) === 'Inverse' + return lexicon.apply(render(first), callArgs.map(argument => render(argument)), grouped, decorated) + } + case 'Norm': + return lexicon.norm(normArgument(first), optional(args[1]), false) + case 'Expectation': + case 'Probability': + case 'Variance': + case 'Covariance': + // `[head, body, condition?]`. A list of conditions, as in `t \sim U, x \sim p_t`, reads as one list. + return lexicon.statistic(head, render(first), args[1] === undefined ? undefined : renderList(args[1])) + case 'About': + return lexicon.about(render(first)) + case 'Distributed': + return lexicon.distributed(render(first), render(second)) + case 'Arrow': + return lexicon.arrow(render(first), render(second)) + case 'Ceil': + return lexicon.ceil(render(first)) + case 'Floor': + return lexicon.floor(render(first)) + case 'Bracket': + return lexicon.bracket(render(first)) + case 'Placeholder': + return lexicon.placeholder + case 'Superscript': { + if (typeof second === 'string' && lexicon.marks[second] !== undefined) + return lexicon.join([render(first), lexicon.marks[second]!]) + // `["Superscript", base, mark]`, where `["Delimiter", label]` is a label in parentheses, as in `f^{(n)}`. + const parenthesized = headOf(second) === 'Delimiter' + return lexicon.superscript(render(first), render(parenthesized ? argsOf(second)[0] ?? nothing : second), parenthesized) + } + case 'Text': + return plain(textOf(first)) + case 'Unit': { + const numerator = textOf(first) + if (second === nothing) + return speakUnit(numerator, lexicon.units) ?? plain(numerator) + // `s^{-1}` has no numerator: "per second". + const top = numerator ? speakUnit(numerator, lexicon.units) : '' + // A unit after "per" can have its own word, as Japanese 時 in 毎時, and is singular, as "meters per second". + const units = lexicon.units + const bottomWords = textOf(second).split(' ').map(symbol => units.denominators?.[symbol] ?? speakUnit(symbol, units)) + const bottom = bottomWords.every(word => word !== undefined) ? (units.joinFactors ?? (words => words.join(' ')))(bottomWords.map(word => units.singular?.(word!) ?? word!)) : undefined + if (top !== undefined && bottom) + return lexicon.unitRatio(top, bottom, isNumber(third) ? third.num : undefined).trim() + return plain(`${numerator}/${textOf(second)}${isNumber(third) ? `^${third.num}` : ''}`) + } + case 'Celsius': + return lexicon.celsius(render(first)) + // A raw command that no extension lowered keeps the words of its source. + case rawHead: + return plain(textOf(second)) + case 'Divide': { + const numericDenominator = isNumber(second) + const shape: FractionShape = { numerator: fractionPart(first), denominator: fractionPart(second), followsOperator: afterOperator && !numericDenominator } + return lexicon.fraction(render(first), render(second), shape) + } + case 'D': + return lexicon.derivative(render(second), render(first), isNumber(third) ? third.num : undefined) + case 'Binomial': + return lexicon.binomial(render(first), render(second)) + case 'Ordinal': + return lexicon.ordinal(render(first)) + case 'Given': + return lexicon.given(render(first), render(second)) + case 'Modulo': + return lexicon.modulo(render(first)) + case 'Inverse': + return lexicon.inverse(render(first)) + case 'Sqrt': + return lexicon.root(render(first), undefined, fractionPart(first) === 'compound') + case 'Root': + return lexicon.root(render(first), render(second)) + case 'Power': { + if (headOf(first) === 'Group' && isNumber(second) && second.num === '2' && lexicon.squareOf) + return lexicon.squareOf(render(first)) + if (headOf(first) === 'Abs' && isNumber(second) && second.num === '2' && lexicon.absoluteSquared) + return lexicon.absoluteSquared(render(argsOf(first)[0] ?? nothing)) + if (headOf(first) === 'Norm' && isNumber(second) && second.num === '2') { + const [inner = nothing, index] = argsOf(first) + return lexicon.norm(normArgument(inner), optional(index), true) + } + const base = render(first) + if (isNumber(second) && second.num === '2') + return lexicon.squared(base) + if (isNumber(second) && second.num === '3') + return lexicon.cubed(base) + return lexicon.power(base, render(second)) + } + case 'Subscript': { + const base = render(first) + // A number under a letter is said after it, as a teacher says x zero for `x_0`. A letter under a letter keeps the + // subscript word, because x t would sound like the product `x t`. + if (isNumber(second) && isSymbol(first) && (latinLetter.test(first) || greekLetters.has(first.slice(1).toLowerCase()))) + return lexicon.numberIndex(base, render(second)) + // One symbol or number is an index, as in `x_1`. A longer subscript names its bounds, as in `x_{i+1}`. + return isSymbol(second) || isNumber(second) || headOf(second) === 'Text' ? lexicon.index(base, render(second)) : lexicon.subscript(base, render(second)) + } + case 'Degrees': + return lexicon.degrees(render(first)) + case 'Prime': + return lexicon.prime(render(first)) + case 'Factorial': + return lexicon.join([render(first), lexicon.operators['!']!]) + case 'Percent': + return lexicon.percent(render(first)) + case 'Complement': + return lexicon.complement(render(first), optional(args[1])) + case 'PowerSet': + return lexicon.powerSet(render(first)) + case 'Cardinality': + return lexicon.cardinality(render(first)) + case 'Abs': + return lexicon.absolute(render(first)) + case 'Tuple': + return lexicon.tuple(renderItems(args)) + case 'Set': { + const condition = headOf(second) === 'Condition' ? argsOf(second)[0] : undefined + return condition === undefined ? lexicon.set(render(first)) : lexicon.set(render(first), render(condition)) + } + case 'Interval': { + const open = (bound: MathJson) => headOf(bound) === 'Open' + const inner = (bound: MathJson) => open(bound) ? argsOf(bound)[0] ?? nothing : bound + return lexicon.interval(render(inner(first)), render(inner(second)), { left: !open(first), right: !open(second) }) + } + case 'Group': + return groupWords(first) ?? lexicon.quantity(render(first)) + case 'Delimiter': + return render(first) + case 'NumberSet': { + const letter = isSymbol(first) ? first : undefined + return (letter && lexicon.numberSets[letter]) ?? render(first) + } + case 'Segment': + return lexicon.segment(lexicon.join(args.map(argument => render(argument)))) + case 'Arc': + return lexicon.accent.overarc!(render(first)) + case 'Accent': { + const accent = lexicon.accent[textOf(first)] + const inner = render(second) + return accent ? accent(inner) : inner + } + case 'Limit': { + const bounds = second + if (headOf(bounds) === 'Bounds') { + const [variable = nothing, target = nothing, side] = argsOf(bounds) + const direction = isNumber(side) ? (side.num === '1' ? 'right' : 'left') : undefined + return lexicon.limit(render(first), render(variable), render(target), undefined, direction) + } + if (headOf(bounds) === 'Condition') + return lexicon.limit(render(first), undefined, undefined, render(argsOf(bounds)[0] ?? nothing)) + return lexicon.limit(render(first)) + } + case 'Sum': + case 'Product': + case 'Integrate': + case 'BigUnion': + case 'BigIntersection': { + const kind = bigOperatorKinds[head]! + const bounds = second + const body = render(first) + if (headOf(bounds) === 'Bounds') { + const [variable = nothing, from = nothing, to = nothing] = argsOf(bounds) + const improper = kind === 'int' && (contains(from, item => isSymbol(item) && infinities.has(item)) || contains(to, item => isSymbol(item) && infinities.has(item))) + return lexicon.bigOperator(kind, body, { variable: optional(variable), from: optional(from), to: optional(to), improper }) + } + if (headOf(bounds) === 'Condition') { + const [condition, to] = argsOf(bounds) + return lexicon.bigOperator(kind, body, { condition: optional(condition), to: optional(to) }) + } + return lexicon.bigOperator(kind, body, {}) + } + default: + return lexicon.join(args.map(argument => render(argument))) + } + } + + return render(expression).replace(/\s+/g, ' ').trim() +} diff --git a/packages/speech-transformer-math/src/speech.bench.ts b/packages/speech-transformer-math/src/speech.bench.ts new file mode 100644 index 0000000000..4bb2f8dbe3 --- /dev/null +++ b/packages/speech-transformer-math/src/speech.bench.ts @@ -0,0 +1,49 @@ +import { bench, describe } from 'vitest' + +import { en } from './locales/en' +import { lowerMath } from './lower' +import { parseMath } from './parse' +import { renderMath } from './render' + +// Formulas of the formula speech eval, one of each stratum, with units. Chemistry is in @proj-airi/speech-transformer-chem. +const formulas = [ + String.raw`\lim_{n \to \infty}\sum_{i=1}^{n}\frac{1}{i^2}=\frac{\pi^2}{6}`, + String.raw`\lim_{x \to 0^{+}}\ln x=-\infty`, + String.raw`\sum_{k=1}^{n} k=\frac{n(n+1)}{2}`, + String.raw`\frac{1}{1+\frac{1}{x}}`, + String.raw`x=\frac{-b\pm\sqrt{b^2-4ac}}{2a}`, + String.raw`\triangle ABC \sim \triangle DEF`, + String.raw`x \in (0, b]`, + String.raw`\{x \mid x > 0\}`, + String.raw`f: X \to Y, x \mapsto y`, + String.raw`\frac{d}{dx}x^{n}=nx^{n-1}`, + String.raw`\int_0^1 x^2\,dx=\frac{1}{3}`, + String.raw`9.8\,\mathrm{m/s^2}`, + String.raw`25^\circ\mathrm{C}`, +] +const parsed = formulas.map(tex => parseMath(tex)) +const lowered = parsed.map(lowerMath) + +// The hand-written parser and reader that this pipeline replaced read these formulas about 2.4 times faster +// (0.22 ms against 0.52 ms for all of them, 2026-10-20). KaTeX parsing is the largest layer. +bench('read all formulas once', () => { + for (const tex of formulas) + renderMath(lowerMath(parseMath(tex)), en) +}) + +describe('layers', () => { + bench('parse (KaTeX and normalize)', () => { + for (const tex of formulas) + parseMath(tex) + }) + + bench('lower (syntax to MathJSON)', () => { + for (const tree of parsed) + lowerMath(tree) + }) + + bench('render (MathJSON to words)', () => { + for (const expression of lowered) + renderMath(expression, en) + }) +}) diff --git a/packages/speech-transformer-math/src/speech.test.ts b/packages/speech-transformer-math/src/speech.test.ts new file mode 100644 index 0000000000..4a21b22b77 --- /dev/null +++ b/packages/speech-transformer-math/src/speech.test.ts @@ -0,0 +1,70 @@ +import type { MathSpeechExtension } from './extension' +import type { MathLocale } from './locale' + +import { describe, expect, it } from 'vitest' + +import { en } from './locales/en' +import { MathSpeech } from './speech' + +describe('mathSpeech', () => { + it('resolves BCP 47 codes to the built-in packs', () => { + const speech = new MathSpeech() + expect(speech.locale('zh-CN').language).toBe('zh') + expect(speech.locale('yue-HK').language).toBe('zh') + expect(speech.locale('ko_KR').language).toBe('ko') + expect(speech.findLocale('th-TH')).toBeUndefined() + expect(speech.locale('th-TH')).toBe(en) + expect(speech.locale(undefined)).toBe(en) + }) + + it('adds a language that a contributor registers', () => { + const pirate: MathLocale = { + ...en, + language: 'pi', + operators: { ...en.operators, '+': 'and a' }, + } + const speech = new MathSpeech() + speech.register(pirate) + + expect(speech.findLocale('pi-CARIB')?.language).toBe('pi') + expect(speech.speak('x+1', 'pi')).toBe('x and a one') + }) + + it('replaces a built-in pack and restores it', () => { + const speech = new MathSpeech() + const restore = speech.register({ ...en, units: { ...en.units, units: { ...en.units.units, m: 'metres' } } }) + expect(speech.speak(String.raw`5\,\mathrm{km}`, 'en')).toBe('five kilometres') + + restore() + expect(speech.speak(String.raw`5\,\mathrm{km}`, 'en')).toBe('five kilometers') + }) + + it('keeps the packs of each instance apart', () => { + const custom = new MathSpeech({ locales: [{ ...en, operators: { ...en.operators, '+': 'and' } }] }) + expect(custom.speak('x+1', 'en')).toBe('x and one') + expect(new MathSpeech().speak('x+1', 'en')).toBe('x plus one') + }) + + it('lowers and reads the raw commands of an extension', () => { + const shout: MathSpeechExtension = { + name: 'shout', + rawCommands: ['shout'], + lower: (_, source) => ['Shout', `'${source}'`], + render: { Shout: ([, value]) => String(value).slice(1, -1).toUpperCase() }, + } + const speech = new MathSpeech({ extensions: [shout] }) + expect(speech.toMathJson(String.raw`\shout{hi}`)).toEqual(['Shout', '\'hi\'']) + expect(speech.speak(String.raw`x = \shout{hi}`, 'en')).toBe('x equals HI') + }) + + it('reads a raw command without an extension as its source text', () => { + expect(new MathSpeech({ extensions: [{ name: 'empty', rawCommands: ['shout'] }] }).speak(String.raw`\shout{hi}`, 'en')).toBe('hi') + }) + + it('rejects two extensions that read one command or one head', () => { + const shout: MathSpeechExtension = { name: 'shout', rawCommands: ['shout'], render: { Shout: () => '' } } + expect(() => new MathSpeech({ extensions: [shout, { name: 'other', rawCommands: ['shout'] }] })).toThrow('both read \\shout') + expect(() => new MathSpeech({ extensions: [shout, { name: 'other', render: { Shout: () => '' } }] })).toThrow('both read the head Shout') + expect(new MathSpeech({ extensions: [shout] }).rawCommands).toEqual(['shout']) + }) +}) diff --git a/packages/speech-transformer-math/src/speech.ts b/packages/speech-transformer-math/src/speech.ts new file mode 100644 index 0000000000..11e88d6eb7 --- /dev/null +++ b/packages/speech-transformer-math/src/speech.ts @@ -0,0 +1,138 @@ +import type { MathJson } from '@proj-airi/mathjson' + +import type { MathSpeechExtension, RenderHandler, RenderHandlers } from './extension' +import type { MathLanguage, MathLocale } from './locale' + +import { argsOf, headOf, isFunction, textOf } from '@proj-airi/mathjson' + +import { rawHead } from './heads' +import { en } from './locales/en' +import { ja } from './locales/ja' +import { ko } from './locales/ko' +import { zh } from './locales/zh' +import { lowerMath } from './lower' +import { parseMath } from './parse' +import { renderMath } from './render' + +/** The built-in locale packs: Chinese, English, Japanese, and Korean. */ +export const builtinLocales: readonly MathLocale[] = [en, ja, ko, zh] + +export interface MathSpeechOptions { + /** The locale packs that the instance reads in. The default is {@link builtinLocales}. */ + locales?: readonly MathLocale[] + /** Notation that the core rules do not read, for example chemistry. */ + extensions?: readonly MathSpeechExtension[] + /** The language of the pack that reads a language without a pack. The default is `en`. */ + fallbackLanguage?: MathLanguage +} + +/** The primary language subtag of a BCP 47 code: `zh` for `zh-CN`, `ko` for `ko_KR`. Chinese dialect codes are `zh`. */ +function primaryLanguage(code: string): string { + const primary = code.toLowerCase().split(/[-_]/)[0] ?? '' + return primary === 'cmn' || primary === 'yue' ? 'zh' : primary +} + +/** + * Reads TeX formulas as words, with a set of locale packs and extensions. + * + * Each instance owns its packs, so two projects, or two tests, can read differently in one process. + * + * Call stack of {@link MathSpeech.speak}: + * + * speak + * -> {@link parseMath} (./parse): KaTeX parse tree to the syntax tree + * -> {@link lowerMath} (./lower): syntax tree to MathJSON, the rules of meaning + * -> extension `lower`: the source of raw commands, for example `\ce{}`, to MathJSON + * -> {@link renderMath} (./render): MathJSON to the words of the language + * + * @example + * const speech = new MathSpeech() + * speech.speak('x^2', 'en') + * // => 'x squared' + */ +export class MathSpeech { + private readonly locales = new Map() + private readonly extensions: readonly MathSpeechExtension[] + /** The TeX commands, without the backslash, whose source the extensions read, for example `ce`. */ + readonly rawCommands: readonly string[] + private readonly handlers: RenderHandlers + private readonly fallbackLanguage: MathLanguage + + constructor(options: MathSpeechOptions = {}) { + for (const locale of options.locales ?? builtinLocales) + this.locales.set(locale.language, locale) + this.extensions = options.extensions ?? [] + // Two extensions that claim one command or one head would replace each other silently, so both fail here. + const commands = new Map() + const handlers: Record = {} + const owners = new Map() + for (const extension of this.extensions) { + for (const command of extension.rawCommands ?? []) { + const owner = commands.get(command) + if (owner !== undefined) + throw new Error(`Extensions ${owner} and ${extension.name} both read \\${command}.`) + commands.set(command, extension.name) + } + for (const [head, handler] of Object.entries(extension.render ?? {})) { + const owner = owners.get(head) + if (owner !== undefined) + throw new Error(`Extensions ${owner} and ${extension.name} both read the head ${head}.`) + owners.set(head, extension.name) + handlers[head] = handler + } + } + this.rawCommands = [...commands.keys()] + this.handlers = handlers + this.fallbackLanguage = options.fallbackLanguage ?? 'en' + } + + /** Adds a locale pack, or replaces the pack of its language. Returns a function that restores the earlier pack. */ + register(locale: MathLocale): () => void { + const previous = this.locales.get(locale.language) + this.locales.set(locale.language, locale) + return () => { + if (previous) + this.locales.set(locale.language, previous) + else + this.locales.delete(locale.language) + } + } + + /** Finds the pack of a BCP 47 code, or undefined when no pack reads it. */ + findLocale(code: string | undefined): MathLocale | undefined { + return code === undefined ? undefined : this.locales.get(primaryLanguage(code)) + } + + /** Finds the pack of a BCP 47 code, or the fallback pack. */ + locale(code: string | undefined): MathLocale { + const locale = this.findLocale(code) ?? this.locales.get(this.fallbackLanguage) + if (!locale) + throw new Error(`No locale pack reads ${code ?? 'an unknown language'}, and the fallback ${this.fallbackLanguage} has no pack.`) + return locale + } + + /** + * Lowers TeX math source into MathJSON, without reading it. + * Tools that inspect the meaning of a formula use it, for example a comparison with Compute Engine. + */ + toMathJson(tex: string): MathJson { + return this.lowerRaw(lowerMath(parseMath(tex, this.rawCommands))) + } + + /** Reads TeX math source as words of one language. A language without a pack is read by the fallback pack. */ + speak(tex: string, language: string): string { + return renderMath(this.toMathJson(tex), this.locale(language), this.handlers) + } + + /** Replaces each `Raw` node with the MathJSON of the extension that owns its command. */ + private lowerRaw(expression: MathJson): MathJson { + if (!isFunction(expression)) + return expression + if (headOf(expression) === rawHead) { + const [command, source] = argsOf(expression).map(textOf) + const extension = this.extensions.find(item => item.lower && item.rawCommands?.includes(command!)) + return extension?.lower?.(command!, source!) ?? expression + } + return [expression[0], ...argsOf(expression).map(argument => this.lowerRaw(argument))] + } +} diff --git a/packages/speech-transformer-math/src/tex.ts b/packages/speech-transformer-math/src/tex.ts new file mode 100644 index 0000000000..87b42746f3 --- /dev/null +++ b/packages/speech-transformer-math/src/tex.ts @@ -0,0 +1,83 @@ +/** + * A node of the syntax tree that lowering reads. ./parse builds it from the KaTeX parse tree. + * + * It keeps only the structure that changes the spoken order: groups, scripts, fractions, roots, delimiters, and the + * commands whose argument is not math, for example `\text` and `\ce`. Every other command is a symbol, so an unknown + * command is read by its name and does not fail. + */ +export type TexNode + = | { readonly type: 'symbol', readonly value: string } + | { readonly type: 'group', readonly children: TexNode[] } + | { readonly type: 'scripts', readonly base: TexNode, sub?: TexNode[], sup?: TexNode[] } + | { readonly type: 'command', readonly name: string, readonly args: TexNode[][], readonly option?: TexNode[] } + | { readonly type: 'text', readonly name: string, readonly value: string } + /** A command that an extension reads from its source, as `\ce{H2O}`. */ + | { readonly type: 'raw', readonly command: string, readonly value: string } + | { readonly type: 'fence', readonly open: string, readonly close: string, readonly children: TexNode[], sub?: TexNode[], sup?: TexNode[] } + | { readonly type: 'rows', readonly name: string, readonly rows: TexNode[][] } + +const fences: Record = { '(': ')', '[': ']', '\\{': '\\}', '\\lbrace': '\\rbrace', '\\langle': '\\rangle', '|': '|', '\\vert': '\\vert', '\\|': '\\|', '\\lvert': '\\rvert', '\\Vert': '\\Vert', '\\lVert': '\\rVert', '‖': '‖', '\\lceil': '\\rceil', '\\lfloor': '\\rfloor', '⌈': '⌉', '⌊': '⌋' } + +/** An interval mixes parentheses and brackets, for example `[0, 1)`, so either one closes the other. */ +function closes(open: string, close: string): boolean { + if (fences[open] === close) + return true + return (open === '(' || open === '[') && (close === ')' || close === ']') +} + +/** Symbols after which a bar opens an absolute value or a norm, instead of closing one. */ +const opensAfter = new Set(['+', '-', '=', '<', '>', ',', '\\times', '\\pm', '\\leq', '\\geq', '\\neq', '\\approx', '/']) + +/** + * Pairs delimiters into fence nodes, so `(a+b)^2` squares the whole group and `|x|` is one absolute value. + * A script on the closing delimiter moves to the fence. + */ +export function fence(nodes: TexNode[]): TexNode[] { + const result: TexNode[] = [] + const stack: { open: string, start: number }[] = [] + + for (const node of nodes.map(deepFence)) { + const value = node.type === 'symbol' ? node.value : node.type === 'scripts' && node.base.type === 'symbol' ? node.base.value : undefined + const top = stack.at(-1) + // A bar after an operator or right after another opening bar opens a new pair, as the inner bars of `|x - |y||`. + const before = result.at(-1) + const opensInner = value !== undefined && top !== undefined && fences[value] === value && top.open === value + && (result.length === top.start + 1 || (before?.type === 'symbol' && opensAfter.has(before.value))) + if (value !== undefined && top && closes(top.open, value) && !opensInner) { + stack.pop() + const children = result.splice(top.start) + children.shift() + const fenced: TexNode = { type: 'fence', open: top.open, close: value, children } + if (node.type === 'scripts') { + fenced.sub = node.sub + fenced.sup = node.sup + } + result.push(fenced) + continue + } + if (node.type === 'symbol' && fences[node.value] !== undefined) + stack.push({ open: node.value, start: result.length }) + result.push(node) + } + return result +} + +function deepFence(node: TexNode): TexNode { + switch (node.type) { + case 'group': + return { ...node, children: fence(node.children) } + case 'scripts': + return { ...node, base: deepFence(node.base), sub: node.sub && fence(node.sub), sup: node.sup && fence(node.sup) } + case 'command': + return { ...node, args: node.args.map(fence), option: node.option && fence(node.option) } + case 'rows': + return { ...node, rows: node.rows.map(fence) } + default: + return node + } +} + +/** Lists the command names in a source, for example to check them against what a provider supports. */ +export function texCommands(source: string): string[] { + return [...source.matchAll(/\\([a-z]+|.)/gi)].map(match => match[1]!) +} diff --git a/packages/speech-transformer-math/src/units.test.ts b/packages/speech-transformer-math/src/units.test.ts new file mode 100644 index 0000000000..a095cb2731 --- /dev/null +++ b/packages/speech-transformer-math/src/units.test.ts @@ -0,0 +1,62 @@ +import { describe, expect, it } from 'vitest' + +import { MathSpeech } from './speech' +import { speakUnit as speakUnitIn } from './units' + +const speech = new MathSpeech() + +function speakUnit(symbol: string, language: string): string | undefined { + return speakUnitIn(symbol, speech.locale(language).units) +} + +describe('speakUnit', () => { + it.each([ + ['µm', 'micrometers', '微米', 'マイクロメートル', '마이크로미터'], + ['nm', 'nanometers', '纳米', 'ナノメートル', '나노미터'], + ['fs', 'femtoseconds', '飞秒', 'フェムト秒', '펨토초'], + ['MHz', 'megahertz', '兆赫兹', 'メガヘルツ', '메가헤르츠'], + ['kPa', 'kilopascals', '千帕斯卡', 'キロパスカル', '킬로파스칼'], + ['dam', 'decameters', '十米', 'デカメートル', '데카미터'], + ['cm^2', 'square centimeters', '平方厘米', '平方センチメートル', '제곱센티미터'], + ['m3', 'cubic meters', '立方米', '立方メートル', '세제곱미터'], + ])('reads %s with its prefix', (symbol, en, zh, ja, ko) => { + expect(speakUnit(symbol, 'en')).toBe(en) + expect(speakUnit(symbol, 'zh')).toBe(zh) + expect(speakUnit(symbol, 'ja')).toBe(ja) + expect(speakUnit(symbol, 'ko')).toBe(ko) + }) + + it('reads a whole unit before it tries a prefix', () => { + // `min` is minutes, not milli-in, and `cd` is candela, not centi-day. `Pa` is pascal, and `pA` is picoampere. + expect(speakUnit('min', 'en')).toBe('minutes') + expect(speakUnit('cd', 'en')).toBe('candelas') + expect(speakUnit('Pa', 'en')).toBe('pascals') + expect(speakUnit('pA', 'en')).toBe('picoamperes') + }) + + it('accepts the spellings of micro, and keeps the Chinese reading of kilograms', () => { + expect(speakUnit('μs', 'en')).toBe('microseconds') + expect(speakUnit('um', 'en')).toBe('micrometers') + expect(speakUnit('kg', 'zh')).toBe('公斤') + }) + + it('gives tonnes and bytes only the prefixes that they take', () => { + // `ft` is feet, not femtotonnes, and `pt` is not picotonnes. + expect(speakUnit('ft', 'en')).toBe('feet') + expect(speakUnit('pt', 'en')).toBeUndefined() + expect(speakUnit('kt', 'en')).toBe('kilotonnes') + expect(speakUnit('MB', 'en')).toBe('megabytes') + }) + + it('returns undefined for text that is not a unit', () => { + expect(speakUnit('card', 'en')).toBeUndefined() + }) +}) + +describe('units in formulas', () => { + it('reads a unit of `\\mathrm`, also with `\\mu`', () => { + expect(speech.speak(String.raw`5\,\mathrm{\mu m}`, 'zh')).toBe('5微米') + expect(speech.speak(String.raw`100\,\mathrm{km/h}`, 'en')).toBe('one hundred kilometers per hour') + expect(speech.speak(String.raw`2.4\,\mathrm{GHz}`, 'ja')).toBe('にてんよんギガヘルツ') + }) +}) diff --git a/packages/speech-transformer-math/src/units.ts b/packages/speech-transformer-math/src/units.ts new file mode 100644 index 0000000000..7231253bb8 --- /dev/null +++ b/packages/speech-transformer-math/src/units.ts @@ -0,0 +1,62 @@ +import type { UnitLocale } from './locale' + +/** + * Reads unit symbols with the SI rules of the SI brochure and NIST SP 811. + * + * A symbol is a prefix and a unit, for example `µm` is micro and meter. A locale pack names each prefix and unit once, + * and this module composes them, so a unit gets every prefix without one entry for each pair. + */ + +/** `da` is the only prefix of two letters, so a symbol tries it before the prefixes of one letter. */ +const prefixLengths = [2, 1] +/** + * Units that take only some prefixes. Without the limit, `ft` reads as femtotonnes and `pt` as picotonnes, and `dB` + * would be decibytes if decibel had no entry. + */ +const prefixLimits: Readonly>> = { + t: new Set(['k', 'M', 'G']), + B: new Set(['k', 'M', 'G', 'T', 'P', 'E']), +} + +/** Finds the words of a unit symbol without its power: an override or a whole unit first, then a prefix and a unit. */ +function unitWords(symbol: string, locale: UnitLocale): string | undefined { + const exact = locale.overrides?.[symbol] ?? locale.plain[symbol] ?? locale.units[symbol] + if (exact !== undefined) + return exact + for (const length of prefixLengths) { + const prefix = locale.prefixes[symbol.slice(0, length)] + const unit = locale.units[symbol.slice(length)] + const limit = prefixLimits[symbol.slice(length)] + if (prefix && unit && (!limit || limit.has(symbol.slice(0, length)))) + return prefix + unit + } + return undefined +} + +/** + * Reads a unit symbol, for example `km`, `µs`, `MHz`, `cm^2`, or `m3`. Returns undefined for text that is not a unit. + * `µ`, `μ`, `\mu`, and a leading `u` all write micro. + * + * @example + * speakUnit('µm', en.units) + * // => 'micrometers' + * speakUnit('cm^2', zh.units) + * // => '平方厘米' + */ +export function speakUnit(symbol: string, locale: UnitLocale): string | undefined { + if (symbol.includes(' ')) { + // `kg m2` of `\mathrm{kg\cdot m^2}`: each factor reads, in order. Only the last one is plural, as "kilogram + // square meters" and "newton meters". + const words = symbol.split(/\s+/).filter(Boolean).map(factor => speakUnit(factor, locale)) + if (words.includes(undefined)) + return undefined + const spoken = (words as string[]).map((word, index) => index < words.length - 1 ? locale.singular?.(word) ?? word : word) + return locale.joinFactors?.(spoken) ?? spoken.join(' ') + } + const normalized = symbol.replace(/\\mu\s*|μ|^u(?=[a-z])/gi, 'µ').replace(/\\Omega/g, 'Ω').replace(/\\AA/g, 'Å').trim() + const power = /^(.+?)\^?([23])$/.exec(normalized) + const words = unitWords(power ? power[1]! : normalized, locale) + if (words === undefined || !power) + return words + return locale.power(words, power[2] as '2' | '3') +} diff --git a/packages/speech-transformer-math/tsconfig.json b/packages/speech-transformer-math/tsconfig.json new file mode 100644 index 0000000000..389367200d --- /dev/null +++ b/packages/speech-transformer-math/tsconfig.json @@ -0,0 +1,14 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "lib": [ + "DOM", + "DOM.Iterable", + "ESNext" + ], + "types": [] + }, + "include": [ + "./src/**/*" + ] +} diff --git a/packages/speech-transformer-math/tsdown.config.ts b/packages/speech-transformer-math/tsdown.config.ts new file mode 100644 index 0000000000..a51052c07e --- /dev/null +++ b/packages/speech-transformer-math/tsdown.config.ts @@ -0,0 +1,6 @@ +import { defineConfig } from 'tsdown' + +export default defineConfig({ + entry: ['src/index.ts'], + dts: true, +}) diff --git a/packages/speech-transformer-math/vitest.config.ts b/packages/speech-transformer-math/vitest.config.ts new file mode 100644 index 0000000000..ceafc24120 --- /dev/null +++ b/packages/speech-transformer-math/vitest.config.ts @@ -0,0 +1,8 @@ +import { defineConfig } from 'vitest/config' + +export default defineConfig({ + test: { + environment: 'node', + include: ['src/**/*.test.ts'], + }, +}) diff --git a/packages/stage-ui/src/components/markdown/markdown-renderer.vue b/packages/stage-ui/src/components/markdown/markdown-renderer.vue index e925e448cb..3fd1951d0a 100644 --- a/packages/stage-ui/src/components/markdown/markdown-renderer.vue +++ b/packages/stage-ui/src/components/markdown/markdown-renderer.vue @@ -3,11 +3,14 @@ import DOMPurify from 'dompurify' import { ref, watch } from 'vue' +import { withoutPendingMath } from '../../composables/chat-math' import { useMarkdown } from '../../composables/markdown' interface Props { content: string class?: string | string[] + /** True while the message streams. A formula that is still open is not rendered, so it does not flash as an error. */ + streaming?: boolean } const props = defineProps() @@ -28,7 +31,7 @@ async function processRichContent(content: string, requestId: number) { } function processContent() { - const content = props.content + const content = props.streaming ? withoutPendingMath(props.content) : props.content const requestId = ++processRequestId if (!content) { @@ -49,7 +52,7 @@ function processContent() { void processRichContent(content, requestId) } -watch(() => props.content, processContent, { immediate: true }) +watch(() => [props.content, props.streaming], processContent, { immediate: true })