diff --git a/evals/tts-speech-formula/README.md b/evals/tts-speech-formula/README.md index acfad429c7..011c1fca39 100644 --- a/evals/tts-speech-formula/README.md +++ b/evals/tts-speech-formula/README.md @@ -141,7 +141,7 @@ Each case gets three listener answers. The case passes with a strict majority. Each answer also has a hearing status: `ok`, `partial`, or `unclear`. The listener sets it through structured output. An `unclear` vote scores 0 with the grading rule `unclear`, so a report tells an unintelligible clip apart from a wrong reading. The `tts_speech_formula.hearing` metric is the status of most votes. -Each case also gets three judge verdicts on the spoken text: `natural`, `mixed`, or `spelled`. The judge never sees the answer of the listener and never judges correctness. The case takes the median verdict as the `tts_speech_formula.reading` metric, and records its score as the vieval `judge` score. The judge reads the spoken text, not the ASR transcript, so ASR errors do not count against the test object. +Each case also gets three judge verdicts on the spoken text: `natural`, `mixed`, or `spelled`. The judge never sees the answer of the listener and never judges correctness. The case takes the median verdict as the `tts_speech_formula.reading` metric, and records its score as `formula.natural`. The judge reads the spoken text, not the ASR transcript, so ASR errors do not count against the test object. ### Judge calibration @@ -150,11 +150,13 @@ A person labels a fixed set of readings, and the share of labels that the judge the verdicts of the judge. 1. After a change of the judge prompt, run the eval on the cases of the set, then refresh the verdicts with - `pnpm -F @proj-airi/evals-tts-speech-formula run judge:collect`. It keeps the set. `-- --pick` picks a new one. -2. Label with `pnpm -F @proj-airi/evals-tts-speech-formula run judge:label`. It opens a local page on port 5299. Keys - 1, 2, and 3 choose natural, mixed, and spelled. The verdict of the judge shows after the label. + `pnpm -F @proj-airi/evals-tts-speech-formula run judge:collect`. It keeps the set and prints how many labels the + judge matches. `-- --pick` picks a new set and keeps the labeled readings. +2. To label a reading, set its `label` in `calibration.json` to `natural`, `mixed`, or `spelled`. `note` says why. On 2026-10-20 the judge matched 31 of 43 labels (72%). After the prompt took the labels into account, it matched 34 (79%). +On 2026-10-21, after the eval review changed the prompt and the readings, it matched 32 (74%). Three labeled readings +have new words since, so they keep an older verdict. The `tts_speech_formula.leftover_notation` metric counts the symbols and LaTeX command names that a test object left in its spoken text, for example `^`, `/`, `$`, or `mapsto`. It needs no TTS. The test `eval/test-objects/rule/src/read-aloud.test.ts` lists these leftovers of the rule engine for every case, without any model call. diff --git a/evals/tts-speech-formula/eval/cases/math-bridge/fetch.ts b/evals/tts-speech-formula/eval/cases/math-bridge/fetch.ts index 84c522242e..5783a01bc2 100644 --- a/evals/tts-speech-formula/eval/cases/math-bridge/fetch.ts +++ b/evals/tts-speech-formula/eval/cases/math-bridge/fetch.ts @@ -10,8 +10,9 @@ import { fileURLToPath } from 'node:url' import katex from 'katex' import { DuckDBDataProtocol } from '@duckdb/duckdb-wasm' -import { connect } from '@proj-airi/duckdb-wasm' -import { getBundles } from '@proj-airi/duckdb-wasm/bundles/default-node' +import { connect, drizzle } from '@proj-airi/drizzle-duckdb-wasm' +import { getBundles } from '@proj-airi/drizzle-duckdb-wasm/bundles/default-node' +import { sql } from 'drizzle-orm' import 'katex/contrib/mhchem' @@ -33,7 +34,7 @@ import 'katex/contrib/mhchem' * * main * -> download, once per revision, then verify - * -> connect (@proj-airi/duckdb-wasm), registerFileURL with NODE_FS + * -> connect (@proj-airi/drizzle-duckdb-wasm), registerFileURL with NODE_FS * -> uniform query and topic query, see {@link gradableRows} * -> sample.json */ @@ -113,8 +114,8 @@ async function download(): Promise { * About 88% of the first 16 million rows hold notation, but under 1% of the rest do: they are single letters, for * example `$ A $`. Gradable rows therefore come from the first 16 million rows, and that is the data, not the hash. */ -function gradableRows(): string { - return ` +function gradableRows() { + return sql` WITH source AS ( SELECT file_row_number AS row_offset, @@ -122,7 +123,8 @@ function gradableRows(): string { equation, context_before, context_after, "spoken_English" AS spoken_english FROM read_parquet('mathbridge.parquet', file_row_number = true) ) - SELECT *, hash(row_offset, ${seed}) AS priority + -- A bound number is a DOUBLE. The cast keeps the INTEGER of a literal seed, so the hash and the order stay the same. + SELECT *, hash(row_offset, CAST(${seed} AS INTEGER)) AS priority FROM source WHERE length(tex) BETWEEN 6 AND ${maxLength} AND regexp_matches(tex, '[\\\\^_=<>]') @@ -144,20 +146,17 @@ function usesStandardTex(tex: string): boolean { } } -/** Escapes a value for a single-quoted SQL string. The patterns are constants, so this only doubles quotes. */ -function sqlString(value: string): string { - return `'${value.replaceAll('\'', '\'\'')}'` -} - async function main() { await download() const client = await connect({ bundles: getBundles() }) + // Drizzle binds every value of a `sql` template as a parameter, so the patterns and numbers need no escaping. + const db = drizzle({ client: Promise.resolve(client) }) try { // NODE_FS reads the file from disk in pieces, so DuckDB never holds the whole gigabyte in WebAssembly memory. await client.db.registerFileURL('mathbridge.parquet', cacheFile, DuckDBDataProtocol.NODE_FS, false) - const topicCase = topics.map(([name, pattern]) => `WHEN regexp_matches(tex, ${sqlString(pattern)}) THEN ${sqlString(name)}`).join(' ') + const topicCase = sql.join(topics.map(([name, pattern]) => sql`WHEN regexp_matches(tex, ${pattern}) THEN ${name}`), sql` `) const toRow = (row: Record, stratum: string): MathBridgeRow => ({ stratum: String(row.tex).length > shortLength ? `${stratum}-long` : stratum, offset: Number(row.row_offset), @@ -167,8 +166,8 @@ async function main() { spokenEnglish: String(row.spoken_english ?? ''), }) - const uniform = await client.query(`${gradableRows()} ORDER BY priority LIMIT ${uniformCases * candidateFactor}`) - const byTopic = await client.query(` + const uniform = await db.execute>(sql`${gradableRows()} ORDER BY priority LIMIT ${uniformCases * candidateFactor}`) + const byTopic = await db.execute>(sql` SELECT * FROM (SELECT *, CASE ${topicCase} END AS topic FROM (${gradableRows()})) WHERE topic IS NOT NULL QUALIFY row_number() OVER (PARTITION BY topic ORDER BY priority) <= ${topicCases * candidateFactor} diff --git a/evals/tts-speech-formula/eval/evaluators/judge/calibration/calibration.json b/evals/tts-speech-formula/eval/evaluators/judge/calibration/calibration.json index c372bee062..bf29991057 100644 --- a/evals/tts-speech-formula/eval/evaluators/judge/calibration/calibration.json +++ b/evals/tts-speech-formula/eval/evaluators/judge/calibration/calibration.json @@ -10,7 +10,7 @@ "spoken": "I", "judge": { "reading": "natural", - "reason": "No part is spelled; “I” naturally reads the letter in the formula." + "reason": "No part is spelled; the spoken text reads the letter I naturally." }, "label": "natural", "note": "" @@ -38,7 +38,7 @@ "spoken": "double struck E of N sub alpha to the T power of v", "judge": { "reading": "natural", - "reason": "No part is spelled; the reading uses standard words for the expectation, subscript, and power." + "reason": "No part is spelled; naming the double-struck E and reading the subscript, superscript, and function argument are natural." }, "label": "natural", "note": "Check whether \"N sub alpha to the T power of v\" keeps the grouping of N_alpha^T(v)." @@ -51,8 +51,8 @@ "reference": "k=1 , \\dots , d", "spoken": "k equals one, dot dot dot, d", "judge": { - "reading": "natural", - "reason": "No part is spelled; “and so on, up to” is a natural way to read the ellipsis." + "reading": "mixed", + "reason": "“Dot dot dot” spells out the ellipsis." }, "label": "mixed", "note": "Read the ellipsis as \"and so on\" or \"up to\", not \"dot dot dot\"." @@ -66,7 +66,7 @@ "spoken": "the fraction with numerator 6 and denominator pi squared", "judge": { "reading": "natural", - "reason": "No part is spelled; the numerator-and-denominator phrasing naturally preserves the grouping of pi squared." + "reason": "No part is spelled; the numerator and denominator are read naturally." }, "label": "mixed", "note": "A simple fraction reads shorter: \"six over pi squared\"." @@ -94,7 +94,7 @@ "spoken": "open paren a plus b close paren squared equals a squared plus 2 a b plus b squared", "judge": { "reading": "spelled", - "reason": "The reading names the parentheses around the compound expression instead of reading its meaning naturally." + "reason": "“Open paren” and “close paren” spell out the grouping brackets instead of reading the quantity naturally." }, "label": "spelled", "note": "Do not read the parentheses. Say \"the square of the sum of a and b\"." @@ -108,7 +108,7 @@ "spoken": "Delta sub k is less than zero.", "judge": { "reading": "natural", - "reason": "No part is spelled; “Delta sub k is less than zero” is a natural reading." + "reason": "No part is spelled; “Delta sub k” is a natural reading of the subscript." }, "label": "natural", "note": "Good reading." @@ -136,7 +136,7 @@ "spoken": "G superscript, open parenthesis, p, close parenthesis.", "judge": { "reading": "spelled", - "reason": "“Open parenthesis” and “close parenthesis” spell out the grouping instead of reading the superscript as G to the power of p." + "reason": "The reading spells out the parentheses around the superscript." }, "label": "mixed", "note": "" @@ -150,7 +150,7 @@ "spoken": "7 over 60 bold k times bold k prime", "judge": { "reading": "mixed", - "reason": "“Bold” names formatting markup for both k’s." + "reason": "The reading spells the boldface markup by saying “bold” for each k." }, "label": "mixed", "note": "Only one k appears, so \"bold\" is not needed: \"k times k prime\"." @@ -163,8 +163,8 @@ "reference": "\\ce{2H2 + O2 -> 2H2O}", "spoken": "係数二、エイチの下付き数字二、プラス、オーの下付き数字二、右向き矢印、係数二、エイチの下付き数字二、オー", "judge": { - "reading": "mixed", - "reason": "No part is spelled; the element symbols are read by letter, and the subscripts and reaction arrow are described in Japanese." + "reading": "spelled", + "reason": "「下付き数字」と「右向き矢印」は数式の表記を説明しており、意味として読む形ではありません。" }, "label": "spelled", "note": "Reads every subscript and the arrow by name. It does not work as a reading of an equation." @@ -178,7 +178,7 @@ "spoken": "分子がディー、分母がディーエックスの分数に、エックスのエヌ乗。イコール、エヌかけるエックスの、肩にエヌひく一。", "judge": { "reading": "mixed", - "reason": "The phrase 「肩に」 spells out the exponent’s visual placement instead of reading it as a power." + "reason": "「肩に」と位置を説明しており、指数を読むのではなく組版上の位置を述べています。" }, "label": "spelled", "note": "Mostly right. Unsure whether 肩に is a usual Japanese reading." @@ -206,7 +206,7 @@ "spoken": "2 empty normal H of sub 2 normal O sub 2 right arrow 2 empty normal H of sub 2 normal O plus normal O sub 2", "judge": { "reading": "spelled", - "reason": "The reading spells the chemical symbols with markup words such as “empty” and “normal,” and says “right arrow” in English." + "reason": "It spells the chemistry element symbols with markup words such as “empty,” “normal,” and “of sub.”" }, "label": "spelled", "note": "" @@ -220,7 +220,7 @@ "spoken": "ケーがゼロから無限大までのにのケー乗分のいちの和 イコール に", "judge": { "reading": "natural", - "reason": "No part is spelled; the sum, fraction, power, and equality are read in standard Japanese mathematical terms." + "reason": "No part is spelled; the sum, fraction, power, and equality are read in standard Japanese math terms." }, "label": "natural", "note": "" @@ -234,7 +234,7 @@ "spoken": "小文字のエーは、大文字のエーに属する。", "judge": { "reading": "natural", - "reason": "No part is spelled; the reading naturally distinguishes lowercase a from uppercase A." + "reason": "No part is spelled; the case of each letter is naturally distinguished." }, "label": "natural", "note": "Shorter case words are enough, for example 小 a and 大 A." @@ -262,7 +262,7 @@ "spoken": "左中かっこ、エックス、縦棒、エックス大なりゼロ、右中かっこ。", "judge": { "reading": "spelled", - "reason": "It spells out the set braces as “left brace” and “right brace,” and reads the set-builder bar as “vertical bar.”" + "reason": "It spells out the set braces and the vertical bar instead of reading the set as the set of x such that x is greater than zero." }, "label": "spelled", "note": "" @@ -276,7 +276,7 @@ "spoken": "派的平方分之6", "judge": { "reading": "natural", - "reason": "No part is spelled; this is a natural reading of the fraction." + "reason": "No part is spelled; the denominator’s square is clear in this Mandarin fraction reading." }, "label": "natural", "note": "" @@ -290,7 +290,7 @@ "spoken": "九点八米每秒平方", "judge": { "reading": "natural", - "reason": "No part is spelled; the quantity is read naturally in Mandarin Chinese." + "reason": "No part is spelled; the number and unit are read naturally in Mandarin Chinese." }, "label": "natural", "note": "" @@ -304,7 +304,7 @@ "spoken": "小写字母 l,下标一,平行于小写字母 l,下标二。", "judge": { "reading": "mixed", - "reason": "The reading spells out “小写字母” even though the formula contains only one case of l." + "reason": "“小写字母” spells out the case of each letter even though the formula uses only one case; the subscript and parallel relation are read naturally." }, "label": "mixed", "note": "The letter has one case in the formula, so the case is not needed. If it is, say 小 l." @@ -317,8 +317,8 @@ "reference": "\\frac{6}{\\pi^2}", "spoken": "the fraction with numerator 6 and denominator pi squared", "judge": { - "reading": "natural", - "reason": "The reading is in English rather than Mandarin Chinese." + "reading": "spelled", + "reason": "The reading is mostly in English rather than Mandarin Chinese." }, "label": "mixed", "note": "Not Chinese. The fraction also reads shorter." @@ -331,8 +331,8 @@ "reference": "a \\in A", "spoken": "小写a 属于 大写A", "judge": { - "reading": "natural", - "reason": "No part is spelled." + "reading": "spelled", + "reason": "The spoken text unnecessarily specifies lowercase and uppercase for letters that appear in only one case in the formula." }, "label": "mixed", "note": "Use the short form 小a and 大A. The case words can go unless a variant of the letter appears." @@ -345,8 +345,8 @@ "reference": "\\{x \\mid x > 0\\}", "spoken": "左花括号,x,使得 x 大于零,右花括号。", "judge": { - "reading": "spelled", - "reason": "The reading names the left and right braces around the set." + "reading": "mixed", + "reason": "The reading names the left and right curly braces around the set, which spells the markup." }, "label": "spelled", "note": "The braces of a set are clear from the meaning and need no reading." @@ -360,7 +360,7 @@ "spoken": "open paren a plus b close paren squared equals a squared plus 2 a b plus b squared", "judge": { "reading": "spelled", - "reason": "The reading is mostly in English rather than Mandarin Chinese, and it names the main parentheses as “open paren” and “close paren.”" + "reason": "The reading is mostly in English rather than Mandarin Chinese, and it spells out the parentheses around the sum." }, "label": "spelled", "note": "Not Chinese, and the parentheses are read. Better: a 与 b 的和的平方 等于 a 方加 2ab 加 b 方." @@ -402,7 +402,7 @@ "spoken": "x is less than or equal to 3", "judge": { "reading": "spelled", - "reason": "The comparison is read in English rather than Mandarin Chinese." + "reason": "The reading is mostly in English rather than Mandarin Chinese." }, "label": "spelled", "note": "Reads in English, not Chinese. A reading in another language does not pass." @@ -416,7 +416,7 @@ "spoken": "Open brace, x such that x is greater than zero, close brace.", "judge": { "reading": "spelled", - "reason": "It spells out the opening and closing braces around the set." + "reason": "It names the opening and closing braces even though “the set of x such that” is a natural reading." }, "label": "mixed", "note": "The braces of a set need no reading." @@ -444,7 +444,7 @@ "spoken": "Capital sigma, subscript, open brace, capital B, subscript capital T, close brace.", "judge": { "reading": "spelled", - "reason": "The reading spells the braces and subscript markup, and adds unnecessary case labels for sigma, B, and T." + "reason": "The reading spells the subscript markup by saying “subscript,” “open brace,” and “close brace,” and adds unnecessary case labels for the only instances of sigma and B." }, "label": "spelled", "note": "" @@ -472,7 +472,7 @@ "spoken": "9.8 empty meters divided by seconds squared", "judge": { "reading": "spelled", - "reason": "“Empty” and “divided by” spell spacing and fraction notation rather than reading the unit naturally in Japanese." + "reason": "The word “empty” spells out the spacing command, and “divided by” reads the unit fraction symbolically rather than as the standard Japanese unit reading." }, "label": "mixed", "note": "\"empty\" is not a reading." @@ -485,8 +485,8 @@ "reference": "N=n+1", "spoken": "大文字のエヌ、イコール、小文字のエヌ、プラス、一。", "judge": { - "reading": "natural", - "reason": "The case words for N and n are natural because both cases appear in the formula; no part is spelled." + "reading": "spelled", + "reason": "It unnecessarily says 大文字の and 小文字の for letters that appear only in one case each." }, "label": "mixed", "note": "Shorter case words are enough, unless the formula has several variants of n." @@ -500,7 +500,7 @@ "spoken": "Celsius plus normal O sub 2 right arrow Celsius normal O sub 2", "judge": { "reading": "spelled", - "reason": "The reading is mostly English, and “sub” and “right arrow” spell the notation rather than reading it in Japanese." + "reason": "The reading is mostly English, and “sub 2” and “right arrow” spell out the notation rather than reading it naturally in Japanese." }, "label": "spelled", "note": "Reads C as Celsius, and the oxygen molecule is lost." @@ -514,7 +514,7 @@ "spoken": "2 empty Celsius plus normal O sub 2 right arrow 2 empty Celsius normal O", "judge": { "reading": "spelled", - "reason": "The reading uses the English phrases “empty Celsius,” “plus,” “normal,” and “right arrow” instead of Japanese readings." + "reason": "“Empty Celsius,” “normal,” and “right arrow” spell the notation, and the reading is mostly English." }, "label": "spelled", "note": "Reads the formula symbols, not the chemical formula." @@ -542,7 +542,7 @@ "spoken": "CuSO4 点 5 H2O", "judge": { "reading": "spelled", - "reason": "It leaves the chemical formulas “CuSO4” and “H2O” as source notation instead of reading the element symbols letter by letter." + "reason": "The reading spells the chemical formula as Latin element symbols and digits, and says “点” for the asterisk instead of reading the hydrate notation naturally." }, "label": "spelled", "note": "" @@ -556,7 +556,7 @@ "spoken": "25 raised to the composed with power Celsius", "judge": { "reading": "spelled", - "reason": "The reading uses the English phrase “raised to the ... power” instead of Mandarin Chinese." + "reason": "“Raised to the composed with power” is English markup-like wording rather than a Mandarin reading." }, "label": "spelled", "note": "Should be 25 摄氏度." @@ -570,7 +570,7 @@ "spoken": "系数二,过氧化氢分子,化学式中氢的下标是二,氧的下标是二,反应生成系数二的水分子,化学式中氢的下标是二,再加一个氧气分子,化学式中氧的下标是二。", "judge": { "reading": "natural", - "reason": "No part is spelled; the coefficients, compound names, and subscript readings use natural Mandarin terminology." + "reason": "No part is spelled; the reading uses compound names and describes the coefficients and subscripts in Mandarin." }, "label": "mixed", "note": "Too mechanical." @@ -584,7 +584,7 @@ "spoken": "lim over x right arrow infinity of 1 over x equals 0", "judge": { "reading": "spelled", - "reason": "The reading is mostly English, and “right arrow” spells the arrow symbol." + "reason": "The reading is mostly English rather than Mandarin Chinese." }, "label": "mixed", "note": "Unsure whether lim reads this way." @@ -598,7 +598,7 @@ "spoken": "f colon X right arrow Y comma x right arrow from bar y", "judge": { "reading": "spelled", - "reason": "The reading spells the symbols as “colon,” “right arrow,” “comma,” and “right arrow from bar” instead of reading their meaning in Mandarin." + "reason": "The reading uses the English markup names “colon,” “right arrow,” “comma,” and “right arrow from bar” instead of Mandarin mathematical phrasing." }, "label": "spelled", "note": "" @@ -612,7 +612,7 @@ "spoken": "시그마 아래첨자 영은 분수 시작, 분자 에프 빼기 일, 분모 이, 분수 끝과 같다.", "judge": { "reading": "mixed", - "reason": "The reading spells out fraction markup with “분수 시작” and “분수 끝.”" + "reason": "The reading spells the fraction markup with “분수 시작” and “분수 끝.”" }, "label": null, "note": "" @@ -625,8 +625,8 @@ "reference": "\\lim_{x \\to \\infty}\\frac{1}{x}=0", "spoken": "리미트 엑스가 무한대로 갈 때, 분수의 분자는 일, 분모는 엑스, 그 값은 영입니다.", "judge": { - "reading": "natural", - "reason": "No part is spelled; the limit and fraction are read in natural Korean mathematical language." + "reading": "mixed", + "reason": "The simple fraction is read in the unnecessarily explicit numerator-and-denominator form." }, "label": null, "note": "" @@ -654,7 +654,7 @@ "spoken": "왼쪽 중괄호, 엑스, 세로줄, 엑스가 영보다 크다, 오른쪽 중괄호.", "judge": { "reading": "spelled", - "reason": "The reading spells out the set’s opening and closing braces and names the vertical bar instead of reading the set-builder structure naturally." + "reason": "The reading spells out the set braces as “왼쪽 중괄호” and “오른쪽 중괄호,” and the separator as “세로줄.”" }, "label": null, "note": "" @@ -668,7 +668,7 @@ "spoken": "에이는 비와의 합집합이며, 중괄호를 열고 일, 이, 삼을 차례로 적은 뒤 중괄호를 닫은 집합과 같다.", "judge": { "reading": "mixed", - "reason": "It spells the set braces by saying to open and close the curly braces." + "reason": "The reading spells out the opening and closing curly braces around the set." }, "label": null, "note": "" @@ -682,7 +682,7 @@ "spoken": "소문자 에이는 대문자 에이의 원소", "judge": { "reading": "natural", - "reason": "No part is spelled; the reading naturally distinguishes lowercase a from uppercase A." + "reason": "No part is spelled; the case words distinguish lowercase a from uppercase A." }, "label": null, "note": "" @@ -695,8 +695,8 @@ "reference": "\\lim_{n \\to \\infty}\\left(1+\\frac{1}{n}\\right)^{n}=e", "spoken": "엔이 무한대로 갈 때 괄호 일 더하기 분자가 일, 분모가 엔인 분수 괄호 닫고의 엔 제곱의 극한은 e", "judge": { - "reading": "mixed", - "reason": "The reading is natural; it uses the standard fraction wording and names the meaningful parentheses around the compound expression." + "reading": "natural", + "reason": "No part is spelled; the fraction is naturally read by naming its numerator and denominator, and the parentheses group the compound expression." }, "label": null, "note": "" @@ -738,7 +738,7 @@ "spoken": "엑스는 영 초과 비 이하인 구간의 원소", "judge": { "reading": "natural", - "reason": "No part is spelled; the interval and membership are read naturally in Korean." + "reason": "No part is spelled; the interval is read naturally as an element between zero and b, inclusive." }, "label": null, "note": "" @@ -751,8 +751,8 @@ "reference": "\\sum_{k=1}^{n} k=\\frac{n(n+1)}{2}", "spoken": "ケーがいちからエヌまでのケーの和 イコール 分子がエヌかっこエヌプラスいちかっことじ、分母がにの分数", "judge": { - "reading": "mixed", - "reason": "The spoken reading spells the parentheses in the denominator expression by naming them." + "reading": "natural", + "reason": "No part is spelled; the sum and fraction are read using natural Japanese math phrasing." }, "label": null, "note": "" @@ -766,7 +766,7 @@ "spoken": "f 콜론 X 오른쪽 화살표 Y 콤마 x 세로선이 붙은 오른쪽 화살표 y", "judge": { "reading": "mixed", - "reason": "The reading spells the mapsto symbol as “세로선이 붙은 오른쪽 화살표” instead of reading its mapping meaning." + "reason": "The phrase “세로선이 붙은 오른쪽 화살표” describes the mapsto symbol rather than reading its meaning." }, "label": null, "note": "" @@ -780,7 +780,7 @@ "spoken": "2 기본 H 의 아래첨자 2 더하기 기본 O 아래첨자 2 오른쪽 화살표 2 기본 H 의 아래첨자 2 기본 O", "judge": { "reading": "mixed", - "reason": "“기본” spells font formatting, and “오른쪽 화살표” names the arrow symbol rather than reading its meaning." + "reason": "The reading spells the subscripts with “아래첨자” and the arrow with “오른쪽 화살표.”" }, "label": null, "note": "" @@ -794,7 +794,7 @@ "spoken": "x 오른쪽 화살표 무한대 에 대한 리미트 의 x 분의 1 은 0", "judge": { "reading": "mixed", - "reason": "The reading names the arrow as a right arrow instead of reading the limit condition naturally." + "reason": "The reading spells the limit notation as “x right arrow infinity” instead of reading the limiting condition naturally." }, "label": null, "note": "" @@ -807,8 +807,8 @@ "reference": "(a+b)^2=a^2+2ab+b^2", "spoken": "괄호 열고 a 더하기 b 괄호 닫고 제곱 은 a 제곱 더하기 2 a b 더하기 b 제곱", "judge": { - "reading": "spelled", - "reason": "The reading names the main parentheses around a plus b instead of using a natural phrase for the squared sum." + "reading": "natural", + "reason": "No part is spelled; the Korean grouping words and power readings are natural." }, "label": null, "note": "" @@ -822,7 +822,7 @@ "spoken": "시그마 i 는 1 에서 n 까지 i 제곱", "judge": { "reading": "natural", - "reason": "No part is spelled; the summation notation is read in a conventional Korean shorthand." + "reason": "No part is spelled; the summation is read naturally as a sum from i equals one to n of i squared." }, "label": null, "note": "" @@ -836,7 +836,7 @@ "spoken": "25 의 합성 함수 제곱 섭씨", "judge": { "reading": "mixed", - "reason": "The reading spells the superscript as “제곱,” which changes the degree symbol into an exponent." + "reason": "The reading spells the degree mark as a superscript and calls it a square rather than reading the temperature unit naturally." }, "label": null, "note": "" diff --git a/evals/tts-speech-formula/eval/evaluators/judge/calibration/collect.ts b/evals/tts-speech-formula/eval/evaluators/judge/calibration/collect.ts index e579605dda..71142756f4 100644 --- a/evals/tts-speech-formula/eval/evaluators/judge/calibration/collect.ts +++ b/evals/tts-speech-formula/eval/evaluators/judge/calibration/collect.ts @@ -8,7 +8,8 @@ import { formulaSpeechCases } from '../../../cases/formula-speech' import { mathBridgeCases } from '../../../cases/math-bridge' /** - * Collects the readings that a person labels in the calibration tool, see ./README.md. + * Collects the readings of the judge calibration set, and prints how many labels of a person the judge matches. See the + * judge calibration section of the eval README. * * It reads the newest `result.json` of each case from the reports. * @@ -50,8 +51,16 @@ function order(key: string): number { return hash >>> 0 } -/** The newest judged reading of each object, case, and language, without TTS. */ -async function newestReadings(): Promise { +/** The key of one spoken text of an item. A label applies to the words that a person read, not to the case. */ +function readingKey(item: Pick): string { + return `${item.id}\u0000${item.spoken}` +} + +/** + * The newest judged reading of each object, case, language, and spoken text, without TTS. A rule change gives a case + * new words, and the result for the old words stays in older reports. + */ +async function newestReadings(): Promise<{ time: number, item: CalibrationItem }[]> { const newest = new Map() for (const object of objects) { for await (const path of glob(`${object}/*/tts-speech-formula/attempt-*/run-*/artifacts/*/*/result.json`, { cwd: reportsRoot.pathname })) { @@ -60,10 +69,11 @@ async function newestReadings(): Promise { if (!currentCases.has(result.id) || result.tts !== 'none' || result.reading === undefined || typeof result.reference !== 'string' || !result.spoken.trim()) continue const id = `${object}|${result.id}|${result.language}` + const key = readingKey({ id, spoken: result.spoken }) const time = (await stat(file)).mtimeMs - if ((newest.get(id)?.time ?? 0) >= time) + if ((newest.get(key)?.time ?? 0) >= time) continue - newest.set(id, { + newest.set(key, { time, item: { id, @@ -79,7 +89,7 @@ async function newestReadings(): Promise { }) } } - return [...newest.values()].map(({ item }) => item) + return [...newest.values()] } /** @@ -108,7 +118,20 @@ function pick(readings: readonly CalibrationItem[]): CalibrationItem[] { return picked } -/** Written next to the file and renamed over it, as the labeling server does, so a reader never sees half a file. */ +/** + * Prints how many labeled readings the judge verdict matches, the measure of the judge. `stale` readings have no + * newer verdict on their words, so they keep an older verdict. + */ +function reportAgreement(items: readonly CalibrationItem[], stale = 0) { + const labeled = items.filter(item => item.label !== null) + const matched = labeled.filter(item => item.label === item.judge.reading).length + const share = labeled.length ? Math.round(100 * matched / labeled.length) : 0 + process.stdout.write(`the judge matches ${matched} of ${labeled.length} labels (${share}%)\n`) + if (stale > 0) + process.stdout.write(`${stale} readings kept an older verdict, because no report judged their words again\n`) +} + +/** Written next to the file and renamed over it, so a reader never sees half a file. */ async function write(items: CalibrationItem[]) { const partial = new URL('./calibration.json.partial', import.meta.url) await writeFile(partial, `${JSON.stringify({ seed, items } satisfies CalibrationFile, null, 2)}\n`) @@ -119,17 +142,25 @@ async function main() { const previous = await readFile(calibrationFile, 'utf8').then(text => JSON.parse(text) as CalibrationFile, () => ({ items: [] })) const labeled = new Map(previous.items.filter(item => item.label !== null).map(item => [item.id, item])) - const readings = await newestReadings() + const judged = await newestReadings() + const readings = judged.map(({ item }) => item) if (previous.items.length > 0 && !process.argv.includes('--pick')) { - const newest = new Map(readings.map(item => [item.id, item])) - const items = previous.items.map(item => ({ ...item, judge: newest.get(item.id)?.judge ?? item.judge })) + // A verdict counts only on the words that the person labeled. A case that reads differently now keeps its verdict. + const newest = new Map(readings.map(item => [readingKey(item), item])) + const items = previous.items.map(item => ({ ...item, judge: newest.get(readingKey(item))?.judge ?? item.judge })) + const stale = previous.items.filter(item => !newest.has(readingKey(item))).length await write(items) - process.stdout.write(`refreshed the judge verdicts of ${items.length} readings, ${labeled.size} labeled\n`) + process.stdout.write(`refreshed the judge verdicts of ${items.length - stale} readings, ${labeled.size} labeled\n`) + reportAgreement(items, stale) return } + // A new pick takes the newest words of each case. + const current = new Map() + for (const { item } of judged.toSorted((left, right) => left.time - right.time)) + current.set(item.id, item) const languages = [...new Set(readings.map(item => item.language))].sort() - const items = languages.flatMap(language => pick(readings.filter(item => item.language === language))) + const items = languages.flatMap(language => pick([...current.values()].filter(item => item.language === language))) // A labeled item keeps its label and note. Its judge verdict is the newest one. .map(item => ({ ...item, label: labeled.get(item.id)?.label ?? null, note: labeled.get(item.id)?.note ?? '' })) // Labeled items that the new pick left out stay, so collecting again never loses work. @@ -141,6 +172,7 @@ async function main() { await write(items) process.stdout.write(`collected ${items.length} readings from ${readings.length}, ${labeled.size} labeled\n`) + reportAgreement(items) } void main() diff --git a/evals/tts-speech-formula/eval/evaluators/judge/calibration/index.html b/evals/tts-speech-formula/eval/evaluators/judge/calibration/index.html deleted file mode 100644 index ebd2226d35..0000000000 --- a/evals/tts-speech-formula/eval/evaluators/judge/calibration/index.html +++ /dev/null @@ -1,74 +0,0 @@ - - - - - - Judge calibration - - - -
-
-
-

Judge calibration

-

Loading…

- -
-
- - -
-
- -
- - - -

- 1 natural · 2 mixed · 3 spelled · press the same key again to clear · - the judge verdict shows after you label. -

-
- -
- - - - - - diff --git a/evals/tts-speech-formula/eval/evaluators/judge/calibration/main.ts b/evals/tts-speech-formula/eval/evaluators/judge/calibration/main.ts deleted file mode 100644 index 1b36c0a5d4..0000000000 --- a/evals/tts-speech-formula/eval/evaluators/judge/calibration/main.ts +++ /dev/null @@ -1,266 +0,0 @@ -/// - -import type { CalibrationFile, CalibrationItem, ReadingLabel } from './types' - -import katex from 'katex' -import rehypeStringify from 'rehype-stringify' -import remarkParse from 'remark-parse' -import remarkRehype from 'remark-rehype' - -import { unified } from 'unified' - -import 'katex/contrib/mhchem' -import 'katex/dist/katex.min.css' - -const labels: readonly ReadingLabel[] = ['natural', 'mixed', 'spelled'] -const languageNames: Readonly> = { zh: 'Mandarin Chinese', en: 'English', ja: 'Japanese', ko: 'Korean' } - -const stage = document.querySelector('#stage')! -const template = document.querySelector('#card-template')! -const stats = document.querySelector('#stats')! -const progress = document.querySelector('#progress')! -const position = document.querySelector('#position')! -const toast = document.querySelector('#toast')! -const languageSelect = document.querySelector('#language')! -const showSelect = document.querySelector('#show')! -const previousButton = document.querySelector('#previous')! -const nextButton = document.querySelector('#next')! - -/** - * Renders the markdown of a judge reason or a note. remark-rehype drops raw HTML, so the text cannot inject markup. - */ -const markdown = unified().use(remarkParse).use(remarkRehype).use(rehypeStringify) -function renderMarkdown(text: string): string { - return String(markdown.processSync(text)) -} - -let items: CalibrationItem[] = [] -/** - * The items in the order of the current filters. It is computed when the page loads or a filter changes, never when a - * label changes, so labeling an item never moves the card to another item. - */ -let order: CalibrationItem[] = [] -let index = 0 - -function computeOrder() { - const language = languageSelect.value - const show = showSelect.value - const filtered = items.filter(item => !language || item.language === language) - if (show === 'disagree') - order = filtered.filter(item => item.label !== null && item.label !== item.judge.reading) - else if (show === 'unlabeled') - order = [...filtered.filter(item => item.label === null), ...filtered.filter(item => item.label !== null)] - else - order = filtered - index = 0 -} - -let toastTimer: ReturnType | undefined -function showToast(message: string, error = false) { - toast.textContent = message - toast.classList.toggle('error', error) - toast.classList.add('visible') - clearTimeout(toastTimer) - toastTimer = setTimeout(() => toast.classList.remove('visible'), error ? 4000 : 1400) -} - -function renderStats() { - const labeled = items.filter(item => item.label !== null) - const agree = labeled.filter(item => item.label === item.judge.reading).length - const rate = labeled.length ? `${Math.round(100 * agree / labeled.length)}%` : '–' - stats.textContent = `${labeled.length} of ${items.length} labeled · the judge agrees on ${agree} of ${labeled.length} (${rate})` - progress.style.width = `${items.length ? 100 * labeled.length / items.length : 0}%` -} - -/** Renders the formula with KaTeX. A formula that KaTeX cannot render shows a short notice above its source. */ -function renderFormula(target: HTMLElement, tex: string) { - try { - target.innerHTML = katex.renderToString(tex, { displayMode: true, throwOnError: true }) - } - catch { - target.innerHTML = '

KaTeX cannot render this source.

' - } -} - -/** A prompt for any agent: translate the reading and say how a native speaker would hear it. */ -function translationPrompt(item: CalibrationItem): string { - return [ - `This is a ${languageNames[item.language] ?? item.language} reading of a math formula for a text-to-speech voice.`, - `LaTeX: ${item.reference}`, - `Reading: ${item.spoken}`, - 'Translate the reading word by word into Chinese. Then say whether a native speaker hears it as a teacher reading the meaning, or as someone spelling the markup, such as naming every bracket or saying command names.', - ].join('\n') -} - -/** The scripts that a reading in each language must use. A reading without them is in another language. */ -const languageScripts: Readonly> = { - zh: /\p{Script=Han}/u, - ja: /[\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Han}]/u, - ko: /\p{Script=Hangul}/u, -} - -function chip(text: string, strong = false): HTMLSpanElement { - const element = document.createElement('span') - element.className = strong ? 'chip strong' : 'chip' - element.textContent = text - return element -} - -/** - * Updates only what a label changes: the chosen choice, the verdict of the judge, and the stats. The note box stays, - * so its focus and cursor stay too. - */ -function renderLabel(card: HTMLElement, item: CalibrationItem) { - for (const choice of card.querySelectorAll('.choice')) - choice.setAttribute('aria-checked', String(choice.dataset.label === item.label)) - - const judge = card.querySelector('.judge')! - // The verdict shows only after the person labels, so it does not decide the label for them. - judge.hidden = item.label === null - if (item.label !== null) { - const agree = item.label === item.judge.reading - judge.className = `judge ${agree ? 'agree' : 'disagree'}` - judge.innerHTML = `
${agree ? '✓ The judge agrees' : '✗ The judge disagrees'}: ${item.judge.reading}
` - + `
${renderMarkdown(item.judge.reason)}
` - } - renderStats() -} - -function renderNotePreview(card: HTMLElement, item: CalibrationItem) { - const preview = card.querySelector('.note-preview')! - preview.hidden = !item.note.trim() - preview.innerHTML = renderMarkdown(item.note) -} - -/** Renders the card of the current item. `direction` slides it in from the side it comes from. */ -function renderCard(direction: 'next' | 'previous' | 'none' = 'none') { - index = Math.min(Math.max(index, 0), Math.max(order.length - 1, 0)) - const item = order[index] - previousButton.disabled = index <= 0 - nextButton.disabled = index >= order.length - 1 - position.textContent = order.length ? `${index + 1} / ${order.length}` : '' - renderStats() - if (!item) { - stage.innerHTML = '
No readings match the filters.
' - return - } - - const card = template.content.firstElementChild!.cloneNode(true) as HTMLElement - if (direction !== 'none') - card.classList.add(`enter-${direction}`) - card.querySelector('.meta')!.append( - chip(languageNames[item.language] ?? item.language, true), - chip(item.object), - chip(item.caseId), - ) - renderFormula(card.querySelector('.formula')!, item.reference) - card.querySelector('.source')!.textContent = item.reference - const spoken = card.querySelector('.spoken')! - spoken.textContent = item.spoken - spoken.lang = item.language - // A test object can answer in another language, for example sre speaks only English. The text is shown as it came, - // so the warning says that this is the reading itself and not a display error. - const script = languageScripts[item.language] - if (script && !script.test(item.spoken)) { - const warning = document.createElement('p') - warning.className = 'language-warning' - warning.textContent = `This reading is not in ${languageNames[item.language] ?? item.language}. The test object wrote it this way. A reading in another language does not pass.` - spoken.after(warning) - } - - for (const choice of card.querySelectorAll('.choice')) - choice.addEventListener('click', () => choose(card, item, choice.dataset.label as ReadingLabel)) - - const noteBox = card.querySelector('textarea')! - noteBox.value = item.note - // The note is kept on the item as the person types, so a label chosen before the note is saved carries the new note. - noteBox.addEventListener('input', () => { - item.note = noteBox.value - renderNotePreview(card, item) - }) - noteBox.addEventListener('change', () => void save(item, 'Note saved')) - - card.querySelector('[data-action="copy"]')!.addEventListener('click', () => { - navigator.clipboard.writeText(translationPrompt(item)) - .then(() => showToast('Translation prompt copied')) - .catch(() => showToast('The browser blocked the clipboard', true)) - }) - - stage.replaceChildren(card) - renderLabel(card, item) - renderNotePreview(card, item) -} - -/** Saves of the page, one at a time, so a later save never lands before an earlier one. */ -let saves: Promise = Promise.resolve() - -/** Saves the label and the note of an item to calibration.json through the dev server. */ -function save(item: CalibrationItem, message: string): Promise { - const body = JSON.stringify({ id: item.id, label: item.label, note: item.note }) - const run = saves.then(async () => { - const response = await fetch('/api/items', { method: 'POST', body }) - if (!response.ok) - throw new Error(`${response.status} ${await response.text()}`) - }) - saves = run.catch(() => undefined) - return run.then( - () => showToast(message), - (error: unknown) => showToast(`Not saved: ${String(error)}`, true), - ) -} - -/** Chooses a label, or clears it when it is the label the item already has. The card stays on the item. */ -function choose(card: HTMLElement, item: CalibrationItem, label: ReadingLabel) { - item.label = item.label === label ? null : label - const choice = card.querySelector(`.choice[data-label="${label}"]`)! - choice.classList.remove('pulse') - // Reading the layout restarts the animation when the same choice is pressed twice in a row. - void choice.offsetWidth - choice.classList.add('pulse') - renderLabel(card, item) - void save(item, item.label === null ? 'Label cleared' : `Saved as ${item.label}`) -} - -function move(step: number) { - const next = index + step - if (next < 0 || next >= order.length) - return - index = next - renderCard(step > 0 ? 'next' : 'previous') -} - -document.addEventListener('keydown', (event) => { - if (event.target instanceof HTMLTextAreaElement || event.target instanceof HTMLSelectElement || event.isComposing || event.metaKey || event.ctrlKey || event.altKey) - return - const card = stage.querySelector('.card') - const item = order[index] - const chosen = ['1', '2', '3'].indexOf(event.key) - if (card && item && chosen >= 0) - choose(card, item, labels[chosen]!) - else if (event.key === 'ArrowRight' || event.key === 'Enter') - move(1) - else if (event.key === 'ArrowLeft') - move(-1) - else - return - event.preventDefault() -}) -previousButton.addEventListener('click', () => move(-1)) -nextButton.addEventListener('click', () => move(1)) -for (const select of [languageSelect, showSelect]) { - select.addEventListener('change', () => { - computeOrder() - renderCard('none') - }) -} - -async function load() { - const file = await (await fetch('/api/items')).json() as CalibrationFile - items = file.items - for (const language of [...new Set(items.map(item => item.language))].sort()) - languageSelect.append(new Option(languageNames[language] ?? language, language)) - computeOrder() - renderCard() -} - -void load() diff --git a/evals/tts-speech-formula/eval/evaluators/judge/calibration/style.css b/evals/tts-speech-formula/eval/evaluators/judge/calibration/style.css deleted file mode 100644 index f3a3e623df..0000000000 --- a/evals/tts-speech-formula/eval/evaluators/judge/calibration/style.css +++ /dev/null @@ -1,548 +0,0 @@ -/* - * Neutral surfaces with the AIRI default hue (220.44) as the accent, see DESIGN.md. - * The label colors name a state only with the text and the check mark next to them. - */ -:root { - color-scheme: light; - --hue: 220.44; - --bg: oklch(98.5% 0.004 var(--hue)); - --surface: oklch(100% 0 0); - --surface-muted: oklch(96.5% 0.006 var(--hue)); - --border: oklch(91% 0.008 var(--hue)); - --border-strong: oklch(84% 0.012 var(--hue)); - --text: oklch(24% 0.015 var(--hue)); - --muted: oklch(52% 0.015 var(--hue)); - --primary: oklch(62% 0.14 var(--hue)); - --primary-soft: oklch(94% 0.04 var(--hue)); - --natural: oklch(62% 0.15 150); - --natural-soft: oklch(95% 0.05 150); - --mixed: oklch(70% 0.15 70); - --mixed-soft: oklch(96% 0.05 80); - --spelled: oklch(60% 0.18 25); - --spelled-soft: oklch(95% 0.04 25); - --shadow: 0 1px 2px oklch(20% 0.02 var(--hue) / 6%), 0 8px 24px oklch(20% 0.02 var(--hue) / 6%); - --radius: 14px; - --ease: cubic-bezier(0.2, 0.8, 0.2, 1); -} - -@media (prefers-color-scheme: dark) { - :root { - color-scheme: dark; - --bg: oklch(17% 0.01 var(--hue)); - --surface: oklch(21% 0.012 var(--hue)); - --surface-muted: oklch(25% 0.012 var(--hue)); - --border: oklch(30% 0.012 var(--hue)); - --border-strong: oklch(38% 0.015 var(--hue)); - --text: oklch(94% 0.006 var(--hue)); - --muted: oklch(70% 0.012 var(--hue)); - --primary: oklch(72% 0.13 var(--hue)); - --primary-soft: oklch(30% 0.05 var(--hue)); - --natural-soft: oklch(30% 0.06 150); - --mixed-soft: oklch(32% 0.06 80); - --spelled-soft: oklch(30% 0.06 25); - --shadow: 0 1px 2px oklch(0% 0 0 / 30%), 0 8px 24px oklch(0% 0 0 / 25%); - } -} - -* { - box-sizing: border-box; -} - -body { - margin: 0; - background: var(--bg); - color: var(--text); - font: 15px/1.6 ui-sans-serif, system-ui, -apple-system, 'Segoe UI', 'PingFang SC', 'Hiragino Sans', 'Apple SD Gothic Neo', sans-serif; - -webkit-font-smoothing: antialiased; -} - -main { - max-width: 880px; - margin: 0 auto; - padding: 28px 16px 72px; -} - -kbd { - display: inline-grid; - place-items: center; - min-width: 1.6em; - height: 1.6em; - padding: 0 0.4em; - border: 1px solid var(--border-strong); - border-bottom-width: 2px; - border-radius: 6px; - background: var(--surface); - color: var(--muted); - font: 600 11px/1 ui-monospace, SFMono-Regular, Menlo, monospace; -} - -/* Header */ - -.top { - display: flex; - flex-wrap: wrap; - gap: 16px; - align-items: flex-end; - justify-content: space-between; - margin-bottom: 20px; -} - -.title { - flex: 1 1 280px; -} - -h1 { - margin: 0; - font-size: 22px; - font-weight: 650; - letter-spacing: -0.01em; -} - -.stats { - margin: 2px 0 10px; - color: var(--muted); - font-size: 13px; -} - -.progress { - height: 6px; - border-radius: 999px; - background: var(--surface-muted); - overflow: hidden; -} - -.progress-bar { - width: 0; - height: 100%; - border-radius: inherit; - background: var(--primary); - transition: width 400ms var(--ease); -} - -.filters { - display: flex; - gap: 10px; -} - -.field { - display: grid; - gap: 4px; - font-size: 12px; - color: var(--muted); -} - -select { - appearance: none; - min-width: 150px; - padding: 8px 32px 8px 12px; - border: 1px solid var(--border); - border-radius: 10px; - background: var(--surface) url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='12' height='12' viewBox='0 0 24 24' fill='none' stroke='%23888' stroke-width='2.5' stroke-linecap='round' stroke-linejoin='round'%3E%3Cpath d='m6 9 6 6 6-6'/%3E%3C/svg%3E") no-repeat right 12px center; - color: var(--text); - font: inherit; - font-size: 14px; - cursor: pointer; - transition: border-color 150ms var(--ease), box-shadow 150ms var(--ease); -} - -select:hover { - border-color: var(--border-strong); -} - -/* Buttons */ - -.button, -.choice { - font: inherit; - color: inherit; - cursor: pointer; - user-select: none; - -webkit-tap-highlight-color: transparent; - transition: - background-color 150ms var(--ease), - border-color 150ms var(--ease), - color 150ms var(--ease), - box-shadow 150ms var(--ease), - transform 120ms var(--ease); -} - -.button { - display: inline-flex; - align-items: center; - gap: 8px; - padding: 8px 14px; - border: 1px solid var(--border); - border-radius: 10px; - background: var(--surface); - font-size: 14px; - font-weight: 500; -} - -.button:hover:not(:disabled) { - border-color: var(--border-strong); - background: var(--surface-muted); -} - -.button:active:not(:disabled), -.choice:active { - transform: scale(0.97); -} - -.button:disabled { - opacity: 0.4; - cursor: default; -} - -.button.ghost { - border-color: transparent; - background: transparent; - color: var(--muted); - padding-inline: 10px; -} - -.button.ghost:hover { - background: var(--surface-muted); - color: var(--text); -} - -:is(.button, .choice, select, textarea):focus-visible { - outline: none; - box-shadow: 0 0 0 3px oklch(from var(--primary) l c h / 35%); -} - -/* Card */ - -.stage { - position: relative; -} - -.card { - padding: 24px; - border: 1px solid var(--border); - border-radius: var(--radius); - background: var(--surface); - box-shadow: var(--shadow); -} - -.card.enter-next { - animation: enter-next 260ms var(--ease); -} - -.card.enter-previous { - animation: enter-previous 260ms var(--ease); -} - -@keyframes enter-next { - from { opacity: 0; transform: translateX(24px); } -} - -@keyframes enter-previous { - from { opacity: 0; transform: translateX(-24px); } -} - -.meta { - display: flex; - flex-wrap: wrap; - gap: 6px; - margin-bottom: 8px; -} - -.chip { - padding: 2px 10px; - border-radius: 999px; - background: var(--surface-muted); - color: var(--muted); - font-size: 12px; - font-weight: 500; -} - -.chip.strong { - background: var(--primary-soft); - color: var(--primary); -} - -.formula { - overflow-x: auto; - padding: 8px 0; - font-size: 1.15em; -} - -.formula .katex-display { - margin: 0.6em 0; -} - -.formula-error { - color: var(--muted); - font-size: 13px; -} - -.source { - display: block; - padding: 8px 12px; - border-radius: 8px; - background: var(--surface-muted); - color: var(--muted); - font: 12px/1.5 ui-monospace, SFMono-Regular, Menlo, monospace; - white-space: pre-wrap; - word-break: break-all; -} - -.spoken { - margin: 18px 0; - padding: 18px 20px; - border-left: 3px solid var(--primary); - border-radius: 4px 12px 12px 4px; - background: var(--primary-soft); - font-size: 21px; - line-height: 1.7; - word-break: break-word; -} - -.language-warning { - margin: -8px 0 16px; - padding: 8px 12px; - border-radius: 8px; - background: var(--spelled-soft); - color: var(--text); - font-size: 13px; -} - -/* Label choices */ - -.choices { - display: grid; - grid-template-columns: repeat(3, 1fr); - gap: 10px; -} - -.choice { - position: relative; - display: grid; - grid-template-columns: auto 1fr; - grid-template-rows: auto auto; - column-gap: 10px; - align-items: center; - padding: 12px 14px; - border: 1.5px solid var(--border); - border-radius: 12px; - background: var(--surface); - text-align: left; -} - -.choice kbd { - grid-row: span 2; -} - -.choice-name { - font-weight: 650; -} - -.choice-help { - color: var(--muted); - font-size: 12px; -} - -.choice:hover { - border-color: var(--border-strong); - background: var(--surface-muted); -} - -.choice[aria-checked='true']::after { - content: '✓'; - position: absolute; - top: 8px; - right: 12px; - font-weight: 700; -} - -.choice[data-label='natural'][aria-checked='true'] { - border-color: var(--natural); - background: var(--natural-soft); -} - -.choice[data-label='mixed'][aria-checked='true'] { - border-color: var(--mixed); - background: var(--mixed-soft); -} - -.choice[data-label='spelled'][aria-checked='true'] { - border-color: var(--spelled); - background: var(--spelled-soft); -} - -.choice[data-label='natural'][aria-checked='true']::after { color: var(--natural); } -.choice[data-label='mixed'][aria-checked='true']::after { color: var(--mixed); } -.choice[data-label='spelled'][aria-checked='true']::after { color: var(--spelled); } - -/* A short pulse confirms the choice where the eye already is. */ -.choice.pulse { - animation: pulse 320ms var(--ease); -} - -@keyframes pulse { - 40% { transform: scale(1.03); } -} - -/* Note and judge */ - -.note { - display: grid; - gap: 6px; - margin-top: 18px; - color: var(--muted); - font-size: 12px; - font-weight: 500; -} - -textarea { - width: 100%; - min-height: 64px; - padding: 10px 12px; - border: 1px solid var(--border); - border-radius: 10px; - background: var(--surface); - color: var(--text); - font: inherit; - font-size: 14px; - resize: vertical; - transition: border-color 150ms var(--ease), box-shadow 150ms var(--ease); -} - -textarea:hover { - border-color: var(--border-strong); -} - -.judge, -.note-preview { - margin-top: 14px; - padding: 12px 14px; - border-radius: 10px; - font-size: 14px; - animation: reveal 220ms var(--ease); -} - -.note-preview { - background: var(--surface-muted); -} - -.judge { - border: 1px solid var(--border); -} - -.judge.agree { - border-color: var(--natural); - background: var(--natural-soft); -} - -.judge.disagree { - border-color: var(--spelled); - background: var(--spelled-soft); -} - -.judge-head { - display: flex; - gap: 8px; - align-items: center; - margin-bottom: 4px; - font-weight: 600; -} - -@keyframes reveal { - from { opacity: 0; transform: translateY(-4px); } -} - -.markdown > :first-child { margin-top: 0; } -.markdown > :last-child { margin-bottom: 0; } -.markdown p { margin: 0.3em 0; } -.markdown code { - padding: 0.1em 0.35em; - border-radius: 5px; - background: oklch(from var(--text) l c h / 8%); - font: 0.92em ui-monospace, SFMono-Regular, Menlo, monospace; -} -.markdown ul, -.markdown ol { margin: 0.3em 0; padding-left: 1.4em; } - -.tools { - display: flex; - justify-content: flex-end; - margin-top: 10px; -} - -/* Pager and hint */ - -.pager { - display: flex; - gap: 12px; - align-items: center; - justify-content: space-between; - margin-top: 16px; -} - -.position { - color: var(--muted); - font-size: 13px; - font-variant-numeric: tabular-nums; -} - -.hint { - margin-top: 20px; - color: var(--muted); - font-size: 13px; - text-align: center; -} - -.empty { - padding: 48px 24px; - color: var(--muted); - text-align: center; -} - -/* Toast */ - -.toast { - position: fixed; - top: 20px; - right: 20px; - padding: 10px 16px; - border-radius: 999px; - background: var(--text); - color: var(--bg); - font-size: 13px; - box-shadow: var(--shadow); - opacity: 0; - pointer-events: none; - transform: translateY(-8px); - transition: opacity 200ms var(--ease), transform 200ms var(--ease); -} - -.toast.visible { - opacity: 1; - transform: translateY(0); -} - -.toast.error { - background: var(--spelled); - color: white; -} - -/* Small screens */ - -@media (max-width: 640px) { - .filters { width: 100%; } - .field { flex: 1; } - select { min-width: 0; width: 100%; } - .card { padding: 18px 16px; } - .spoken { font-size: 18px; } - .choices { grid-template-columns: 1fr; } - .pager .button { padding-inline: 10px; } -} - -@media (prefers-reduced-motion: reduce) { - *, - *::before, - *::after { - animation: none !important; - transition: none !important; - } -} diff --git a/evals/tts-speech-formula/eval/evaluators/judge/calibration/vite.config.ts b/evals/tts-speech-formula/eval/evaluators/judge/calibration/vite.config.ts deleted file mode 100644 index 77b7e7f766..0000000000 --- a/evals/tts-speech-formula/eval/evaluators/judge/calibration/vite.config.ts +++ /dev/null @@ -1,90 +0,0 @@ -import type { Plugin } from 'vite' - -import type { CalibrationFile, ReadingLabel } from './types' - -import { readFile, rename, writeFile } from 'node:fs/promises' -import { env } from 'node:process' -import { fileURLToPath } from 'node:url' - -import { defineConfig } from 'vite' - -/** `CALIBRATION_FILE` points the tool at another file, for example a copy to test the tool without touching labels. */ -const calibrationFile = env.CALIBRATION_FILE ?? fileURLToPath(new URL('./calibration.json', import.meta.url)) -const labels = new Set(['natural', 'mixed', 'spelled']) - -/** - * Runs the updates of the file one at a time. - * - * Leaving a note box and clicking a label sends two saves at once. Two read-modify-write updates that overlap wrote - * over each other and left two JSON documents in one file, so every update waits for the one before it. The new - * file is written next to the old one and renamed over it, so a failed write never leaves half a file. - */ -let updates: Promise = Promise.resolve() -function update(change: (file: CalibrationFile) => boolean): Promise { - const run = updates.then(async () => { - const file = JSON.parse(await readFile(calibrationFile, 'utf8')) as CalibrationFile - if (!change(file)) - return false - const partial = `${calibrationFile}.partial` - await writeFile(partial, `${JSON.stringify(file, null, 2)}\n`) - await rename(partial, calibrationFile) - return true - }) - updates = run.catch(() => undefined) - return run -} - -/** - * Serves `calibration.json` to the page and writes each label back to it. - * - * - `GET /api/items` returns the file. - * - `POST /api/items` with `{ id, label, note }` sets the label of one item. `label: null` clears it. - * - * The file is the only state, so a label is saved as soon as the person chooses it. - */ -function calibrationApi(): Plugin { - return { - name: 'calibration-api', - configureServer(server) { - server.middlewares.use('/api/items', async (request, response) => { - if (request.method === 'GET') { - response.setHeader('content-type', 'application/json') - response.end(await readFile(calibrationFile, 'utf8')) - return - } - if (request.method !== 'POST') { - response.statusCode = 405 - response.end() - return - } - - let body = '' - for await (const chunk of request) - body += chunk - const { id, label, note } = JSON.parse(body) as { id: string, label: ReadingLabel | null, note: string } - if (label !== null && !labels.has(label)) { - response.statusCode = 400 - response.end('unknown label') - return - } - const found = await update((file) => { - const item = file.items.find(candidate => candidate.id === id) - if (!item) - return false - item.label = label - item.note = note - return true - }) - response.statusCode = found ? 200 : 404 - response.end(found ? 'ok' : 'unknown item') - }) - }, - } -} - -/** The labeling tool of the judge calibration set. Run: pnpm -F @proj-airi/evals-tts-speech-formula run judge:label */ -export default defineConfig({ - root: fileURLToPath(new URL('.', import.meta.url)), - plugins: [calibrationApi()], - server: { port: Number(env.CALIBRATION_PORT ?? 5299), open: !env.CALIBRATION_FILE }, -}) diff --git a/evals/tts-speech-formula/package.json b/evals/tts-speech-formula/package.json index 2d32fc9456..38d229d539 100644 --- a/evals/tts-speech-formula/package.json +++ b/evals/tts-speech-formula/package.json @@ -24,15 +24,14 @@ "eval:compare": "pnpm -F @proj-airi/core-agent build && vieval run --config ./vieval.config.ts --with-viewer", "typecheck": "tsc --noEmit", "cases:math-bridge": "tsx eval/cases/math-bridge/fetch.ts", - "judge:collect": "tsx eval/evaluators/judge/calibration/collect.ts", - "judge:label": "vite --config eval/evaluators/judge/calibration/vite.config.ts" + "judge:collect": "tsx eval/evaluators/judge/calibration/collect.ts" }, "dependencies": { "@cortex-js/compute-engine": "catalog:", "@duckdb/duckdb-wasm": "catalog:", "@moeru/std": "catalog:", "@proj-airi/core-agent": "workspace:^", - "@proj-airi/duckdb-wasm": "catalog:", + "@proj-airi/drizzle-duckdb-wasm": "catalog:", "@proj-airi/mathjson": "workspace:^", "@proj-airi/speech-transformer-math": "workspace:^", "@velin-dev/core-vue": "catalog:", @@ -41,6 +40,7 @@ "@xsai/generate-speech": "catalog:", "@xsai/generate-text": "catalog:", "@xsai/generate-transcription": "catalog:", + "drizzle-orm": "catalog:", "katex": "catalog:", "speech-rule-engine": "catalog:", "valibot": "catalog:", @@ -52,12 +52,7 @@ }, "devDependencies": { "@types/node": "catalog:", - "rehype-stringify": "catalog:", - "remark-parse": "catalog:", - "remark-rehype": "catalog:", - "unified": "catalog:", "vieval": "catalog:", - "vite": "catalog:", "vitest": "catalog:vitest" } } diff --git a/packages/core-agent/src/voice/output/speech-transformation/language.ts b/packages/core-agent/src/voice/output/speech-transformation/language.ts index b4c72604ef..5c28d8330a 100644 --- a/packages/core-agent/src/voice/output/speech-transformation/language.ts +++ b/packages/core-agent/src/voice/output/speech-transformation/language.ts @@ -1,14 +1,8 @@ -/** - * The primary language subtag of a BCP 47 code: `zh` for `zh-CN`, `ko` for `ko_KR`. Chinese dialect codes are `zh`, - * because the readers have one Chinese pack. - */ +import { primaryLanguage } from '@proj-airi/speech-transformer-math' + +/** The primary language subtag of a BCP 47 code, see `primaryLanguage` of `@proj-airi/speech-transformer-math`. */ export function toSpeechLanguage(code: string | undefined): string | undefined { - if (code === undefined) - return undefined - const primary = code.toLowerCase().split(/[-_]/)[0] ?? '' - if (primary === 'cmn' || primary === 'yue') - return 'zh' - return primary || undefined + return code === undefined ? undefined : primaryLanguage(code) || undefined } const kana = /[\p{Script=Hiragana}\p{Script=Katakana}]/u diff --git a/packages/speech-transformer-math/src/index.ts b/packages/speech-transformer-math/src/index.ts index d915220962..f5fce016ff 100644 --- a/packages/speech-transformer-math/src/index.ts +++ b/packages/speech-transformer-math/src/index.ts @@ -10,7 +10,7 @@ export { lowerMath } from './lower' export { spellNumber } from './numbers' export { parseMath } from './parse' export { renderMath } from './render' -export { builtinLocales, MathSpeech } from './speech' +export { builtinLocales, MathSpeech, primaryLanguage } from './speech' export type { MathSpeechOptions } from './speech' export { texCommands } from './tex' export type { TexNode } from './tex' diff --git a/packages/speech-transformer-math/src/speech.ts b/packages/speech-transformer-math/src/speech.ts index 11e88d6eb7..751563b682 100644 --- a/packages/speech-transformer-math/src/speech.ts +++ b/packages/speech-transformer-math/src/speech.ts @@ -26,8 +26,11 @@ export interface MathSpeechOptions { fallbackLanguage?: MathLanguage } -/** The primary language subtag of a BCP 47 code: `zh` for `zh-CN`, `ko` for `ko_KR`. Chinese dialect codes are `zh`. */ -function primaryLanguage(code: string): string { +/** + * The primary language subtag of a BCP 47 code: `zh` for `zh-CN`, `ko` for `ko_KR`. Chinese dialect codes are `zh`, + * because the readers have one Chinese pack. + */ +export function primaryLanguage(code: string): string { const primary = code.toLowerCase().split(/[-_]/)[0] ?? '' return primary === 'cmn' || primary === 'yue' ? 'zh' : primary } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 948f864f88..33e73e8625 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -405,9 +405,6 @@ catalogs: '@proj-airi/drizzle-duckdb-wasm': specifier: ^0.6.0 version: 0.6.0 - '@proj-airi/duckdb-wasm': - specifier: ^0.6.0 - version: 0.6.0 '@proj-airi/iconify-meteocons': specifier: ^0.1.5 version: 0.1.5 @@ -3547,7 +3544,7 @@ importers: '@proj-airi/core-agent': specifier: workspace:^ version: link:../../packages/core-agent - '@proj-airi/duckdb-wasm': + '@proj-airi/drizzle-duckdb-wasm': specifier: 'catalog:' version: 0.6.0(drizzle-orm@0.45.2(@electric-sql/pglite@0.5.6)(@opentelemetry/api@1.9.1)(@prisma/client@5.22.0)(@types/pg@8.23.1)(better-sqlite3@12.11.1)(kysely@0.29.5)(pg@8.23.0)(postgres@3.4.9))(web-worker@1.5.0) '@proj-airi/mathjson': @@ -3574,6 +3571,9 @@ importers: '@xsai/generate-transcription': specifier: 'catalog:' version: 0.5.1 + drizzle-orm: + specifier: 'catalog:' + version: 0.45.2(@electric-sql/pglite@0.5.6)(@opentelemetry/api@1.9.1)(@prisma/client@5.22.0)(@types/pg@8.23.1)(better-sqlite3@12.11.1)(kysely@0.29.5)(pg@8.23.0)(postgres@3.4.9) katex: specifier: 'catalog:' version: 0.19.0 @@ -3590,24 +3590,9 @@ importers: '@types/node': specifier: 'catalog:' version: 26.2.0 - rehype-stringify: - specifier: 'catalog:' - version: 10.0.1 - remark-parse: - specifier: 'catalog:' - version: 11.0.0(supports-color@10.2.2) - remark-rehype: - specifier: 'catalog:' - version: 11.1.2 - unified: - specifier: 'catalog:' - version: 11.0.5 vieval: specifier: 'catalog:' version: 0.10.3(@types/node@26.2.0)(@vitest/browser-playwright@4.1.11)(@vitest/coverage-v8@4.1.11)(chokidar@5.0.0)(dotenv@17.4.2)(electron@43.4.1(supports-color@10.2.2))(esbuild@0.28.2)(giget@3.3.1)(h3@2.0.2(crossws@0.4.12(srvx@1.0.5)))(hono@4.13.3)(jiti@2.7.0)(jsdom@30.0.1(@noble/hashes@2.3.0)(canvas@3.2.3))(less@4.9.0(supports-color@10.2.2))(magicast@0.5.4)(terser@5.50.0)(tsx@4.23.12)(typescript@6.0.3)(web-worker@1.5.0)(yaml@2.9.0) - vite: - specifier: 'catalog:' - version: 8.3.4(@types/node@26.2.0)(esbuild@0.28.2)(jiti@2.7.0)(less@4.9.0(supports-color@10.2.2))(terser@5.50.0)(tsx@4.23.12)(yaml@2.9.0) vitest: specifier: catalog:vitest version: 4.1.11(@opentelemetry/api@1.9.1)(@types/node@26.2.0)(@vitest/browser-playwright@4.1.11)(@vitest/coverage-v8@4.1.11)(jsdom@30.0.1(@noble/hashes@2.3.0)(canvas@3.2.3))(vite@8.3.4(@types/node@26.2.0)(esbuild@0.28.2)(jiti@2.7.0)(less@4.9.0(supports-color@10.2.2))(terser@5.50.0)(tsx@4.23.12)(yaml@2.9.0)) diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index d1c566baa7..cc5b7097c3 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -189,7 +189,6 @@ catalog: '@pnpm/find-workspace-dir': ^1000.1.5 '@proj-airi/chromatic': ^1.1.4 '@proj-airi/drizzle-duckdb-wasm': ^0.6.0 - '@proj-airi/duckdb-wasm': ^0.6.0 '@proj-airi/iconify-meteocons': ^0.1.5 '@proj-airi/lobe-icons': ^1.0.19 '@proj-airi/unocss-preset-chromatic': ^1.1.4