mirror of
https://github.com/moeru-ai/airi.git
synced 2026-10-30 19:34:10 +08:00
chore(evals,core-agent): clean up the formula speech eval (#3058)
This commit is contained in:
Neko
@@ -141,7 +141,7 @@ Each case gets three listener answers. The case passes with a strict majority.
|
||||
|
||||
Each answer also has a hearing status: `ok`, `partial`, or `unclear`. The listener sets it through structured output. An `unclear` vote scores 0 with the grading rule `unclear`, so a report tells an unintelligible clip apart from a wrong reading. The `tts_speech_formula.hearing` metric is the status of most votes.
|
||||
|
||||
Each case also gets three judge verdicts on the spoken text: `natural`, `mixed`, or `spelled`. The judge never sees the answer of the listener and never judges correctness. The case takes the median verdict as the `tts_speech_formula.reading` metric, and records its score as the vieval `judge` score. The judge reads the spoken text, not the ASR transcript, so ASR errors do not count against the test object.
|
||||
Each case also gets three judge verdicts on the spoken text: `natural`, `mixed`, or `spelled`. The judge never sees the answer of the listener and never judges correctness. The case takes the median verdict as the `tts_speech_formula.reading` metric, and records its score as `formula.natural`. The judge reads the spoken text, not the ASR transcript, so ASR errors do not count against the test object.
|
||||
|
||||
### Judge calibration
|
||||
|
||||
@@ -150,11 +150,13 @@ A person labels a fixed set of readings, and the share of labels that the judge
|
||||
the verdicts of the judge.
|
||||
|
||||
1. After a change of the judge prompt, run the eval on the cases of the set, then refresh the verdicts with
|
||||
`pnpm -F @proj-airi/evals-tts-speech-formula run judge:collect`. It keeps the set. `-- --pick` picks a new one.
|
||||
2. Label with `pnpm -F @proj-airi/evals-tts-speech-formula run judge:label`. It opens a local page on port 5299. Keys
|
||||
1, 2, and 3 choose natural, mixed, and spelled. The verdict of the judge shows after the label.
|
||||
`pnpm -F @proj-airi/evals-tts-speech-formula run judge:collect`. It keeps the set and prints how many labels the
|
||||
judge matches. `-- --pick` picks a new set and keeps the labeled readings.
|
||||
2. To label a reading, set its `label` in `calibration.json` to `natural`, `mixed`, or `spelled`. `note` says why.
|
||||
|
||||
On 2026-10-20 the judge matched 31 of 43 labels (72%). After the prompt took the labels into account, it matched 34 (79%).
|
||||
On 2026-10-21, after the eval review changed the prompt and the readings, it matched 32 (74%). Three labeled readings
|
||||
have new words since, so they keep an older verdict.
|
||||
|
||||
The `tts_speech_formula.leftover_notation` metric counts the symbols and LaTeX command names that a test object left in its spoken text, for example `^`, `/`, `$`, or `mapsto`. It needs no TTS. The test `eval/test-objects/rule/src/read-aloud.test.ts` lists these leftovers of the rule engine for every case, without any model call.
|
||||
|
||||
|
||||
@@ -10,8 +10,9 @@ import { fileURLToPath } from 'node:url'
|
||||
import katex from 'katex'
|
||||
|
||||
import { DuckDBDataProtocol } from '@duckdb/duckdb-wasm'
|
||||
import { connect } from '@proj-airi/duckdb-wasm'
|
||||
import { getBundles } from '@proj-airi/duckdb-wasm/bundles/default-node'
|
||||
import { connect, drizzle } from '@proj-airi/drizzle-duckdb-wasm'
|
||||
import { getBundles } from '@proj-airi/drizzle-duckdb-wasm/bundles/default-node'
|
||||
import { sql } from 'drizzle-orm'
|
||||
|
||||
import 'katex/contrib/mhchem'
|
||||
|
||||
@@ -33,7 +34,7 @@ import 'katex/contrib/mhchem'
|
||||
*
|
||||
* main
|
||||
* -> download, once per revision, then verify
|
||||
* -> connect (@proj-airi/duckdb-wasm), registerFileURL with NODE_FS
|
||||
* -> connect (@proj-airi/drizzle-duckdb-wasm), registerFileURL with NODE_FS
|
||||
* -> uniform query and topic query, see {@link gradableRows}
|
||||
* -> sample.json
|
||||
*/
|
||||
@@ -113,8 +114,8 @@ async function download(): Promise<void> {
|
||||
* About 88% of the first 16 million rows hold notation, but under 1% of the rest do: they are single letters, for
|
||||
* example `$ A $`. Gradable rows therefore come from the first 16 million rows, and that is the data, not the hash.
|
||||
*/
|
||||
function gradableRows(): string {
|
||||
return `
|
||||
function gradableRows() {
|
||||
return sql`
|
||||
WITH source AS (
|
||||
SELECT
|
||||
file_row_number AS row_offset,
|
||||
@@ -122,7 +123,8 @@ function gradableRows(): string {
|
||||
equation, context_before, context_after, "spoken_English" AS spoken_english
|
||||
FROM read_parquet('mathbridge.parquet', file_row_number = true)
|
||||
)
|
||||
SELECT *, hash(row_offset, ${seed}) AS priority
|
||||
-- A bound number is a DOUBLE. The cast keeps the INTEGER of a literal seed, so the hash and the order stay the same.
|
||||
SELECT *, hash(row_offset, CAST(${seed} AS INTEGER)) AS priority
|
||||
FROM source
|
||||
WHERE length(tex) BETWEEN 6 AND ${maxLength}
|
||||
AND regexp_matches(tex, '[\\\\^_=<>]')
|
||||
@@ -144,20 +146,17 @@ function usesStandardTex(tex: string): boolean {
|
||||
}
|
||||
}
|
||||
|
||||
/** Escapes a value for a single-quoted SQL string. The patterns are constants, so this only doubles quotes. */
|
||||
function sqlString(value: string): string {
|
||||
return `'${value.replaceAll('\'', '\'\'')}'`
|
||||
}
|
||||
|
||||
async function main() {
|
||||
await download()
|
||||
|
||||
const client = await connect({ bundles: getBundles() })
|
||||
// Drizzle binds every value of a `sql` template as a parameter, so the patterns and numbers need no escaping.
|
||||
const db = drizzle({ client: Promise.resolve(client) })
|
||||
try {
|
||||
// NODE_FS reads the file from disk in pieces, so DuckDB never holds the whole gigabyte in WebAssembly memory.
|
||||
await client.db.registerFileURL('mathbridge.parquet', cacheFile, DuckDBDataProtocol.NODE_FS, false)
|
||||
|
||||
const topicCase = topics.map(([name, pattern]) => `WHEN regexp_matches(tex, ${sqlString(pattern)}) THEN ${sqlString(name)}`).join(' ')
|
||||
const topicCase = sql.join(topics.map(([name, pattern]) => sql`WHEN regexp_matches(tex, ${pattern}) THEN ${name}`), sql` `)
|
||||
const toRow = (row: Record<string, unknown>, stratum: string): MathBridgeRow => ({
|
||||
stratum: String(row.tex).length > shortLength ? `${stratum}-long` : stratum,
|
||||
offset: Number(row.row_offset),
|
||||
@@ -167,8 +166,8 @@ async function main() {
|
||||
spokenEnglish: String(row.spoken_english ?? ''),
|
||||
})
|
||||
|
||||
const uniform = await client.query(`${gradableRows()} ORDER BY priority LIMIT ${uniformCases * candidateFactor}`)
|
||||
const byTopic = await client.query(`
|
||||
const uniform = await db.execute<Record<string, unknown>>(sql`${gradableRows()} ORDER BY priority LIMIT ${uniformCases * candidateFactor}`)
|
||||
const byTopic = await db.execute<Record<string, unknown>>(sql`
|
||||
SELECT * FROM (SELECT *, CASE ${topicCase} END AS topic FROM (${gradableRows()}))
|
||||
WHERE topic IS NOT NULL
|
||||
QUALIFY row_number() OVER (PARTITION BY topic ORDER BY priority) <= ${topicCases * candidateFactor}
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
"spoken": "I",
|
||||
"judge": {
|
||||
"reading": "natural",
|
||||
"reason": "No part is spelled; “I” naturally reads the letter in the formula."
|
||||
"reason": "No part is spelled; the spoken text reads the letter I naturally."
|
||||
},
|
||||
"label": "natural",
|
||||
"note": ""
|
||||
@@ -38,7 +38,7 @@
|
||||
"spoken": "double struck E of N sub alpha to the T power of v",
|
||||
"judge": {
|
||||
"reading": "natural",
|
||||
"reason": "No part is spelled; the reading uses standard words for the expectation, subscript, and power."
|
||||
"reason": "No part is spelled; naming the double-struck E and reading the subscript, superscript, and function argument are natural."
|
||||
},
|
||||
"label": "natural",
|
||||
"note": "Check whether \"N sub alpha to the T power of v\" keeps the grouping of N_alpha^T(v)."
|
||||
@@ -51,8 +51,8 @@
|
||||
"reference": "k=1 , \\dots , d",
|
||||
"spoken": "k equals one, dot dot dot, d",
|
||||
"judge": {
|
||||
"reading": "natural",
|
||||
"reason": "No part is spelled; “and so on, up to” is a natural way to read the ellipsis."
|
||||
"reading": "mixed",
|
||||
"reason": "“Dot dot dot” spells out the ellipsis."
|
||||
},
|
||||
"label": "mixed",
|
||||
"note": "Read the ellipsis as \"and so on\" or \"up to\", not \"dot dot dot\"."
|
||||
@@ -66,7 +66,7 @@
|
||||
"spoken": "the fraction with numerator 6 and denominator pi squared",
|
||||
"judge": {
|
||||
"reading": "natural",
|
||||
"reason": "No part is spelled; the numerator-and-denominator phrasing naturally preserves the grouping of pi squared."
|
||||
"reason": "No part is spelled; the numerator and denominator are read naturally."
|
||||
},
|
||||
"label": "mixed",
|
||||
"note": "A simple fraction reads shorter: \"six over pi squared\"."
|
||||
@@ -94,7 +94,7 @@
|
||||
"spoken": "open paren a plus b close paren squared equals a squared plus 2 a b plus b squared",
|
||||
"judge": {
|
||||
"reading": "spelled",
|
||||
"reason": "The reading names the parentheses around the compound expression instead of reading its meaning naturally."
|
||||
"reason": "“Open paren” and “close paren” spell out the grouping brackets instead of reading the quantity naturally."
|
||||
},
|
||||
"label": "spelled",
|
||||
"note": "Do not read the parentheses. Say \"the square of the sum of a and b\"."
|
||||
@@ -108,7 +108,7 @@
|
||||
"spoken": "Delta sub k is less than zero.",
|
||||
"judge": {
|
||||
"reading": "natural",
|
||||
"reason": "No part is spelled; “Delta sub k is less than zero” is a natural reading."
|
||||
"reason": "No part is spelled; “Delta sub k” is a natural reading of the subscript."
|
||||
},
|
||||
"label": "natural",
|
||||
"note": "Good reading."
|
||||
@@ -136,7 +136,7 @@
|
||||
"spoken": "G superscript, open parenthesis, p, close parenthesis.",
|
||||
"judge": {
|
||||
"reading": "spelled",
|
||||
"reason": "“Open parenthesis” and “close parenthesis” spell out the grouping instead of reading the superscript as G to the power of p."
|
||||
"reason": "The reading spells out the parentheses around the superscript."
|
||||
},
|
||||
"label": "mixed",
|
||||
"note": ""
|
||||
@@ -150,7 +150,7 @@
|
||||
"spoken": "7 over 60 bold k times bold k prime",
|
||||
"judge": {
|
||||
"reading": "mixed",
|
||||
"reason": "“Bold” names formatting markup for both k’s."
|
||||
"reason": "The reading spells the boldface markup by saying “bold” for each k."
|
||||
},
|
||||
"label": "mixed",
|
||||
"note": "Only one k appears, so \"bold\" is not needed: \"k times k prime\"."
|
||||
@@ -163,8 +163,8 @@
|
||||
"reference": "\\ce{2H2 + O2 -> 2H2O}",
|
||||
"spoken": "係数二、エイチの下付き数字二、プラス、オーの下付き数字二、右向き矢印、係数二、エイチの下付き数字二、オー",
|
||||
"judge": {
|
||||
"reading": "mixed",
|
||||
"reason": "No part is spelled; the element symbols are read by letter, and the subscripts and reaction arrow are described in Japanese."
|
||||
"reading": "spelled",
|
||||
"reason": "「下付き数字」と「右向き矢印」は数式の表記を説明しており、意味として読む形ではありません。"
|
||||
},
|
||||
"label": "spelled",
|
||||
"note": "Reads every subscript and the arrow by name. It does not work as a reading of an equation."
|
||||
@@ -178,7 +178,7 @@
|
||||
"spoken": "分子がディー、分母がディーエックスの分数に、エックスのエヌ乗。イコール、エヌかけるエックスの、肩にエヌひく一。",
|
||||
"judge": {
|
||||
"reading": "mixed",
|
||||
"reason": "The phrase 「肩に」 spells out the exponent’s visual placement instead of reading it as a power."
|
||||
"reason": "「肩に」と位置を説明しており、指数を読むのではなく組版上の位置を述べています。"
|
||||
},
|
||||
"label": "spelled",
|
||||
"note": "Mostly right. Unsure whether 肩に is a usual Japanese reading."
|
||||
@@ -206,7 +206,7 @@
|
||||
"spoken": "2 empty normal H of sub 2 normal O sub 2 right arrow 2 empty normal H of sub 2 normal O plus normal O sub 2",
|
||||
"judge": {
|
||||
"reading": "spelled",
|
||||
"reason": "The reading spells the chemical symbols with markup words such as “empty” and “normal,” and says “right arrow” in English."
|
||||
"reason": "It spells the chemistry element symbols with markup words such as “empty,” “normal,” and “of sub.”"
|
||||
},
|
||||
"label": "spelled",
|
||||
"note": ""
|
||||
@@ -220,7 +220,7 @@
|
||||
"spoken": "ケーがゼロから無限大までのにのケー乗分のいちの和 イコール に",
|
||||
"judge": {
|
||||
"reading": "natural",
|
||||
"reason": "No part is spelled; the sum, fraction, power, and equality are read in standard Japanese mathematical terms."
|
||||
"reason": "No part is spelled; the sum, fraction, power, and equality are read in standard Japanese math terms."
|
||||
},
|
||||
"label": "natural",
|
||||
"note": ""
|
||||
@@ -234,7 +234,7 @@
|
||||
"spoken": "小文字のエーは、大文字のエーに属する。",
|
||||
"judge": {
|
||||
"reading": "natural",
|
||||
"reason": "No part is spelled; the reading naturally distinguishes lowercase a from uppercase A."
|
||||
"reason": "No part is spelled; the case of each letter is naturally distinguished."
|
||||
},
|
||||
"label": "natural",
|
||||
"note": "Shorter case words are enough, for example 小 a and 大 A."
|
||||
@@ -262,7 +262,7 @@
|
||||
"spoken": "左中かっこ、エックス、縦棒、エックス大なりゼロ、右中かっこ。",
|
||||
"judge": {
|
||||
"reading": "spelled",
|
||||
"reason": "It spells out the set braces as “left brace” and “right brace,” and reads the set-builder bar as “vertical bar.”"
|
||||
"reason": "It spells out the set braces and the vertical bar instead of reading the set as the set of x such that x is greater than zero."
|
||||
},
|
||||
"label": "spelled",
|
||||
"note": ""
|
||||
@@ -276,7 +276,7 @@
|
||||
"spoken": "派的平方分之6",
|
||||
"judge": {
|
||||
"reading": "natural",
|
||||
"reason": "No part is spelled; this is a natural reading of the fraction."
|
||||
"reason": "No part is spelled; the denominator’s square is clear in this Mandarin fraction reading."
|
||||
},
|
||||
"label": "natural",
|
||||
"note": ""
|
||||
@@ -290,7 +290,7 @@
|
||||
"spoken": "九点八米每秒平方",
|
||||
"judge": {
|
||||
"reading": "natural",
|
||||
"reason": "No part is spelled; the quantity is read naturally in Mandarin Chinese."
|
||||
"reason": "No part is spelled; the number and unit are read naturally in Mandarin Chinese."
|
||||
},
|
||||
"label": "natural",
|
||||
"note": ""
|
||||
@@ -304,7 +304,7 @@
|
||||
"spoken": "小写字母 l，下标一，平行于小写字母 l，下标二。",
|
||||
"judge": {
|
||||
"reading": "mixed",
|
||||
"reason": "The reading spells out “小写字母” even though the formula contains only one case of l."
|
||||
"reason": "“小写字母” spells out the case of each letter even though the formula uses only one case; the subscript and parallel relation are read naturally."
|
||||
},
|
||||
"label": "mixed",
|
||||
"note": "The letter has one case in the formula, so the case is not needed. If it is, say 小 l."
|
||||
@@ -317,8 +317,8 @@
|
||||
"reference": "\\frac{6}{\\pi^2}",
|
||||
"spoken": "the fraction with numerator 6 and denominator pi squared",
|
||||
"judge": {
|
||||
"reading": "natural",
|
||||
"reason": "The reading is in English rather than Mandarin Chinese."
|
||||
"reading": "spelled",
|
||||
"reason": "The reading is mostly in English rather than Mandarin Chinese."
|
||||
},
|
||||
"label": "mixed",
|
||||
"note": "Not Chinese. The fraction also reads shorter."
|
||||
@@ -331,8 +331,8 @@
|
||||
"reference": "a \\in A",
|
||||
"spoken": "小写a 属于 大写A",
|
||||
"judge": {
|
||||
"reading": "natural",
|
||||
"reason": "No part is spelled."
|
||||
"reading": "spelled",
|
||||
"reason": "The spoken text unnecessarily specifies lowercase and uppercase for letters that appear in only one case in the formula."
|
||||
},
|
||||
"label": "mixed",
|
||||
"note": "Use the short form 小a and 大A. The case words can go unless a variant of the letter appears."
|
||||
@@ -345,8 +345,8 @@
|
||||
"reference": "\\{x \\mid x > 0\\}",
|
||||
"spoken": "左花括号，x，使得 x 大于零，右花括号。",
|
||||
"judge": {
|
||||
"reading": "spelled",
|
||||
"reason": "The reading names the left and right braces around the set."
|
||||
"reading": "mixed",
|
||||
"reason": "The reading names the left and right curly braces around the set, which spells the markup."
|
||||
},
|
||||
"label": "spelled",
|
||||
"note": "The braces of a set are clear from the meaning and need no reading."
|
||||
@@ -360,7 +360,7 @@
|
||||
"spoken": "open paren a plus b close paren squared equals a squared plus 2 a b plus b squared",
|
||||
"judge": {
|
||||
"reading": "spelled",
|
||||
"reason": "The reading is mostly in English rather than Mandarin Chinese, and it names the main parentheses as “open paren” and “close paren.”"
|
||||
"reason": "The reading is mostly in English rather than Mandarin Chinese, and it spells out the parentheses around the sum."
|
||||
},
|
||||
"label": "spelled",
|
||||
"note": "Not Chinese, and the parentheses are read. Better: a 与 b 的和的平方 等于 a 方加 2ab 加 b 方."
|
||||
@@ -402,7 +402,7 @@
|
||||
"spoken": "x is less than or equal to 3",
|
||||
"judge": {
|
||||
"reading": "spelled",
|
||||
"reason": "The comparison is read in English rather than Mandarin Chinese."
|
||||
"reason": "The reading is mostly in English rather than Mandarin Chinese."
|
||||
},
|
||||
"label": "spelled",
|
||||
"note": "Reads in English, not Chinese. A reading in another language does not pass."
|
||||
@@ -416,7 +416,7 @@
|
||||
"spoken": "Open brace, x such that x is greater than zero, close brace.",
|
||||
"judge": {
|
||||
"reading": "spelled",
|
||||
"reason": "It spells out the opening and closing braces around the set."
|
||||
"reason": "It names the opening and closing braces even though “the set of x such that” is a natural reading."
|
||||
},
|
||||
"label": "mixed",
|
||||
"note": "The braces of a set need no reading."
|
||||
@@ -444,7 +444,7 @@
|
||||
"spoken": "Capital sigma, subscript, open brace, capital B, subscript capital T, close brace.",
|
||||
"judge": {
|
||||
"reading": "spelled",
|
||||
"reason": "The reading spells the braces and subscript markup, and adds unnecessary case labels for sigma, B, and T."
|
||||
"reason": "The reading spells the subscript markup by saying “subscript,” “open brace,” and “close brace,” and adds unnecessary case labels for the only instances of sigma and B."
|
||||
},
|
||||
"label": "spelled",
|
||||
"note": ""
|
||||
@@ -472,7 +472,7 @@
|
||||
"spoken": "9.8 empty meters divided by seconds squared",
|
||||
"judge": {
|
||||
"reading": "spelled",
|
||||
"reason": "“Empty” and “divided by” spell spacing and fraction notation rather than reading the unit naturally in Japanese."
|
||||
"reason": "The word “empty” spells out the spacing command, and “divided by” reads the unit fraction symbolically rather than as the standard Japanese unit reading."
|
||||
},
|
||||
"label": "mixed",
|
||||
"note": "\"empty\" is not a reading."
|
||||
@@ -485,8 +485,8 @@
|
||||
"reference": "N=n+1",
|
||||
"spoken": "大文字のエヌ、イコール、小文字のエヌ、プラス、一。",
|
||||
"judge": {
|
||||
"reading": "natural",
|
||||
"reason": "The case words for N and n are natural because both cases appear in the formula; no part is spelled."
|
||||
"reading": "spelled",
|
||||
"reason": "It unnecessarily says 大文字の and 小文字の for letters that appear only in one case each."
|
||||
},
|
||||
"label": "mixed",
|
||||
"note": "Shorter case words are enough, unless the formula has several variants of n."
|
||||
@@ -500,7 +500,7 @@
|
||||
"spoken": "Celsius plus normal O sub 2 right arrow Celsius normal O sub 2",
|
||||
"judge": {
|
||||
"reading": "spelled",
|
||||
"reason": "The reading is mostly English, and “sub” and “right arrow” spell the notation rather than reading it in Japanese."
|
||||
"reason": "The reading is mostly English, and “sub 2” and “right arrow” spell out the notation rather than reading it naturally in Japanese."
|
||||
},
|
||||
"label": "spelled",
|
||||
"note": "Reads C as Celsius, and the oxygen molecule is lost."
|
||||
@@ -514,7 +514,7 @@
|
||||
"spoken": "2 empty Celsius plus normal O sub 2 right arrow 2 empty Celsius normal O",
|
||||
"judge": {
|
||||
"reading": "spelled",
|
||||
"reason": "The reading uses the English phrases “empty Celsius,” “plus,” “normal,” and “right arrow” instead of Japanese readings."
|
||||
"reason": "“Empty Celsius,” “normal,” and “right arrow” spell the notation, and the reading is mostly English."
|
||||
},
|
||||
"label": "spelled",
|
||||
"note": "Reads the formula symbols, not the chemical formula."
|
||||
@@ -542,7 +542,7 @@
|
||||
"spoken": "CuSO4 点 5 H2O",
|
||||
"judge": {
|
||||
"reading": "spelled",
|
||||
"reason": "It leaves the chemical formulas “CuSO4” and “H2O” as source notation instead of reading the element symbols letter by letter."
|
||||
"reason": "The reading spells the chemical formula as Latin element symbols and digits, and says “点” for the asterisk instead of reading the hydrate notation naturally."
|
||||
},
|
||||
"label": "spelled",
|
||||
"note": ""
|
||||
@@ -556,7 +556,7 @@
|
||||
"spoken": "25 raised to the composed with power Celsius",
|
||||
"judge": {
|
||||
"reading": "spelled",
|
||||
"reason": "The reading uses the English phrase “raised to the ... power” instead of Mandarin Chinese."
|
||||
"reason": "“Raised to the composed with power” is English markup-like wording rather than a Mandarin reading."
|
||||
},
|
||||
"label": "spelled",
|
||||
"note": "Should be 25 摄氏度."
|
||||
@@ -570,7 +570,7 @@
|
||||
"spoken": "系数二，过氧化氢分子，化学式中氢的下标是二，氧的下标是二，反应生成系数二的水分子，化学式中氢的下标是二，再加一个氧气分子，化学式中氧的下标是二。",
|
||||
"judge": {
|
||||
"reading": "natural",
|
||||
"reason": "No part is spelled; the coefficients, compound names, and subscript readings use natural Mandarin terminology."
|
||||
"reason": "No part is spelled; the reading uses compound names and describes the coefficients and subscripts in Mandarin."
|
||||
},
|
||||
"label": "mixed",
|
||||
"note": "Too mechanical."
|
||||
@@ -584,7 +584,7 @@
|
||||
"spoken": "lim over x right arrow infinity of 1 over x equals 0",
|
||||
"judge": {
|
||||
"reading": "spelled",
|
||||
"reason": "The reading is mostly English, and “right arrow” spells the arrow symbol."
|
||||
"reason": "The reading is mostly English rather than Mandarin Chinese."
|
||||
},
|
||||
"label": "mixed",
|
||||
"note": "Unsure whether lim reads this way."
|
||||
@@ -598,7 +598,7 @@
|
||||
"spoken": "f colon X right arrow Y comma x right arrow from bar y",
|
||||
"judge": {
|
||||
"reading": "spelled",
|
||||
"reason": "The reading spells the symbols as “colon,” “right arrow,” “comma,” and “right arrow from bar” instead of reading their meaning in Mandarin."
|
||||
"reason": "The reading uses the English markup names “colon,” “right arrow,” “comma,” and “right arrow from bar” instead of Mandarin mathematical phrasing."
|
||||
},
|
||||
"label": "spelled",
|
||||
"note": ""
|
||||
@@ -612,7 +612,7 @@
|
||||
"spoken": "시그마 아래첨자 영은 분수 시작, 분자 에프 빼기 일, 분모 이, 분수 끝과 같다.",
|
||||
"judge": {
|
||||
"reading": "mixed",
|
||||
"reason": "The reading spells out fraction markup with “분수 시작” and “분수 끝.”"
|
||||
"reason": "The reading spells the fraction markup with “분수 시작” and “분수 끝.”"
|
||||
},
|
||||
"label": null,
|
||||
"note": ""
|
||||
@@ -625,8 +625,8 @@
|
||||
"reference": "\\lim_{x \\to \\infty}\\frac{1}{x}=0",
|
||||
"spoken": "리미트 엑스가 무한대로 갈 때, 분수의 분자는 일, 분모는 엑스, 그 값은 영입니다.",
|
||||
"judge": {
|
||||
"reading": "natural",
|
||||
"reason": "No part is spelled; the limit and fraction are read in natural Korean mathematical language."
|
||||
"reading": "mixed",
|
||||
"reason": "The simple fraction is read in the unnecessarily explicit numerator-and-denominator form."
|
||||
},
|
||||
"label": null,
|
||||
"note": ""
|
||||
@@ -654,7 +654,7 @@
|
||||
"spoken": "왼쪽 중괄호, 엑스, 세로줄, 엑스가 영보다 크다, 오른쪽 중괄호.",
|
||||
"judge": {
|
||||
"reading": "spelled",
|
||||
"reason": "The reading spells out the set’s opening and closing braces and names the vertical bar instead of reading the set-builder structure naturally."
|
||||
"reason": "The reading spells out the set braces as “왼쪽 중괄호” and “오른쪽 중괄호,” and the separator as “세로줄.”"
|
||||
},
|
||||
"label": null,
|
||||
"note": ""
|
||||
@@ -668,7 +668,7 @@
|
||||
"spoken": "에이는 비와의 합집합이며, 중괄호를 열고 일, 이, 삼을 차례로 적은 뒤 중괄호를 닫은 집합과 같다.",
|
||||
"judge": {
|
||||
"reading": "mixed",
|
||||
"reason": "It spells the set braces by saying to open and close the curly braces."
|
||||
"reason": "The reading spells out the opening and closing curly braces around the set."
|
||||
},
|
||||
"label": null,
|
||||
"note": ""
|
||||
@@ -682,7 +682,7 @@
|
||||
"spoken": "소문자 에이는 대문자 에이의 원소",
|
||||
"judge": {
|
||||
"reading": "natural",
|
||||
"reason": "No part is spelled; the reading naturally distinguishes lowercase a from uppercase A."
|
||||
"reason": "No part is spelled; the case words distinguish lowercase a from uppercase A."
|
||||
},
|
||||
"label": null,
|
||||
"note": ""
|
||||
@@ -695,8 +695,8 @@
|
||||
"reference": "\\lim_{n \\to \\infty}\\left(1+\\frac{1}{n}\\right)^{n}=e",
|
||||
"spoken": "엔이 무한대로 갈 때 괄호 일 더하기 분자가 일, 분모가 엔인 분수 괄호 닫고의 엔 제곱의 극한은 e",
|
||||
"judge": {
|
||||
"reading": "mixed",
|
||||
"reason": "The reading is natural; it uses the standard fraction wording and names the meaningful parentheses around the compound expression."
|
||||
"reading": "natural",
|
||||
"reason": "No part is spelled; the fraction is naturally read by naming its numerator and denominator, and the parentheses group the compound expression."
|
||||
},
|
||||
"label": null,
|
||||
"note": ""
|
||||
@@ -738,7 +738,7 @@
|
||||
"spoken": "엑스는 영 초과 비 이하인 구간의 원소",
|
||||
"judge": {
|
||||
"reading": "natural",
|
||||
"reason": "No part is spelled; the interval and membership are read naturally in Korean."
|
||||
"reason": "No part is spelled; the interval is read naturally as an element between zero and b, inclusive."
|
||||
},
|
||||
"label": null,
|
||||
"note": ""
|
||||
@@ -751,8 +751,8 @@
|
||||
"reference": "\\sum_{k=1}^{n} k=\\frac{n(n+1)}{2}",
|
||||
"spoken": "ケーがいちからエヌまでのケーの和 イコール 分子がエヌかっこエヌプラスいちかっことじ、分母がにの分数",
|
||||
"judge": {
|
||||
"reading": "mixed",
|
||||
"reason": "The spoken reading spells the parentheses in the denominator expression by naming them."
|
||||
"reading": "natural",
|
||||
"reason": "No part is spelled; the sum and fraction are read using natural Japanese math phrasing."
|
||||
},
|
||||
"label": null,
|
||||
"note": ""
|
||||
@@ -766,7 +766,7 @@
|
||||
"spoken": "f 콜론 X 오른쪽 화살표 Y 콤마 x 세로선이 붙은 오른쪽 화살표 y",
|
||||
"judge": {
|
||||
"reading": "mixed",
|
||||
"reason": "The reading spells the mapsto symbol as “세로선이 붙은 오른쪽 화살표” instead of reading its mapping meaning."
|
||||
"reason": "The phrase “세로선이 붙은 오른쪽 화살표” describes the mapsto symbol rather than reading its meaning."
|
||||
},
|
||||
"label": null,
|
||||
"note": ""
|
||||
@@ -780,7 +780,7 @@
|
||||
"spoken": "2 기본 H 의 아래첨자 2 더하기 기본 O 아래첨자 2 오른쪽 화살표 2 기본 H 의 아래첨자 2 기본 O",
|
||||
"judge": {
|
||||
"reading": "mixed",
|
||||
"reason": "“기본” spells font formatting, and “오른쪽 화살표” names the arrow symbol rather than reading its meaning."
|
||||
"reason": "The reading spells the subscripts with “아래첨자” and the arrow with “오른쪽 화살표.”"
|
||||
},
|
||||
"label": null,
|
||||
"note": ""
|
||||
@@ -794,7 +794,7 @@
|
||||
"spoken": "x 오른쪽 화살표 무한대 에 대한 리미트 의 x 분의 1 은 0",
|
||||
"judge": {
|
||||
"reading": "mixed",
|
||||
"reason": "The reading names the arrow as a right arrow instead of reading the limit condition naturally."
|
||||
"reason": "The reading spells the limit notation as “x right arrow infinity” instead of reading the limiting condition naturally."
|
||||
},
|
||||
"label": null,
|
||||
"note": ""
|
||||
@@ -807,8 +807,8 @@
|
||||
"reference": "(a+b)^2=a^2+2ab+b^2",
|
||||
"spoken": "괄호 열고 a 더하기 b 괄호 닫고 제곱 은 a 제곱 더하기 2 a b 더하기 b 제곱",
|
||||
"judge": {
|
||||
"reading": "spelled",
|
||||
"reason": "The reading names the main parentheses around a plus b instead of using a natural phrase for the squared sum."
|
||||
"reading": "natural",
|
||||
"reason": "No part is spelled; the Korean grouping words and power readings are natural."
|
||||
},
|
||||
"label": null,
|
||||
"note": ""
|
||||
@@ -822,7 +822,7 @@
|
||||
"spoken": "시그마 i 는 1 에서 n 까지 i 제곱",
|
||||
"judge": {
|
||||
"reading": "natural",
|
||||
"reason": "No part is spelled; the summation notation is read in a conventional Korean shorthand."
|
||||
"reason": "No part is spelled; the summation is read naturally as a sum from i equals one to n of i squared."
|
||||
},
|
||||
"label": null,
|
||||
"note": ""
|
||||
@@ -836,7 +836,7 @@
|
||||
"spoken": "25 의 합성 함수 제곱 섭씨",
|
||||
"judge": {
|
||||
"reading": "mixed",
|
||||
"reason": "The reading spells the superscript as “제곱,” which changes the degree symbol into an exponent."
|
||||
"reason": "The reading spells the degree mark as a superscript and calls it a square rather than reading the temperature unit naturally."
|
||||
},
|
||||
"label": null,
|
||||
"note": ""
|
||||
|
||||
@@ -8,7 +8,8 @@ import { formulaSpeechCases } from '../../../cases/formula-speech'
|
||||
import { mathBridgeCases } from '../../../cases/math-bridge'
|
||||
|
||||
/**
|
||||
* Collects the readings that a person labels in the calibration tool, see ./README.md.
|
||||
* Collects the readings of the judge calibration set, and prints how many labels of a person the judge matches. See the
|
||||
* judge calibration section of the eval README.
|
||||
*
|
||||
* It reads the newest `result.json` of each case from the reports.
|
||||
*
|
||||
@@ -50,8 +51,16 @@ function order(key: string): number {
|
||||
return hash >>> 0
|
||||
}
|
||||
|
||||
/** The newest judged reading of each object, case, and language, without TTS. */
|
||||
async function newestReadings(): Promise<CalibrationItem[]> {
|
||||
/** The key of one spoken text of an item. A label applies to the words that a person read, not to the case. */
|
||||
function readingKey(item: Pick<CalibrationItem, 'id' | 'spoken'>): string {
|
||||
return `${item.id}\u0000${item.spoken}`
|
||||
}
|
||||
|
||||
/**
|
||||
* The newest judged reading of each object, case, language, and spoken text, without TTS. A rule change gives a case
|
||||
* new words, and the result for the old words stays in older reports.
|
||||
*/
|
||||
async function newestReadings(): Promise<{ time: number, item: CalibrationItem }[]> {
|
||||
const newest = new Map<string, { time: number, item: CalibrationItem }>()
|
||||
for (const object of objects) {
|
||||
for await (const path of glob(`${object}/*/tts-speech-formula/attempt-*/run-*/artifacts/*/*/result.json`, { cwd: reportsRoot.pathname })) {
|
||||
@@ -60,10 +69,11 @@ async function newestReadings(): Promise<CalibrationItem[]> {
|
||||
if (!currentCases.has(result.id) || result.tts !== 'none' || result.reading === undefined || typeof result.reference !== 'string' || !result.spoken.trim())
|
||||
continue
|
||||
const id = `${object}|${result.id}|${result.language}`
|
||||
const key = readingKey({ id, spoken: result.spoken })
|
||||
const time = (await stat(file)).mtimeMs
|
||||
if ((newest.get(id)?.time ?? 0) >= time)
|
||||
if ((newest.get(key)?.time ?? 0) >= time)
|
||||
continue
|
||||
newest.set(id, {
|
||||
newest.set(key, {
|
||||
time,
|
||||
item: {
|
||||
id,
|
||||
@@ -79,7 +89,7 @@ async function newestReadings(): Promise<CalibrationItem[]> {
|
||||
})
|
||||
}
|
||||
}
|
||||
return [...newest.values()].map(({ item }) => item)
|
||||
return [...newest.values()]
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -108,7 +118,20 @@ function pick(readings: readonly CalibrationItem[]): CalibrationItem[] {
|
||||
return picked
|
||||
}
|
||||
|
||||
/** Written next to the file and renamed over it, as the labeling server does, so a reader never sees half a file. */
|
||||
/**
|
||||
* Prints how many labeled readings the judge verdict matches, the measure of the judge. `stale` readings have no
|
||||
* newer verdict on their words, so they keep an older verdict.
|
||||
*/
|
||||
function reportAgreement(items: readonly CalibrationItem[], stale = 0) {
|
||||
const labeled = items.filter(item => item.label !== null)
|
||||
const matched = labeled.filter(item => item.label === item.judge.reading).length
|
||||
const share = labeled.length ? Math.round(100 * matched / labeled.length) : 0
|
||||
process.stdout.write(`the judge matches ${matched} of ${labeled.length} labels (${share}%)\n`)
|
||||
if (stale > 0)
|
||||
process.stdout.write(`${stale} readings kept an older verdict, because no report judged their words again\n`)
|
||||
}
|
||||
|
||||
/** Written next to the file and renamed over it, so a reader never sees half a file. */
|
||||
async function write(items: CalibrationItem[]) {
|
||||
const partial = new URL('./calibration.json.partial', import.meta.url)
|
||||
await writeFile(partial, `${JSON.stringify({ seed, items } satisfies CalibrationFile, null, 2)}\n`)
|
||||
@@ -119,17 +142,25 @@ async function main() {
|
||||
const previous = await readFile(calibrationFile, 'utf8').then(text => JSON.parse(text) as CalibrationFile, () => ({ items: [] }))
|
||||
const labeled = new Map(previous.items.filter(item => item.label !== null).map(item => [item.id, item]))
|
||||
|
||||
const readings = await newestReadings()
|
||||
const judged = await newestReadings()
|
||||
const readings = judged.map(({ item }) => item)
|
||||
if (previous.items.length > 0 && !process.argv.includes('--pick')) {
|
||||
const newest = new Map(readings.map(item => [item.id, item]))
|
||||
const items = previous.items.map(item => ({ ...item, judge: newest.get(item.id)?.judge ?? item.judge }))
|
||||
// A verdict counts only on the words that the person labeled. A case that reads differently now keeps its verdict.
|
||||
const newest = new Map(readings.map(item => [readingKey(item), item]))
|
||||
const items = previous.items.map(item => ({ ...item, judge: newest.get(readingKey(item))?.judge ?? item.judge }))
|
||||
const stale = previous.items.filter(item => !newest.has(readingKey(item))).length
|
||||
await write(items)
|
||||
process.stdout.write(`refreshed the judge verdicts of ${items.length} readings, ${labeled.size} labeled\n`)
|
||||
process.stdout.write(`refreshed the judge verdicts of ${items.length - stale} readings, ${labeled.size} labeled\n`)
|
||||
reportAgreement(items, stale)
|
||||
return
|
||||
}
|
||||
|
||||
// A new pick takes the newest words of each case.
|
||||
const current = new Map<string, CalibrationItem>()
|
||||
for (const { item } of judged.toSorted((left, right) => left.time - right.time))
|
||||
current.set(item.id, item)
|
||||
const languages = [...new Set(readings.map(item => item.language))].sort()
|
||||
const items = languages.flatMap(language => pick(readings.filter(item => item.language === language)))
|
||||
const items = languages.flatMap(language => pick([...current.values()].filter(item => item.language === language)))
|
||||
// A labeled item keeps its label and note. Its judge verdict is the newest one.
|
||||
.map(item => ({ ...item, label: labeled.get(item.id)?.label ?? null, note: labeled.get(item.id)?.note ?? '' }))
|
||||
// Labeled items that the new pick left out stay, so collecting again never loses work.
|
||||
@@ -141,6 +172,7 @@ async function main() {
|
||||
|
||||
await write(items)
|
||||
process.stdout.write(`collected ${items.length} readings from ${readings.length}, ${labeled.size} labeled\n`)
|
||||
reportAgreement(items)
|
||||
}
|
||||
|
||||
void main()
|
||||
|
||||
@@ -1,74 +0,0 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Judge calibration</title>
|
||||
<link rel="stylesheet" href="./style.css" />
|
||||
</head>
|
||||
<body>
|
||||
<main>
|
||||
<header class="top">
|
||||
<div class="title">
|
||||
<h1>Judge calibration</h1>
|
||||
<p class="stats" id="stats">Loading…</p>
|
||||
<div class="progress" aria-hidden="true"><div class="progress-bar" id="progress"></div></div>
|
||||
</div>
|
||||
<div class="filters">
|
||||
<label class="field">
|
||||
<span>Language</span>
|
||||
<select id="language"><option value="">All languages</option></select>
|
||||
</label>
|
||||
<label class="field">
|
||||
<span>Show</span>
|
||||
<select id="show">
|
||||
<option value="unlabeled">Unlabeled first</option>
|
||||
<option value="all">All</option>
|
||||
<option value="disagree">Disagreements</option>
|
||||
</select>
|
||||
</label>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<section class="stage" id="stage" aria-live="polite"></section>
|
||||
|
||||
<nav class="pager">
|
||||
<button class="button" id="previous" type="button"><kbd>←</kbd> Previous</button>
|
||||
<span class="position" id="position"></span>
|
||||
<button class="button" id="next" type="button">Next <kbd>Enter</kbd></button>
|
||||
</nav>
|
||||
|
||||
<p class="hint">
|
||||
<kbd>1</kbd> natural · <kbd>2</kbd> mixed · <kbd>3</kbd> spelled · press the same key again to clear ·
|
||||
the judge verdict shows after you label.
|
||||
</p>
|
||||
</main>
|
||||
|
||||
<div class="toast" id="toast" role="status"></div>
|
||||
|
||||
<template id="card-template">
|
||||
<article class="card">
|
||||
<div class="meta"></div>
|
||||
<div class="formula"></div>
|
||||
<code class="source"></code>
|
||||
<p class="spoken"></p>
|
||||
<div class="choices" role="radiogroup" aria-label="Reading style">
|
||||
<button class="choice" type="button" role="radio" data-label="natural"><kbd>1</kbd><span class="choice-name">Natural</span><span class="choice-help">says the meaning</span></button>
|
||||
<button class="choice" type="button" role="radio" data-label="mixed"><kbd>2</kbd><span class="choice-name">Mixed</span><span class="choice-help">one or two spelled parts</span></button>
|
||||
<button class="choice" type="button" role="radio" data-label="spelled"><kbd>3</kbd><span class="choice-name">Spelled</span><span class="choice-help">reads the markup</span></button>
|
||||
</div>
|
||||
<label class="note">
|
||||
<span>Note</span>
|
||||
<textarea rows="2" placeholder="Which part is spelled, or how it should be read. Markdown works."></textarea>
|
||||
</label>
|
||||
<div class="note-preview markdown" hidden></div>
|
||||
<div class="judge" hidden></div>
|
||||
<div class="tools">
|
||||
<button class="button ghost" type="button" data-action="copy">Copy translation prompt</button>
|
||||
</div>
|
||||
</article>
|
||||
</template>
|
||||
|
||||
<script type="module" src="./main.ts"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,266 +0,0 @@
|
||||
/// <reference types="vite/client" />
|
||||
|
||||
import type { CalibrationFile, CalibrationItem, ReadingLabel } from './types'
|
||||
|
||||
import katex from 'katex'
|
||||
import rehypeStringify from 'rehype-stringify'
|
||||
import remarkParse from 'remark-parse'
|
||||
import remarkRehype from 'remark-rehype'
|
||||
|
||||
import { unified } from 'unified'
|
||||
|
||||
import 'katex/contrib/mhchem'
|
||||
import 'katex/dist/katex.min.css'
|
||||
|
||||
const labels: readonly ReadingLabel[] = ['natural', 'mixed', 'spelled']
|
||||
const languageNames: Readonly<Record<string, string>> = { zh: 'Mandarin Chinese', en: 'English', ja: 'Japanese', ko: 'Korean' }
|
||||
|
||||
const stage = document.querySelector<HTMLElement>('#stage')!
|
||||
const template = document.querySelector<HTMLTemplateElement>('#card-template')!
|
||||
const stats = document.querySelector<HTMLElement>('#stats')!
|
||||
const progress = document.querySelector<HTMLElement>('#progress')!
|
||||
const position = document.querySelector<HTMLElement>('#position')!
|
||||
const toast = document.querySelector<HTMLElement>('#toast')!
|
||||
const languageSelect = document.querySelector<HTMLSelectElement>('#language')!
|
||||
const showSelect = document.querySelector<HTMLSelectElement>('#show')!
|
||||
const previousButton = document.querySelector<HTMLButtonElement>('#previous')!
|
||||
const nextButton = document.querySelector<HTMLButtonElement>('#next')!
|
||||
|
||||
/**
|
||||
* Renders the markdown of a judge reason or a note. remark-rehype drops raw HTML, so the text cannot inject markup.
|
||||
*/
|
||||
const markdown = unified().use(remarkParse).use(remarkRehype).use(rehypeStringify)
|
||||
function renderMarkdown(text: string): string {
|
||||
return String(markdown.processSync(text))
|
||||
}
|
||||
|
||||
let items: CalibrationItem[] = []
|
||||
/**
|
||||
* The items in the order of the current filters. It is computed when the page loads or a filter changes, never when a
|
||||
* label changes, so labeling an item never moves the card to another item.
|
||||
*/
|
||||
let order: CalibrationItem[] = []
|
||||
let index = 0
|
||||
|
||||
function computeOrder() {
|
||||
const language = languageSelect.value
|
||||
const show = showSelect.value
|
||||
const filtered = items.filter(item => !language || item.language === language)
|
||||
if (show === 'disagree')
|
||||
order = filtered.filter(item => item.label !== null && item.label !== item.judge.reading)
|
||||
else if (show === 'unlabeled')
|
||||
order = [...filtered.filter(item => item.label === null), ...filtered.filter(item => item.label !== null)]
|
||||
else
|
||||
order = filtered
|
||||
index = 0
|
||||
}
|
||||
|
||||
let toastTimer: ReturnType<typeof setTimeout> | undefined
|
||||
function showToast(message: string, error = false) {
|
||||
toast.textContent = message
|
||||
toast.classList.toggle('error', error)
|
||||
toast.classList.add('visible')
|
||||
clearTimeout(toastTimer)
|
||||
toastTimer = setTimeout(() => toast.classList.remove('visible'), error ? 4000 : 1400)
|
||||
}
|
||||
|
||||
function renderStats() {
|
||||
const labeled = items.filter(item => item.label !== null)
|
||||
const agree = labeled.filter(item => item.label === item.judge.reading).length
|
||||
const rate = labeled.length ? `${Math.round(100 * agree / labeled.length)}%` : '–'
|
||||
stats.textContent = `${labeled.length} of ${items.length} labeled · the judge agrees on ${agree} of ${labeled.length} (${rate})`
|
||||
progress.style.width = `${items.length ? 100 * labeled.length / items.length : 0}%`
|
||||
}
|
||||
|
||||
/** Renders the formula with KaTeX. A formula that KaTeX cannot render shows a short notice above its source. */
|
||||
function renderFormula(target: HTMLElement, tex: string) {
|
||||
try {
|
||||
target.innerHTML = katex.renderToString(tex, { displayMode: true, throwOnError: true })
|
||||
}
|
||||
catch {
|
||||
target.innerHTML = '<p class="formula-error">KaTeX cannot render this source.</p>'
|
||||
}
|
||||
}
|
||||
|
||||
/** A prompt for any agent: translate the reading and say how a native speaker would hear it. */
|
||||
function translationPrompt(item: CalibrationItem): string {
|
||||
return [
|
||||
`This is a ${languageNames[item.language] ?? item.language} reading of a math formula for a text-to-speech voice.`,
|
||||
`LaTeX: ${item.reference}`,
|
||||
`Reading: ${item.spoken}`,
|
||||
'Translate the reading word by word into Chinese. Then say whether a native speaker hears it as a teacher reading the meaning, or as someone spelling the markup, such as naming every bracket or saying command names.',
|
||||
].join('\n')
|
||||
}
|
||||
|
||||
/** The scripts that a reading in each language must use. A reading without them is in another language. */
|
||||
const languageScripts: Readonly<Record<string, RegExp>> = {
|
||||
zh: /\p{Script=Han}/u,
|
||||
ja: /[\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Han}]/u,
|
||||
ko: /\p{Script=Hangul}/u,
|
||||
}
|
||||
|
||||
function chip(text: string, strong = false): HTMLSpanElement {
|
||||
const element = document.createElement('span')
|
||||
element.className = strong ? 'chip strong' : 'chip'
|
||||
element.textContent = text
|
||||
return element
|
||||
}
|
||||
|
||||
/**
|
||||
* Updates only what a label changes: the chosen choice, the verdict of the judge, and the stats. The note box stays,
|
||||
* so its focus and cursor stay too.
|
||||
*/
|
||||
function renderLabel(card: HTMLElement, item: CalibrationItem) {
|
||||
for (const choice of card.querySelectorAll<HTMLButtonElement>('.choice'))
|
||||
choice.setAttribute('aria-checked', String(choice.dataset.label === item.label))
|
||||
|
||||
const judge = card.querySelector<HTMLElement>('.judge')!
|
||||
// The verdict shows only after the person labels, so it does not decide the label for them.
|
||||
judge.hidden = item.label === null
|
||||
if (item.label !== null) {
|
||||
const agree = item.label === item.judge.reading
|
||||
judge.className = `judge ${agree ? 'agree' : 'disagree'}`
|
||||
judge.innerHTML = `<div class="judge-head">${agree ? '✓ The judge agrees' : '✗ The judge disagrees'}: ${item.judge.reading}</div>`
|
||||
+ `<div class="markdown">${renderMarkdown(item.judge.reason)}</div>`
|
||||
}
|
||||
renderStats()
|
||||
}
|
||||
|
||||
function renderNotePreview(card: HTMLElement, item: CalibrationItem) {
|
||||
const preview = card.querySelector<HTMLElement>('.note-preview')!
|
||||
preview.hidden = !item.note.trim()
|
||||
preview.innerHTML = renderMarkdown(item.note)
|
||||
}
|
||||
|
||||
/** Renders the card of the current item. `direction` slides it in from the side it comes from. */
|
||||
function renderCard(direction: 'next' | 'previous' | 'none' = 'none') {
|
||||
index = Math.min(Math.max(index, 0), Math.max(order.length - 1, 0))
|
||||
const item = order[index]
|
||||
previousButton.disabled = index <= 0
|
||||
nextButton.disabled = index >= order.length - 1
|
||||
position.textContent = order.length ? `${index + 1} / ${order.length}` : ''
|
||||
renderStats()
|
||||
if (!item) {
|
||||
stage.innerHTML = '<div class="card empty">No readings match the filters.</div>'
|
||||
return
|
||||
}
|
||||
|
||||
const card = template.content.firstElementChild!.cloneNode(true) as HTMLElement
|
||||
if (direction !== 'none')
|
||||
card.classList.add(`enter-${direction}`)
|
||||
card.querySelector('.meta')!.append(
|
||||
chip(languageNames[item.language] ?? item.language, true),
|
||||
chip(item.object),
|
||||
chip(item.caseId),
|
||||
)
|
||||
renderFormula(card.querySelector<HTMLElement>('.formula')!, item.reference)
|
||||
card.querySelector('.source')!.textContent = item.reference
|
||||
const spoken = card.querySelector<HTMLElement>('.spoken')!
|
||||
spoken.textContent = item.spoken
|
||||
spoken.lang = item.language
|
||||
// A test object can answer in another language, for example sre speaks only English. The text is shown as it came,
|
||||
// so the warning says that this is the reading itself and not a display error.
|
||||
const script = languageScripts[item.language]
|
||||
if (script && !script.test(item.spoken)) {
|
||||
const warning = document.createElement('p')
|
||||
warning.className = 'language-warning'
|
||||
warning.textContent = `This reading is not in ${languageNames[item.language] ?? item.language}. The test object wrote it this way. A reading in another language does not pass.`
|
||||
spoken.after(warning)
|
||||
}
|
||||
|
||||
for (const choice of card.querySelectorAll<HTMLButtonElement>('.choice'))
|
||||
choice.addEventListener('click', () => choose(card, item, choice.dataset.label as ReadingLabel))
|
||||
|
||||
const noteBox = card.querySelector<HTMLTextAreaElement>('textarea')!
|
||||
noteBox.value = item.note
|
||||
// The note is kept on the item as the person types, so a label chosen before the note is saved carries the new note.
|
||||
noteBox.addEventListener('input', () => {
|
||||
item.note = noteBox.value
|
||||
renderNotePreview(card, item)
|
||||
})
|
||||
noteBox.addEventListener('change', () => void save(item, 'Note saved'))
|
||||
|
||||
card.querySelector<HTMLButtonElement>('[data-action="copy"]')!.addEventListener('click', () => {
|
||||
navigator.clipboard.writeText(translationPrompt(item))
|
||||
.then(() => showToast('Translation prompt copied'))
|
||||
.catch(() => showToast('The browser blocked the clipboard', true))
|
||||
})
|
||||
|
||||
stage.replaceChildren(card)
|
||||
renderLabel(card, item)
|
||||
renderNotePreview(card, item)
|
||||
}
|
||||
|
||||
/** Saves of the page, one at a time, so a later save never lands before an earlier one. */
|
||||
let saves: Promise<unknown> = Promise.resolve()
|
||||
|
||||
/** Saves the label and the note of an item to calibration.json through the dev server. */
|
||||
function save(item: CalibrationItem, message: string): Promise<void> {
|
||||
const body = JSON.stringify({ id: item.id, label: item.label, note: item.note })
|
||||
const run = saves.then(async () => {
|
||||
const response = await fetch('/api/items', { method: 'POST', body })
|
||||
if (!response.ok)
|
||||
throw new Error(`${response.status} ${await response.text()}`)
|
||||
})
|
||||
saves = run.catch(() => undefined)
|
||||
return run.then(
|
||||
() => showToast(message),
|
||||
(error: unknown) => showToast(`Not saved: ${String(error)}`, true),
|
||||
)
|
||||
}
|
||||
|
||||
/** Chooses a label, or clears it when it is the label the item already has. The card stays on the item. */
|
||||
function choose(card: HTMLElement, item: CalibrationItem, label: ReadingLabel) {
|
||||
item.label = item.label === label ? null : label
|
||||
const choice = card.querySelector<HTMLButtonElement>(`.choice[data-label="${label}"]`)!
|
||||
choice.classList.remove('pulse')
|
||||
// Reading the layout restarts the animation when the same choice is pressed twice in a row.
|
||||
void choice.offsetWidth
|
||||
choice.classList.add('pulse')
|
||||
renderLabel(card, item)
|
||||
void save(item, item.label === null ? 'Label cleared' : `Saved as ${item.label}`)
|
||||
}
|
||||
|
||||
function move(step: number) {
|
||||
const next = index + step
|
||||
if (next < 0 || next >= order.length)
|
||||
return
|
||||
index = next
|
||||
renderCard(step > 0 ? 'next' : 'previous')
|
||||
}
|
||||
|
||||
document.addEventListener('keydown', (event) => {
|
||||
if (event.target instanceof HTMLTextAreaElement || event.target instanceof HTMLSelectElement || event.isComposing || event.metaKey || event.ctrlKey || event.altKey)
|
||||
return
|
||||
const card = stage.querySelector<HTMLElement>('.card')
|
||||
const item = order[index]
|
||||
const chosen = ['1', '2', '3'].indexOf(event.key)
|
||||
if (card && item && chosen >= 0)
|
||||
choose(card, item, labels[chosen]!)
|
||||
else if (event.key === 'ArrowRight' || event.key === 'Enter')
|
||||
move(1)
|
||||
else if (event.key === 'ArrowLeft')
|
||||
move(-1)
|
||||
else
|
||||
return
|
||||
event.preventDefault()
|
||||
})
|
||||
previousButton.addEventListener('click', () => move(-1))
|
||||
nextButton.addEventListener('click', () => move(1))
|
||||
for (const select of [languageSelect, showSelect]) {
|
||||
select.addEventListener('change', () => {
|
||||
computeOrder()
|
||||
renderCard('none')
|
||||
})
|
||||
}
|
||||
|
||||
async function load() {
|
||||
const file = await (await fetch('/api/items')).json() as CalibrationFile
|
||||
items = file.items
|
||||
for (const language of [...new Set(items.map(item => item.language))].sort())
|
||||
languageSelect.append(new Option(languageNames[language] ?? language, language))
|
||||
computeOrder()
|
||||
renderCard()
|
||||
}
|
||||
|
||||
void load()
|
||||
@@ -1,548 +0,0 @@
|
||||
/*
|
||||
* Neutral surfaces with the AIRI default hue (220.44) as the accent, see DESIGN.md.
|
||||
* The label colors name a state only with the text and the check mark next to them.
|
||||
*/
|
||||
:root {
|
||||
color-scheme: light;
|
||||
--hue: 220.44;
|
||||
--bg: oklch(98.5% 0.004 var(--hue));
|
||||
--surface: oklch(100% 0 0);
|
||||
--surface-muted: oklch(96.5% 0.006 var(--hue));
|
||||
--border: oklch(91% 0.008 var(--hue));
|
||||
--border-strong: oklch(84% 0.012 var(--hue));
|
||||
--text: oklch(24% 0.015 var(--hue));
|
||||
--muted: oklch(52% 0.015 var(--hue));
|
||||
--primary: oklch(62% 0.14 var(--hue));
|
||||
--primary-soft: oklch(94% 0.04 var(--hue));
|
||||
--natural: oklch(62% 0.15 150);
|
||||
--natural-soft: oklch(95% 0.05 150);
|
||||
--mixed: oklch(70% 0.15 70);
|
||||
--mixed-soft: oklch(96% 0.05 80);
|
||||
--spelled: oklch(60% 0.18 25);
|
||||
--spelled-soft: oklch(95% 0.04 25);
|
||||
--shadow: 0 1px 2px oklch(20% 0.02 var(--hue) / 6%), 0 8px 24px oklch(20% 0.02 var(--hue) / 6%);
|
||||
--radius: 14px;
|
||||
--ease: cubic-bezier(0.2, 0.8, 0.2, 1);
|
||||
}
|
||||
|
||||
@media (prefers-color-scheme: dark) {
|
||||
:root {
|
||||
color-scheme: dark;
|
||||
--bg: oklch(17% 0.01 var(--hue));
|
||||
--surface: oklch(21% 0.012 var(--hue));
|
||||
--surface-muted: oklch(25% 0.012 var(--hue));
|
||||
--border: oklch(30% 0.012 var(--hue));
|
||||
--border-strong: oklch(38% 0.015 var(--hue));
|
||||
--text: oklch(94% 0.006 var(--hue));
|
||||
--muted: oklch(70% 0.012 var(--hue));
|
||||
--primary: oklch(72% 0.13 var(--hue));
|
||||
--primary-soft: oklch(30% 0.05 var(--hue));
|
||||
--natural-soft: oklch(30% 0.06 150);
|
||||
--mixed-soft: oklch(32% 0.06 80);
|
||||
--spelled-soft: oklch(30% 0.06 25);
|
||||
--shadow: 0 1px 2px oklch(0% 0 0 / 30%), 0 8px 24px oklch(0% 0 0 / 25%);
|
||||
}
|
||||
}
|
||||
|
||||
* {
|
||||
box-sizing: border-box;
|
||||
}
|
||||
|
||||
body {
|
||||
margin: 0;
|
||||
background: var(--bg);
|
||||
color: var(--text);
|
||||
font: 15px/1.6 ui-sans-serif, system-ui, -apple-system, 'Segoe UI', 'PingFang SC', 'Hiragino Sans', 'Apple SD Gothic Neo', sans-serif;
|
||||
-webkit-font-smoothing: antialiased;
|
||||
}
|
||||
|
||||
main {
|
||||
max-width: 880px;
|
||||
margin: 0 auto;
|
||||
padding: 28px 16px 72px;
|
||||
}
|
||||
|
||||
kbd {
|
||||
display: inline-grid;
|
||||
place-items: center;
|
||||
min-width: 1.6em;
|
||||
height: 1.6em;
|
||||
padding: 0 0.4em;
|
||||
border: 1px solid var(--border-strong);
|
||||
border-bottom-width: 2px;
|
||||
border-radius: 6px;
|
||||
background: var(--surface);
|
||||
color: var(--muted);
|
||||
font: 600 11px/1 ui-monospace, SFMono-Regular, Menlo, monospace;
|
||||
}
|
||||
|
||||
/* Header */
|
||||
|
||||
.top {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 16px;
|
||||
align-items: flex-end;
|
||||
justify-content: space-between;
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
|
||||
.title {
|
||||
flex: 1 1 280px;
|
||||
}
|
||||
|
||||
h1 {
|
||||
margin: 0;
|
||||
font-size: 22px;
|
||||
font-weight: 650;
|
||||
letter-spacing: -0.01em;
|
||||
}
|
||||
|
||||
.stats {
|
||||
margin: 2px 0 10px;
|
||||
color: var(--muted);
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
.progress {
|
||||
height: 6px;
|
||||
border-radius: 999px;
|
||||
background: var(--surface-muted);
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.progress-bar {
|
||||
width: 0;
|
||||
height: 100%;
|
||||
border-radius: inherit;
|
||||
background: var(--primary);
|
||||
transition: width 400ms var(--ease);
|
||||
}
|
||||
|
||||
.filters {
|
||||
display: flex;
|
||||
gap: 10px;
|
||||
}
|
||||
|
||||
.field {
|
||||
display: grid;
|
||||
gap: 4px;
|
||||
font-size: 12px;
|
||||
color: var(--muted);
|
||||
}
|
||||
|
||||
select {
|
||||
appearance: none;
|
||||
min-width: 150px;
|
||||
padding: 8px 32px 8px 12px;
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 10px;
|
||||
background: var(--surface) url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='12' height='12' viewBox='0 0 24 24' fill='none' stroke='%23888' stroke-width='2.5' stroke-linecap='round' stroke-linejoin='round'%3E%3Cpath d='m6 9 6 6 6-6'/%3E%3C/svg%3E") no-repeat right 12px center;
|
||||
color: var(--text);
|
||||
font: inherit;
|
||||
font-size: 14px;
|
||||
cursor: pointer;
|
||||
transition: border-color 150ms var(--ease), box-shadow 150ms var(--ease);
|
||||
}
|
||||
|
||||
select:hover {
|
||||
border-color: var(--border-strong);
|
||||
}
|
||||
|
||||
/* Buttons */
|
||||
|
||||
.button,
|
||||
.choice {
|
||||
font: inherit;
|
||||
color: inherit;
|
||||
cursor: pointer;
|
||||
user-select: none;
|
||||
-webkit-tap-highlight-color: transparent;
|
||||
transition:
|
||||
background-color 150ms var(--ease),
|
||||
border-color 150ms var(--ease),
|
||||
color 150ms var(--ease),
|
||||
box-shadow 150ms var(--ease),
|
||||
transform 120ms var(--ease);
|
||||
}
|
||||
|
||||
.button {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
padding: 8px 14px;
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 10px;
|
||||
background: var(--surface);
|
||||
font-size: 14px;
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
.button:hover:not(:disabled) {
|
||||
border-color: var(--border-strong);
|
||||
background: var(--surface-muted);
|
||||
}
|
||||
|
||||
.button:active:not(:disabled),
|
||||
.choice:active {
|
||||
transform: scale(0.97);
|
||||
}
|
||||
|
||||
.button:disabled {
|
||||
opacity: 0.4;
|
||||
cursor: default;
|
||||
}
|
||||
|
||||
.button.ghost {
|
||||
border-color: transparent;
|
||||
background: transparent;
|
||||
color: var(--muted);
|
||||
padding-inline: 10px;
|
||||
}
|
||||
|
||||
.button.ghost:hover {
|
||||
background: var(--surface-muted);
|
||||
color: var(--text);
|
||||
}
|
||||
|
||||
:is(.button, .choice, select, textarea):focus-visible {
|
||||
outline: none;
|
||||
box-shadow: 0 0 0 3px oklch(from var(--primary) l c h / 35%);
|
||||
}
|
||||
|
||||
/* Card */
|
||||
|
||||
.stage {
|
||||
position: relative;
|
||||
}
|
||||
|
||||
.card {
|
||||
padding: 24px;
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius);
|
||||
background: var(--surface);
|
||||
box-shadow: var(--shadow);
|
||||
}
|
||||
|
||||
.card.enter-next {
|
||||
animation: enter-next 260ms var(--ease);
|
||||
}
|
||||
|
||||
.card.enter-previous {
|
||||
animation: enter-previous 260ms var(--ease);
|
||||
}
|
||||
|
||||
@keyframes enter-next {
|
||||
from { opacity: 0; transform: translateX(24px); }
|
||||
}
|
||||
|
||||
@keyframes enter-previous {
|
||||
from { opacity: 0; transform: translateX(-24px); }
|
||||
}
|
||||
|
||||
.meta {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 6px;
|
||||
margin-bottom: 8px;
|
||||
}
|
||||
|
||||
.chip {
|
||||
padding: 2px 10px;
|
||||
border-radius: 999px;
|
||||
background: var(--surface-muted);
|
||||
color: var(--muted);
|
||||
font-size: 12px;
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
.chip.strong {
|
||||
background: var(--primary-soft);
|
||||
color: var(--primary);
|
||||
}
|
||||
|
||||
.formula {
|
||||
overflow-x: auto;
|
||||
padding: 8px 0;
|
||||
font-size: 1.15em;
|
||||
}
|
||||
|
||||
.formula .katex-display {
|
||||
margin: 0.6em 0;
|
||||
}
|
||||
|
||||
.formula-error {
|
||||
color: var(--muted);
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
.source {
|
||||
display: block;
|
||||
padding: 8px 12px;
|
||||
border-radius: 8px;
|
||||
background: var(--surface-muted);
|
||||
color: var(--muted);
|
||||
font: 12px/1.5 ui-monospace, SFMono-Regular, Menlo, monospace;
|
||||
white-space: pre-wrap;
|
||||
word-break: break-all;
|
||||
}
|
||||
|
||||
.spoken {
|
||||
margin: 18px 0;
|
||||
padding: 18px 20px;
|
||||
border-left: 3px solid var(--primary);
|
||||
border-radius: 4px 12px 12px 4px;
|
||||
background: var(--primary-soft);
|
||||
font-size: 21px;
|
||||
line-height: 1.7;
|
||||
word-break: break-word;
|
||||
}
|
||||
|
||||
.language-warning {
|
||||
margin: -8px 0 16px;
|
||||
padding: 8px 12px;
|
||||
border-radius: 8px;
|
||||
background: var(--spelled-soft);
|
||||
color: var(--text);
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
/* Label choices */
|
||||
|
||||
.choices {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(3, 1fr);
|
||||
gap: 10px;
|
||||
}
|
||||
|
||||
.choice {
|
||||
position: relative;
|
||||
display: grid;
|
||||
grid-template-columns: auto 1fr;
|
||||
grid-template-rows: auto auto;
|
||||
column-gap: 10px;
|
||||
align-items: center;
|
||||
padding: 12px 14px;
|
||||
border: 1.5px solid var(--border);
|
||||
border-radius: 12px;
|
||||
background: var(--surface);
|
||||
text-align: left;
|
||||
}
|
||||
|
||||
.choice kbd {
|
||||
grid-row: span 2;
|
||||
}
|
||||
|
||||
.choice-name {
|
||||
font-weight: 650;
|
||||
}
|
||||
|
||||
.choice-help {
|
||||
color: var(--muted);
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.choice:hover {
|
||||
border-color: var(--border-strong);
|
||||
background: var(--surface-muted);
|
||||
}
|
||||
|
||||
.choice[aria-checked='true']::after {
|
||||
content: '✓';
|
||||
position: absolute;
|
||||
top: 8px;
|
||||
right: 12px;
|
||||
font-weight: 700;
|
||||
}
|
||||
|
||||
.choice[data-label='natural'][aria-checked='true'] {
|
||||
border-color: var(--natural);
|
||||
background: var(--natural-soft);
|
||||
}
|
||||
|
||||
.choice[data-label='mixed'][aria-checked='true'] {
|
||||
border-color: var(--mixed);
|
||||
background: var(--mixed-soft);
|
||||
}
|
||||
|
||||
.choice[data-label='spelled'][aria-checked='true'] {
|
||||
border-color: var(--spelled);
|
||||
background: var(--spelled-soft);
|
||||
}
|
||||
|
||||
.choice[data-label='natural'][aria-checked='true']::after { color: var(--natural); }
|
||||
.choice[data-label='mixed'][aria-checked='true']::after { color: var(--mixed); }
|
||||
.choice[data-label='spelled'][aria-checked='true']::after { color: var(--spelled); }
|
||||
|
||||
/* A short pulse confirms the choice where the eye already is. */
|
||||
.choice.pulse {
|
||||
animation: pulse 320ms var(--ease);
|
||||
}
|
||||
|
||||
@keyframes pulse {
|
||||
40% { transform: scale(1.03); }
|
||||
}
|
||||
|
||||
/* Note and judge */
|
||||
|
||||
.note {
|
||||
display: grid;
|
||||
gap: 6px;
|
||||
margin-top: 18px;
|
||||
color: var(--muted);
|
||||
font-size: 12px;
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
textarea {
|
||||
width: 100%;
|
||||
min-height: 64px;
|
||||
padding: 10px 12px;
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 10px;
|
||||
background: var(--surface);
|
||||
color: var(--text);
|
||||
font: inherit;
|
||||
font-size: 14px;
|
||||
resize: vertical;
|
||||
transition: border-color 150ms var(--ease), box-shadow 150ms var(--ease);
|
||||
}
|
||||
|
||||
textarea:hover {
|
||||
border-color: var(--border-strong);
|
||||
}
|
||||
|
||||
.judge,
|
||||
.note-preview {
|
||||
margin-top: 14px;
|
||||
padding: 12px 14px;
|
||||
border-radius: 10px;
|
||||
font-size: 14px;
|
||||
animation: reveal 220ms var(--ease);
|
||||
}
|
||||
|
||||
.note-preview {
|
||||
background: var(--surface-muted);
|
||||
}
|
||||
|
||||
.judge {
|
||||
border: 1px solid var(--border);
|
||||
}
|
||||
|
||||
.judge.agree {
|
||||
border-color: var(--natural);
|
||||
background: var(--natural-soft);
|
||||
}
|
||||
|
||||
.judge.disagree {
|
||||
border-color: var(--spelled);
|
||||
background: var(--spelled-soft);
|
||||
}
|
||||
|
||||
.judge-head {
|
||||
display: flex;
|
||||
gap: 8px;
|
||||
align-items: center;
|
||||
margin-bottom: 4px;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
@keyframes reveal {
|
||||
from { opacity: 0; transform: translateY(-4px); }
|
||||
}
|
||||
|
||||
.markdown > :first-child { margin-top: 0; }
|
||||
.markdown > :last-child { margin-bottom: 0; }
|
||||
.markdown p { margin: 0.3em 0; }
|
||||
.markdown code {
|
||||
padding: 0.1em 0.35em;
|
||||
border-radius: 5px;
|
||||
background: oklch(from var(--text) l c h / 8%);
|
||||
font: 0.92em ui-monospace, SFMono-Regular, Menlo, monospace;
|
||||
}
|
||||
.markdown ul,
|
||||
.markdown ol { margin: 0.3em 0; padding-left: 1.4em; }
|
||||
|
||||
.tools {
|
||||
display: flex;
|
||||
justify-content: flex-end;
|
||||
margin-top: 10px;
|
||||
}
|
||||
|
||||
/* Pager and hint */
|
||||
|
||||
.pager {
|
||||
display: flex;
|
||||
gap: 12px;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
margin-top: 16px;
|
||||
}
|
||||
|
||||
.position {
|
||||
color: var(--muted);
|
||||
font-size: 13px;
|
||||
font-variant-numeric: tabular-nums;
|
||||
}
|
||||
|
||||
.hint {
|
||||
margin-top: 20px;
|
||||
color: var(--muted);
|
||||
font-size: 13px;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.empty {
|
||||
padding: 48px 24px;
|
||||
color: var(--muted);
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
/* Toast */
|
||||
|
||||
.toast {
|
||||
position: fixed;
|
||||
top: 20px;
|
||||
right: 20px;
|
||||
padding: 10px 16px;
|
||||
border-radius: 999px;
|
||||
background: var(--text);
|
||||
color: var(--bg);
|
||||
font-size: 13px;
|
||||
box-shadow: var(--shadow);
|
||||
opacity: 0;
|
||||
pointer-events: none;
|
||||
transform: translateY(-8px);
|
||||
transition: opacity 200ms var(--ease), transform 200ms var(--ease);
|
||||
}
|
||||
|
||||
.toast.visible {
|
||||
opacity: 1;
|
||||
transform: translateY(0);
|
||||
}
|
||||
|
||||
.toast.error {
|
||||
background: var(--spelled);
|
||||
color: white;
|
||||
}
|
||||
|
||||
/* Small screens */
|
||||
|
||||
@media (max-width: 640px) {
|
||||
.filters { width: 100%; }
|
||||
.field { flex: 1; }
|
||||
select { min-width: 0; width: 100%; }
|
||||
.card { padding: 18px 16px; }
|
||||
.spoken { font-size: 18px; }
|
||||
.choices { grid-template-columns: 1fr; }
|
||||
.pager .button { padding-inline: 10px; }
|
||||
}
|
||||
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
*,
|
||||
*::before,
|
||||
*::after {
|
||||
animation: none !important;
|
||||
transition: none !important;
|
||||
}
|
||||
}
|
||||
@@ -1,90 +0,0 @@
|
||||
import type { Plugin } from 'vite'
|
||||
|
||||
import type { CalibrationFile, ReadingLabel } from './types'
|
||||
|
||||
import { readFile, rename, writeFile } from 'node:fs/promises'
|
||||
import { env } from 'node:process'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
import { defineConfig } from 'vite'
|
||||
|
||||
/** `CALIBRATION_FILE` points the tool at another file, for example a copy to test the tool without touching labels. */
|
||||
const calibrationFile = env.CALIBRATION_FILE ?? fileURLToPath(new URL('./calibration.json', import.meta.url))
|
||||
const labels = new Set<ReadingLabel>(['natural', 'mixed', 'spelled'])
|
||||
|
||||
/**
|
||||
* Runs the updates of the file one at a time.
|
||||
*
|
||||
* Leaving a note box and clicking a label sends two saves at once. Two read-modify-write updates that overlap wrote
|
||||
* over each other and left two JSON documents in one file, so every update waits for the one before it. The new
|
||||
* file is written next to the old one and renamed over it, so a failed write never leaves half a file.
|
||||
*/
|
||||
let updates: Promise<unknown> = Promise.resolve()
|
||||
function update(change: (file: CalibrationFile) => boolean): Promise<boolean> {
|
||||
const run = updates.then(async () => {
|
||||
const file = JSON.parse(await readFile(calibrationFile, 'utf8')) as CalibrationFile
|
||||
if (!change(file))
|
||||
return false
|
||||
const partial = `${calibrationFile}.partial`
|
||||
await writeFile(partial, `${JSON.stringify(file, null, 2)}\n`)
|
||||
await rename(partial, calibrationFile)
|
||||
return true
|
||||
})
|
||||
updates = run.catch(() => undefined)
|
||||
return run
|
||||
}
|
||||
|
||||
/**
|
||||
* Serves `calibration.json` to the page and writes each label back to it.
|
||||
*
|
||||
* - `GET /api/items` returns the file.
|
||||
* - `POST /api/items` with `{ id, label, note }` sets the label of one item. `label: null` clears it.
|
||||
*
|
||||
* The file is the only state, so a label is saved as soon as the person chooses it.
|
||||
*/
|
||||
function calibrationApi(): Plugin {
|
||||
return {
|
||||
name: 'calibration-api',
|
||||
configureServer(server) {
|
||||
server.middlewares.use('/api/items', async (request, response) => {
|
||||
if (request.method === 'GET') {
|
||||
response.setHeader('content-type', 'application/json')
|
||||
response.end(await readFile(calibrationFile, 'utf8'))
|
||||
return
|
||||
}
|
||||
if (request.method !== 'POST') {
|
||||
response.statusCode = 405
|
||||
response.end()
|
||||
return
|
||||
}
|
||||
|
||||
let body = ''
|
||||
for await (const chunk of request)
|
||||
body += chunk
|
||||
const { id, label, note } = JSON.parse(body) as { id: string, label: ReadingLabel | null, note: string }
|
||||
if (label !== null && !labels.has(label)) {
|
||||
response.statusCode = 400
|
||||
response.end('unknown label')
|
||||
return
|
||||
}
|
||||
const found = await update((file) => {
|
||||
const item = file.items.find(candidate => candidate.id === id)
|
||||
if (!item)
|
||||
return false
|
||||
item.label = label
|
||||
item.note = note
|
||||
return true
|
||||
})
|
||||
response.statusCode = found ? 200 : 404
|
||||
response.end(found ? 'ok' : 'unknown item')
|
||||
})
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/** The labeling tool of the judge calibration set. Run: pnpm -F @proj-airi/evals-tts-speech-formula run judge:label */
|
||||
export default defineConfig({
|
||||
root: fileURLToPath(new URL('.', import.meta.url)),
|
||||
plugins: [calibrationApi()],
|
||||
server: { port: Number(env.CALIBRATION_PORT ?? 5299), open: !env.CALIBRATION_FILE },
|
||||
})
|
||||
@@ -24,15 +24,14 @@
|
||||
"eval:compare": "pnpm -F @proj-airi/core-agent build && vieval run --config ./vieval.config.ts --with-viewer",
|
||||
"typecheck": "tsc --noEmit",
|
||||
"cases:math-bridge": "tsx eval/cases/math-bridge/fetch.ts",
|
||||
"judge:collect": "tsx eval/evaluators/judge/calibration/collect.ts",
|
||||
"judge:label": "vite --config eval/evaluators/judge/calibration/vite.config.ts"
|
||||
"judge:collect": "tsx eval/evaluators/judge/calibration/collect.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
"@cortex-js/compute-engine": "catalog:",
|
||||
"@duckdb/duckdb-wasm": "catalog:",
|
||||
"@moeru/std": "catalog:",
|
||||
"@proj-airi/core-agent": "workspace:^",
|
||||
"@proj-airi/duckdb-wasm": "catalog:",
|
||||
"@proj-airi/drizzle-duckdb-wasm": "catalog:",
|
||||
"@proj-airi/mathjson": "workspace:^",
|
||||
"@proj-airi/speech-transformer-math": "workspace:^",
|
||||
"@velin-dev/core-vue": "catalog:",
|
||||
@@ -41,6 +40,7 @@
|
||||
"@xsai/generate-speech": "catalog:",
|
||||
"@xsai/generate-text": "catalog:",
|
||||
"@xsai/generate-transcription": "catalog:",
|
||||
"drizzle-orm": "catalog:",
|
||||
"katex": "catalog:",
|
||||
"speech-rule-engine": "catalog:",
|
||||
"valibot": "catalog:",
|
||||
@@ -52,12 +52,7 @@
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "catalog:",
|
||||
"rehype-stringify": "catalog:",
|
||||
"remark-parse": "catalog:",
|
||||
"remark-rehype": "catalog:",
|
||||
"unified": "catalog:",
|
||||
"vieval": "catalog:",
|
||||
"vite": "catalog:",
|
||||
"vitest": "catalog:vitest"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,14 +1,8 @@
|
||||
/**
|
||||
* The primary language subtag of a BCP 47 code: `zh` for `zh-CN`, `ko` for `ko_KR`. Chinese dialect codes are `zh`,
|
||||
* because the readers have one Chinese pack.
|
||||
*/
|
||||
import { primaryLanguage } from '@proj-airi/speech-transformer-math'
|
||||
|
||||
/** The primary language subtag of a BCP 47 code, see `primaryLanguage` of `@proj-airi/speech-transformer-math`. */
|
||||
export function toSpeechLanguage(code: string | undefined): string | undefined {
|
||||
if (code === undefined)
|
||||
return undefined
|
||||
const primary = code.toLowerCase().split(/[-_]/)[0] ?? ''
|
||||
if (primary === 'cmn' || primary === 'yue')
|
||||
return 'zh'
|
||||
return primary || undefined
|
||||
return code === undefined ? undefined : primaryLanguage(code) || undefined
|
||||
}
|
||||
|
||||
const kana = /[\p{Script=Hiragana}\p{Script=Katakana}]/u
|
||||
|
||||
@@ -10,7 +10,7 @@ export { lowerMath } from './lower'
|
||||
export { spellNumber } from './numbers'
|
||||
export { parseMath } from './parse'
|
||||
export { renderMath } from './render'
|
||||
export { builtinLocales, MathSpeech } from './speech'
|
||||
export { builtinLocales, MathSpeech, primaryLanguage } from './speech'
|
||||
export type { MathSpeechOptions } from './speech'
|
||||
export { texCommands } from './tex'
|
||||
export type { TexNode } from './tex'
|
||||
|
||||
@@ -26,8 +26,11 @@ export interface MathSpeechOptions {
|
||||
fallbackLanguage?: MathLanguage
|
||||
}
|
||||
|
||||
/** The primary language subtag of a BCP 47 code: `zh` for `zh-CN`, `ko` for `ko_KR`. Chinese dialect codes are `zh`. */
|
||||
function primaryLanguage(code: string): string {
|
||||
/**
|
||||
* The primary language subtag of a BCP 47 code: `zh` for `zh-CN`, `ko` for `ko_KR`. Chinese dialect codes are `zh`,
|
||||
* because the readers have one Chinese pack.
|
||||
*/
|
||||
export function primaryLanguage(code: string): string {
|
||||
const primary = code.toLowerCase().split(/[-_]/)[0] ?? ''
|
||||
return primary === 'cmn' || primary === 'yue' ? 'zh' : primary
|
||||
}
|
||||
|
||||
Generated
+4
-19
@@ -405,9 +405,6 @@ catalogs:
|
||||
'@proj-airi/drizzle-duckdb-wasm':
|
||||
specifier: ^0.6.0
|
||||
version: 0.6.0
|
||||
'@proj-airi/duckdb-wasm':
|
||||
specifier: ^0.6.0
|
||||
version: 0.6.0
|
||||
'@proj-airi/iconify-meteocons':
|
||||
specifier: ^0.1.5
|
||||
version: 0.1.5
|
||||
@@ -3547,7 +3544,7 @@ importers:
|
||||
'@proj-airi/core-agent':
|
||||
specifier: workspace:^
|
||||
version: link:../../packages/core-agent
|
||||
'@proj-airi/duckdb-wasm':
|
||||
'@proj-airi/drizzle-duckdb-wasm':
|
||||
specifier: 'catalog:'
|
||||
version: 0.6.0(drizzle-orm@0.45.2(@electric-sql/pglite@0.5.6)(@opentelemetry/api@1.9.1)(@prisma/client@5.22.0)(@types/pg@8.23.1)(better-sqlite3@12.11.1)(kysely@0.29.5)(pg@8.23.0)(postgres@3.4.9))(web-worker@1.5.0)
|
||||
'@proj-airi/mathjson':
|
||||
@@ -3574,6 +3571,9 @@ importers:
|
||||
'@xsai/generate-transcription':
|
||||
specifier: 'catalog:'
|
||||
version: 0.5.1
|
||||
drizzle-orm:
|
||||
specifier: 'catalog:'
|
||||
version: 0.45.2(@electric-sql/pglite@0.5.6)(@opentelemetry/api@1.9.1)(@prisma/client@5.22.0)(@types/pg@8.23.1)(better-sqlite3@12.11.1)(kysely@0.29.5)(pg@8.23.0)(postgres@3.4.9)
|
||||
katex:
|
||||
specifier: 'catalog:'
|
||||
version: 0.19.0
|
||||
@@ -3590,24 +3590,9 @@ importers:
|
||||
'@types/node':
|
||||
specifier: 'catalog:'
|
||||
version: 26.2.0
|
||||
rehype-stringify:
|
||||
specifier: 'catalog:'
|
||||
version: 10.0.1
|
||||
remark-parse:
|
||||
specifier: 'catalog:'
|
||||
version: 11.0.0(supports-color@10.2.2)
|
||||
remark-rehype:
|
||||
specifier: 'catalog:'
|
||||
version: 11.1.2
|
||||
unified:
|
||||
specifier: 'catalog:'
|
||||
version: 11.0.5
|
||||
vieval:
|
||||
specifier: 'catalog:'
|
||||
version: 0.10.3(@types/node@26.2.0)(@vitest/browser-playwright@4.1.11)(@vitest/coverage-v8@4.1.11)(chokidar@5.0.0)(dotenv@17.4.2)(electron@43.4.1(supports-color@10.2.2))(esbuild@0.28.2)(giget@3.3.1)(h3@2.0.2(crossws@0.4.12(srvx@1.0.5)))(hono@4.13.3)(jiti@2.7.0)(jsdom@30.0.1(@noble/hashes@2.3.0)(canvas@3.2.3))(less@4.9.0(supports-color@10.2.2))(magicast@0.5.4)(terser@5.50.0)(tsx@4.23.12)(typescript@6.0.3)(web-worker@1.5.0)(yaml@2.9.0)
|
||||
vite:
|
||||
specifier: 'catalog:'
|
||||
version: 8.3.4(@types/node@26.2.0)(esbuild@0.28.2)(jiti@2.7.0)(less@4.9.0(supports-color@10.2.2))(terser@5.50.0)(tsx@4.23.12)(yaml@2.9.0)
|
||||
vitest:
|
||||
specifier: catalog:vitest
|
||||
version: 4.1.11(@opentelemetry/api@1.9.1)(@types/node@26.2.0)(@vitest/browser-playwright@4.1.11)(@vitest/coverage-v8@4.1.11)(jsdom@30.0.1(@noble/hashes@2.3.0)(canvas@3.2.3))(vite@8.3.4(@types/node@26.2.0)(esbuild@0.28.2)(jiti@2.7.0)(less@4.9.0(supports-color@10.2.2))(terser@5.50.0)(tsx@4.23.12)(yaml@2.9.0))
|
||||
|
||||
@@ -189,7 +189,6 @@ catalog:
|
||||
'@pnpm/find-workspace-dir': ^1000.1.5
|
||||
'@proj-airi/chromatic': ^1.1.4
|
||||
'@proj-airi/drizzle-duckdb-wasm': ^0.6.0
|
||||
'@proj-airi/duckdb-wasm': ^0.6.0
|
||||
'@proj-airi/iconify-meteocons': ^0.1.5
|
||||
'@proj-airi/lobe-icons': ^1.0.19
|
||||
'@proj-airi/unocss-preset-chromatic': ^1.1.4
|
||||
|
||||
Reference in New Issue
Block a user