mirror of
https://github.com/jgm/pandoc.git
synced 2026-09-17 17:39:54 +08:00
379 lines
12 KiB
Python
379 lines
12 KiB
Python
#!/usr/bin/env python3
|
|
# Update translations in data/translations
|
|
# based on data from Polyglossia and Babel.
|
|
#
|
|
# usage: python tools/update-translations.py
|
|
|
|
import json
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
from configparser import ConfigParser
|
|
from importlib.util import module_from_spec, spec_from_file_location
|
|
from pathlib import Path
|
|
from shutil import rmtree
|
|
|
|
YAML = dict[str, str]
|
|
AST = dict[str, "AST"] | list["AST"] | str
|
|
|
|
# missing Listing
|
|
BABEL_KEYS = {
|
|
"abstract": "Abstract",
|
|
"also": "SeeAlso",
|
|
"appendix": "Appendix",
|
|
# "appendix.template",
|
|
# [chapter].[ ][[appendix]], take appendix
|
|
"bib": "Bibliography",
|
|
"cc": "Cc",
|
|
"chapter": "Chapter",
|
|
# "chapter.template",
|
|
# [chapter].[ ][[chapter]], take chapter
|
|
# [[prechapter]] [chapter] [[postchapter]], take postchapter (cjk)
|
|
"contents": "Contents",
|
|
"encl": "Encl",
|
|
"figure": "Figure",
|
|
# "figure.template",
|
|
# [figure].[ ][[figure]], take figure
|
|
"glossary": "Glossary",
|
|
"headto": "To",
|
|
"index": "Index",
|
|
"listfigure": "ListOfFigures",
|
|
"listtable": "ListOfTables",
|
|
"page": "Page",
|
|
"part": "Part",
|
|
# "part.template",
|
|
# [part][ ][[part]], take part
|
|
# [[prepart]] [part] [[postpart]], take postpart (cjk)
|
|
# "postchapter", # see chapter.template
|
|
# "postpart", # see part.template
|
|
# "prechapter", # see chapter.template
|
|
"preface": "Preface",
|
|
# "prepart", # see part.template
|
|
"proof": "Proof",
|
|
"ref": "References",
|
|
"see": "See",
|
|
"table": "Table",
|
|
# "table.template",
|
|
# [table].[ ][[table]], take table
|
|
}
|
|
|
|
POLYGLOSSIA_KEYS = {
|
|
"abstract": "Abstract",
|
|
"also": "SeeAlso",
|
|
"appendix": "Appendix",
|
|
"bib": "Bibliography",
|
|
"cc": "Cc",
|
|
"chapter": "Chapter",
|
|
"contents": "Contents",
|
|
"encl": "Encl",
|
|
"figure": "Figure",
|
|
"glossary": "Glossary",
|
|
"headto": "To",
|
|
"index": "Index",
|
|
"listfigure": "ListOfFigures",
|
|
"listtable": "ListOfTables",
|
|
"page": "Page",
|
|
"part": "Part",
|
|
"preface": "Preface",
|
|
"proof": "Proof",
|
|
"ref": "References",
|
|
"see": "See",
|
|
"table": "Table",
|
|
}
|
|
|
|
|
|
def git_clone(url: str, branch: str | None = None) -> Path:
|
|
"""Download the Git repository at the provided url."""
|
|
return Path(
|
|
subprocess.run(
|
|
["git", "clone", "--depth", "1", url]
|
|
+ ([] if branch is None else ["--branch", branch]),
|
|
capture_output=True,
|
|
text=True,
|
|
).stderr.split("'")[1]
|
|
)
|
|
|
|
|
|
def parse_ast(tree: AST, is_map: bool = False):
|
|
"""Parse the pandoc value into a string."""
|
|
if isinstance(tree, dict) and is_map:
|
|
return {key: parse_ast(value) for key, value in tree.items()}
|
|
elif isinstance(tree, list):
|
|
return [parse_ast(value) for value in tree]
|
|
elif isinstance(tree, str):
|
|
return tree
|
|
ast_type = tree["t"]
|
|
assert ast_type in [
|
|
"MetaInlines",
|
|
"MetaString",
|
|
"Str",
|
|
"Space",
|
|
], f"Type {ast_type} is unsupported."
|
|
if ast_type == "MetaInlines":
|
|
return "".join(map(parse_ast, tree["c"]))
|
|
elif ast_type == "Space":
|
|
return " "
|
|
else:
|
|
return parse_ast(tree["c"])
|
|
|
|
|
|
def pandoc_parse(src: Path) -> YAML:
|
|
"""Parse YAML with pandoc's metadata parser."""
|
|
with src.open() as f:
|
|
data = f.read()
|
|
try:
|
|
# HACK: disable (most) markdown parsing of strings
|
|
# TODO: commonmark, with unicode normalization disabled
|
|
# https://github.com/jgm/pandoc/issues/8341
|
|
format = "markdown-smart-subscript"
|
|
pandoc_ast = json.loads(
|
|
subprocess.run(
|
|
["pandoc", f"--from={format}", "--to=json"],
|
|
input=f"---\n{data}\n---",
|
|
capture_output=True,
|
|
text=True,
|
|
check=True,
|
|
).stdout
|
|
)
|
|
except subprocess.CalledProcessError as error:
|
|
raise ValueError(error.stderr.strip())
|
|
return parse_ast(pandoc_ast["meta"], is_map=True) # type: ignore
|
|
|
|
|
|
def read_yaml(src: Path) -> YAML:
|
|
"""Read the YAML data with an ad hoc reader."""
|
|
with src.open() as f:
|
|
return {
|
|
(tokens := line.split(": "))[0]: (
|
|
value
|
|
if not (value := ": ".join(tokens[1:]).strip()).startswith("'")
|
|
or not value.endswith("'")
|
|
else value[1:-1]
|
|
)
|
|
for line in f.readlines()
|
|
}
|
|
|
|
|
|
def save_yaml(data: YAML, dst: Path) -> None:
|
|
"""Save the YAML data with an ad hoc writer."""
|
|
with dst.open("w") as f:
|
|
f.write(
|
|
"\n".join(
|
|
f"{key}: {value}"
|
|
for key, value in sorted(data.items(), key=lambda x: x[0])
|
|
) + "\n"
|
|
)
|
|
assert pandoc_parse(dst) == data, "Serialized different from expected."
|
|
|
|
|
|
def parse_babel(src: Path) -> tuple[str, set[str], YAML]:
|
|
"""Parse Babel's language files."""
|
|
# TODO: strict=True once https://github.com/latex3/babel/pull/303 lands
|
|
config = ConfigParser(strict=False)
|
|
config.read(src)
|
|
bcp47tag = config["identification"]["tag.bcp47"]
|
|
captions = config["captions"]
|
|
# HACK: manual modifications (see BABEL_KEYS)
|
|
if "postchapter" in captions:
|
|
captions["chapter"] = captions["postchapter"]
|
|
if "postpart" in captions:
|
|
captions["part"] = captions["postpart"]
|
|
data = {
|
|
BABEL_KEYS[key]: value if not value.endswith(":") else value[:-1]
|
|
for key, value in captions.items()
|
|
if key in BABEL_KEYS and len(value) > 0 and value != "<++>"
|
|
}
|
|
return bcp47tag, set(captions.keys()), data
|
|
|
|
|
|
def parse_braces(s: str, i: int) -> str:
|
|
"""Return the contiguous block starting at index i defined by braces."""
|
|
assert s[i] == "{", f"{s} at character {i} does not start with a brace."
|
|
left = 0
|
|
for j in range(i, len(s)):
|
|
ch = s[j]
|
|
if ch == "{":
|
|
left += 1
|
|
elif ch == "}":
|
|
left -= 1
|
|
if left == 0:
|
|
return s[i : j + 1]
|
|
raise ValueError(f"{s} is mismatched.")
|
|
|
|
|
|
def parse_polyglossia_value(value: str) -> str:
|
|
"""Parse the Polyglossia value."""
|
|
patterns = [
|
|
re.compile(r"\\@ensure@RTL\{(.*)\}"),
|
|
re.compile(r".*##1##2\s*(.*)"),
|
|
re.compile(r"\\textsc\{(.*)\}"),
|
|
]
|
|
for pattern in patterns:
|
|
match = pattern.match(value)
|
|
if match is not None:
|
|
return match.group(1)
|
|
if "xpg@hr@digraph" in value:
|
|
value = re.sub(
|
|
r"\\xpg@hr@digraph\{(.)\}\{(.)\}",
|
|
lambda match: match.group(1) + match.group(2),
|
|
value,
|
|
)
|
|
return value.replace("\\", "").replace(":", "").strip()
|
|
|
|
|
|
def parse_polyglossia(data: list[tuple[str, str]]) -> YAML:
|
|
"""Process the Polyglossia data."""
|
|
return {
|
|
POLYGLOSSIA_KEYS[key]: parse_polyglossia_value(value)
|
|
for key, value in data
|
|
if key in POLYGLOSSIA_KEYS
|
|
and value not in ["", r"$\rightarrow$", r"$\Rightarrow$"]
|
|
}
|
|
|
|
|
|
def parse_ldf(src: Path) -> list[tuple[str, dict[str, str], YAML]]:
|
|
"""Parse Polyglossia's language definition files."""
|
|
with src.open() as f:
|
|
data = f.read()
|
|
# HACK: regex parsing of latex
|
|
pattern = re.compile(r"(?m:^)[^%]*\\def\\([^\{]+)name[^\{]*\{(.*)\}")
|
|
languages = []
|
|
extra = []
|
|
for match in re.finditer(
|
|
r"\\def\\captions@?([^@\{]*)@?([^@\{]*)@?([^@\{]*)\{", data
|
|
):
|
|
language, variant, script = match.groups()
|
|
options = {}
|
|
if len(variant) > 0:
|
|
options["variant"] = variant
|
|
if len(script) > 0:
|
|
options["script"] = script
|
|
body = parse_braces(data, match.end() - 1)
|
|
matches = pattern.findall(body)
|
|
lines = [
|
|
line
|
|
for line in body.strip().splitlines()
|
|
if "def" in line
|
|
and "name" in line
|
|
and not line.strip().startswith("%")
|
|
]
|
|
assert len(lines) == len(matches), "Missing matches."
|
|
captions = parse_polyglossia(matches)
|
|
languages.append((language, options, captions))
|
|
if len(options) == 1 and "variant" in options:
|
|
extra.append((language, {"script": options["variant"]}, captions))
|
|
assert len(languages) == data.count(r"\def\captions"), "Missing captions."
|
|
return languages + extra
|
|
|
|
|
|
def get_tags(
|
|
bcp472lang: dict[str, str], bcp472opts: dict[str, str], language: str
|
|
) -> dict[str, dict[str, str]]:
|
|
"""Get the bcp47 tags and options matching the language."""
|
|
bcp47tags = {}
|
|
for bcp47tag, name in bcp472lang.items():
|
|
if name == language:
|
|
options = (
|
|
bcp472opts[bcp47tag].lower().split(",")
|
|
if bcp47tag in bcp472opts
|
|
else ()
|
|
)
|
|
bcp47tags[bcp47tag] = {
|
|
s[0]: s[1] for s in map(lambda s: s.split("="), options)
|
|
}
|
|
return bcp47tags
|
|
|
|
|
|
if __name__ == "__main__":
|
|
translations = Path("data/translations")
|
|
translations.mkdir(parents=True, exist_ok=True)
|
|
for translation in translations.rglob("*.yaml"):
|
|
assert read_yaml(translation) == pandoc_parse(
|
|
translation
|
|
), f"Pandoc parsing doesn't match on {translation}."
|
|
|
|
# original repository: https://github.com/latex3/babel/
|
|
babel = git_clone(
|
|
"https://github.com/stephen-huan/babel/", branch="pandoc"
|
|
)
|
|
# original repository: https://github.com/reutenauer/polyglossia/
|
|
polyglossia = git_clone(
|
|
"https://github.com/stephen-huan/polyglossia/", branch="pandoc"
|
|
)
|
|
|
|
babel_data = {}
|
|
babel_keys = set()
|
|
for path in (babel / "locale").glob("*/*.ini"):
|
|
bcp47tag, keys, data = parse_babel(path)
|
|
babel_keys |= keys
|
|
babel_data[bcp47tag] = data
|
|
# print(f"Babel keys: {babel_keys}")
|
|
|
|
# https://docs.python.org/3/library/importlib.html#importing-a-source-file-directly
|
|
spec = spec_from_file_location("bcp47", polyglossia / "tools" / "bcp47.py")
|
|
assert spec is not None and spec.loader is not None, "Can't find bcp47.py."
|
|
bcp47 = module_from_spec(spec)
|
|
sys.modules["bcp47"] = bcp47
|
|
spec.loader.exec_module(bcp47)
|
|
|
|
# check coverage of bcp47.bcp472lang against actual polyglossia/tex/*.ldf
|
|
known = bcp47.bcp472lang.keys()
|
|
assert set(bcp47.babelname2bcp47.values()) <= known, "Some values missing."
|
|
known |= bcp47.babelname2bcp47.keys()
|
|
known |= set(bcp47.bcp472lang.values())
|
|
for path in (polyglossia / "tex").glob("*.ldf"):
|
|
name = path.stem.split("-")[1]
|
|
assert name in known or name == "latex", f"{name} not found."
|
|
|
|
polyglossia_data = {}
|
|
polyglossia_keys = set()
|
|
for name in sorted(set(bcp47.bcp472lang.values())):
|
|
bcp47tags = get_tags(bcp47.bcp472lang, bcp47.bcp472opts, name)
|
|
ldf = parse_ldf(polyglossia / "tex" / f"gloss-{name}.ldf")
|
|
if len(ldf) > 0:
|
|
languages, _, entries = zip(*ldf)
|
|
for data in entries:
|
|
polyglossia_keys |= data.keys()
|
|
assert {name} == set(languages), "Languages don't match."
|
|
for bcp47tag, options in bcp47tags.items():
|
|
polyglossia_data[bcp47tag] = {}
|
|
# go in order of specificity and only apply if more general
|
|
for language, opt, data in sorted(ldf, key=lambda x: len(x[1])):
|
|
if opt.keys() <= options.keys() and all(
|
|
value == options[key] for key, value in opt.items()
|
|
):
|
|
polyglossia_data[bcp47tag] |= data
|
|
# print(f"Polyglossia keys: {polyglossia_keys}")
|
|
|
|
# bcp47tag unique to either Babel or Polyglossia
|
|
unique = babel_data.keys() ^ polyglossia_data.keys()
|
|
shared = babel_data.keys() & polyglossia_data.keys()
|
|
assert (
|
|
unique | shared == babel_data.keys() | polyglossia_data.keys()
|
|
), "Missing keys."
|
|
for bcp47tag in unique:
|
|
data, other_data = (
|
|
(babel_data, polyglossia_data)
|
|
if bcp47tag in babel_data
|
|
else (polyglossia_data, babel_data)
|
|
)
|
|
assert bcp47tag not in other_data, "Shared key."
|
|
captions = data[bcp47tag]
|
|
if len(captions) > 0:
|
|
save_yaml(captions, translations / f"{bcp47tag}.yaml")
|
|
# merge Babel and Polyglossia data
|
|
for bcp47tag in shared:
|
|
data, other_data = (
|
|
(babel_data, polyglossia_data)
|
|
if len(babel_data[bcp47tag]) >= len(polyglossia_data[bcp47tag])
|
|
else (polyglossia_data, babel_data)
|
|
)
|
|
# prefer values from data over other_data
|
|
captions = other_data[bcp47tag] | data[bcp47tag]
|
|
if len(captions) > 0:
|
|
save_yaml(captions, translations / f"{bcp47tag}.yaml")
|
|
|
|
# clean up after ourselves
|
|
rmtree(babel)
|
|
rmtree(polyglossia)
|