mirror of
https://github.com/opendatalab/MinerU.git
synced 2026-10-29 19:07:28 +08:00
refactor: enforce backend and render ownership boundaries
This commit is contained in:
+1
-1
@@ -8,7 +8,7 @@ import httpx
|
||||
|
||||
from mineru.cli_old import api_client as _api_client
|
||||
from mineru.cli_old.common import image_suffixes, office_suffixes, pdf_suffixes
|
||||
from mineru.utils.guess_suffix_or_lang import guess_suffix_by_path
|
||||
from mineru.parser.file_type import guess_suffix_by_path
|
||||
|
||||
SUPPORTED_INPUT_SUFFIXES = set(pdf_suffixes + image_suffixes + office_suffixes)
|
||||
|
||||
|
||||
@@ -1 +1,3 @@
|
||||
# Copyright (c) Opendatalab. All rights reserved.
|
||||
|
||||
__all__: list[str] = []
|
||||
|
||||
@@ -1,2 +1,4 @@
|
||||
# Copyright (c) Opendatalab. All rights reserved.
|
||||
"""文档 model-list 生产层。"""
|
||||
|
||||
__all__: list[str] = []
|
||||
|
||||
@@ -6,7 +6,7 @@ from __future__ import annotations
|
||||
import time
|
||||
from io import BytesIO
|
||||
|
||||
from mineru.model.flash import DocModel, DocxModel, PptModel, PptxModel, XlsModel, XlsxModel
|
||||
from ...model.flash import DocModel, DocxModel, PptModel, PptxModel, XlsModel, XlsxModel
|
||||
|
||||
from .contracts import AnalysisResult, OfficeSuffix
|
||||
|
||||
|
||||
@@ -1,2 +1,4 @@
|
||||
# Copyright (c) Opendatalab. All rights reserved.
|
||||
"""PDF 分析流水线。"""
|
||||
|
||||
__all__: list[str] = []
|
||||
|
||||
@@ -5,7 +5,7 @@ from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from mineru.types import RAW_ALGORITHM, RAW_CAPTION, RAW_FOOTNOTE, RAW_FORMULA_NUMBER, RAW_PHONETIC, BlockType
|
||||
from ....types import RAW_ALGORITHM, RAW_CAPTION, RAW_FOOTNOTE, RAW_FORMULA_NUMBER, RAW_PHONETIC, BlockType
|
||||
|
||||
LAYOUT_BASE_BATCH_SIZE = 1
|
||||
MFR_BASE_BATCH_SIZE = 16
|
||||
|
||||
@@ -10,10 +10,10 @@ import cv2
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
|
||||
from mineru.backend.local_model_runtime import HybridLocalModelContext
|
||||
from mineru.utils.bbox_utils import normalize_to_int_bbox
|
||||
from mineru.utils.ocr_utils import get_rotate_crop_image_for_text_rec
|
||||
from mineru.utils.text_utils import build_tagged_formula_content
|
||||
from ....model.runtime.hybrid import HybridLocalModelContext
|
||||
from ....utils.geometry import normalize_to_int_bbox
|
||||
from ....model.ocr.image import get_rotate_crop_image_for_text_rec
|
||||
from ....utils.text import build_tagged_formula_content
|
||||
|
||||
from ....types import RAW_FORMULA_NUMBER, BlockType
|
||||
|
||||
|
||||
@@ -10,8 +10,8 @@ import cv2
|
||||
import numpy as np
|
||||
from loguru import logger
|
||||
|
||||
from mineru.types import BBox
|
||||
from mineru.utils.bbox_utils import normalize_to_int_bbox
|
||||
from ....types import BBox
|
||||
from ....utils.geometry import normalize_to_int_bbox
|
||||
|
||||
|
||||
def _normalize_page_size(page_image: Any) -> tuple[int, int]:
|
||||
|
||||
@@ -8,21 +8,49 @@ import threading
|
||||
import time
|
||||
from concurrent.futures import ALL_COMPLETED, Future, ProcessPoolExecutor, wait
|
||||
from concurrent.futures.process import BrokenProcessPool
|
||||
from io import BytesIO
|
||||
import math
|
||||
from typing import Any, Callable, Literal
|
||||
|
||||
import numpy as np
|
||||
import pypdfium2 as pdfium
|
||||
from loguru import logger
|
||||
from PIL import Image, ImageOps
|
||||
from PIL import Image
|
||||
|
||||
from ....model.flash.pdf.pdfium import close_pdfium_child, pdfium_guard
|
||||
from ....model.flash.pdf.raster import image_to_b64str, page_to_image
|
||||
from ....types import BBox, IntBBox
|
||||
from ....utils.geometry import normalize_to_int_bbox
|
||||
from ....utils.platform import is_windows_environment
|
||||
|
||||
|
||||
class ImageType:
|
||||
"""限定 PDF 页面渲染支持的两种图像返回形态。"""
|
||||
|
||||
PIL = "pil_img"
|
||||
BASE64 = "base64_img"
|
||||
|
||||
|
||||
def _positive_int_env(name: str, default: int) -> int:
|
||||
"""读取正整数环境变量,缺失或非法时返回默认值。"""
|
||||
raw_value = os.getenv(name)
|
||||
if raw_value is None:
|
||||
return default
|
||||
try:
|
||||
value = int(raw_value)
|
||||
except ValueError:
|
||||
return default
|
||||
return value if value > 0 else default
|
||||
|
||||
|
||||
def get_load_images_timeout() -> int:
|
||||
"""返回 PDF 批量渲染的超时秒数。"""
|
||||
return _positive_int_env("MINERU_PDF_RENDER_TIMEOUT", 300)
|
||||
|
||||
|
||||
def get_load_images_threads() -> int:
|
||||
"""返回 PDF 批量渲染允许使用的进程数。"""
|
||||
return _positive_int_env("MINERU_PDF_RENDER_THREADS", 3)
|
||||
|
||||
from ..types import BBox
|
||||
from .bbox_utils import normalize_to_int_bbox
|
||||
from .check_sys_env import is_windows_environment
|
||||
from .enum_class import ImageType
|
||||
from .os_env_config import get_load_images_threads, get_load_images_timeout
|
||||
from .pdf_reader import image_to_b64str, page_to_image
|
||||
from .pdfium_guard import close_pdfium_child, pdfium_guard
|
||||
|
||||
DEFAULT_PDF_IMAGE_DPI = 200
|
||||
# DEFAULT_PDF_IMAGE_DPI = 144
|
||||
@@ -34,6 +62,7 @@ PDF_RENDER_KILL_JOIN_TIMEOUT_SECONDS = 0.1
|
||||
|
||||
_pdf_render_executor: ProcessPoolExecutor | None = None
|
||||
_pdf_render_executor_lock = threading.Lock()
|
||||
_pdf_render_atexit_registered = False
|
||||
_pdf_render_spawn_submit_lock = threading.Lock()
|
||||
_pdf_render_spawn_submit_executor_id: int | None = None
|
||||
_pdf_render_spawn_submit_count = 0
|
||||
@@ -44,16 +73,7 @@ def pdf_page_to_image(
|
||||
dpi: int = DEFAULT_PDF_IMAGE_DPI,
|
||||
image_type: Literal["pil_img", "base64_img"] = ImageType.PIL,
|
||||
) -> dict[str, Any]:
|
||||
"""Convert pdfium.PdfDocument to image, Then convert the image to base64.
|
||||
|
||||
Args:
|
||||
page (_type_): pdfium.PdfPage
|
||||
dpi (int, optional): reset the dpi of dpi. Defaults to DEFAULT_PDF_IMAGE_DPI.
|
||||
image_type (ImageType, optional): The type of image to return. Defaults to ImageType.PIL.
|
||||
|
||||
Returns:
|
||||
dict: {'img_base64': str, 'img_pil': pil_img, 'scale': float }
|
||||
"""
|
||||
"""将单个 PDFium 页面渲染为 Pillow 图像或 base64 载荷。"""
|
||||
pil_img, scale = page_to_image(page, dpi=dpi)
|
||||
image_dict: dict[str, Any] = {"scale": scale}
|
||||
if image_type == ImageType.BASE64:
|
||||
@@ -109,6 +129,7 @@ def _close_image_dicts(images_list: list[dict[str, Any]] | None) -> None:
|
||||
|
||||
|
||||
def _calculate_render_process_count(total_pages: int, threads: int, cpu_count: int | None = None) -> int:
|
||||
"""按页数、配置和 CPU 数量计算实际渲染进程数。"""
|
||||
requested_threads = max(1, threads)
|
||||
available_cpus = max(1, cpu_count if cpu_count is not None else (os.cpu_count() or 1))
|
||||
page_limited_threads = max(1, total_pages // MIN_PAGES_PER_RENDER_PROCESS)
|
||||
@@ -125,6 +146,7 @@ def _build_render_page_ranges(
|
||||
end_page_id: int,
|
||||
process_count: int,
|
||||
) -> list[tuple[int, int]]:
|
||||
"""把闭区间页范围均匀拆分为指定数量的子范围。"""
|
||||
total_pages = end_page_id - start_page_id + 1
|
||||
base_pages, remainder = divmod(total_pages, process_count)
|
||||
page_ranges = []
|
||||
@@ -145,12 +167,14 @@ def _get_render_process_plan(
|
||||
threads: int,
|
||||
cpu_count: int | None = None,
|
||||
) -> tuple[int, list[tuple[int, int]]]:
|
||||
"""同时返回实际进程数及其页面分片计划。"""
|
||||
total_pages = end_page_id - start_page_id + 1
|
||||
actual_threads = _calculate_render_process_count(total_pages, threads, cpu_count)
|
||||
return actual_threads, _build_render_page_ranges(start_page_id, end_page_id, actual_threads)
|
||||
|
||||
|
||||
def _get_pdf_render_pool_capacity(cpu_count: int | None = None) -> int:
|
||||
"""返回持久 PDF 渲染进程池的最大容量。"""
|
||||
available_cpus = max(1, cpu_count if cpu_count is not None else (os.cpu_count() or 1))
|
||||
configured_threads = max(1, get_load_images_threads())
|
||||
return min(
|
||||
@@ -161,6 +185,7 @@ def _get_pdf_render_pool_capacity(cpu_count: int | None = None) -> int:
|
||||
|
||||
|
||||
def _exit_pdf_render_worker_when_parent_exits() -> None:
|
||||
"""等待父进程退出后立即终止孤立渲染 worker。"""
|
||||
parent = multiprocessing.parent_process()
|
||||
if parent is None:
|
||||
return
|
||||
@@ -169,6 +194,7 @@ def _exit_pdf_render_worker_when_parent_exits() -> None:
|
||||
|
||||
|
||||
def _install_pdf_render_parent_exit_watcher() -> None:
|
||||
"""在渲染 worker 中安装父进程退出监听线程。"""
|
||||
watcher = threading.Thread(
|
||||
target=_exit_pdf_render_worker_when_parent_exits,
|
||||
name="mineru-pdf-render-parent-exit-watcher",
|
||||
@@ -178,6 +204,7 @@ def _install_pdf_render_parent_exit_watcher() -> None:
|
||||
|
||||
|
||||
def _create_pdf_render_executor(max_workers: int) -> ProcessPoolExecutor:
|
||||
"""使用安全 multiprocessing 上下文创建 PDF 渲染进程池。"""
|
||||
if is_windows_environment():
|
||||
return ProcessPoolExecutor(max_workers=max_workers, initializer=_install_pdf_render_parent_exit_watcher)
|
||||
|
||||
@@ -234,6 +261,7 @@ def _submit_pdf_render_task(
|
||||
def _get_pdf_render_future_states(
|
||||
future_to_range: dict[Future[Any], tuple[int, int]],
|
||||
) -> list[dict[str, Any]]:
|
||||
"""汇总每个渲染 Future 的页面范围和运行状态。"""
|
||||
states = []
|
||||
for future, (range_start, range_end) in future_to_range.items():
|
||||
try:
|
||||
@@ -257,6 +285,7 @@ def _get_pdf_render_future_states(
|
||||
|
||||
|
||||
def _get_pdf_render_worker_states(executor: ProcessPoolExecutor) -> list[dict[str, Any]]:
|
||||
"""读取进程池 worker 的 pid、存活状态和退出码。"""
|
||||
try:
|
||||
process_map = getattr(executor, "_processes", None) or {}
|
||||
processes = list(process_map.values())
|
||||
@@ -288,10 +317,14 @@ def _get_pdf_render_worker_states(executor: ProcessPoolExecutor) -> list[dict[st
|
||||
|
||||
|
||||
def _get_pdf_render_executor() -> ProcessPoolExecutor:
|
||||
global _pdf_render_executor
|
||||
"""惰性创建并复用持久 PDF 渲染进程池。"""
|
||||
global _pdf_render_atexit_registered, _pdf_render_executor
|
||||
|
||||
with _pdf_render_executor_lock:
|
||||
if _pdf_render_executor is None:
|
||||
if not _pdf_render_atexit_registered:
|
||||
atexit.register(shutdown_pdf_render_executor)
|
||||
_pdf_render_atexit_registered = True
|
||||
max_workers = _get_pdf_render_pool_capacity()
|
||||
_pdf_render_executor = _create_pdf_render_executor(max_workers=max_workers)
|
||||
logger.debug(f"Created persistent PDF render executor with max_workers={max_workers}")
|
||||
@@ -303,6 +336,7 @@ def _recycle_pdf_render_executor(
|
||||
*,
|
||||
terminate_processes: bool,
|
||||
) -> None:
|
||||
"""从全局缓存移除指定进程池并按需终止 worker。"""
|
||||
global _pdf_render_executor
|
||||
|
||||
if executor is None:
|
||||
@@ -324,6 +358,7 @@ def _recycle_pdf_render_executor(
|
||||
|
||||
|
||||
def shutdown_pdf_render_executor() -> None:
|
||||
"""关闭当前持久 PDF 渲染进程池。"""
|
||||
global _pdf_render_executor
|
||||
|
||||
with _pdf_render_executor_lock:
|
||||
@@ -337,9 +372,6 @@ def shutdown_pdf_render_executor() -> None:
|
||||
)
|
||||
|
||||
|
||||
atexit.register(shutdown_pdf_render_executor)
|
||||
|
||||
|
||||
def load_images_from_pdf_bytes_range(
|
||||
pdf_bytes: bytes,
|
||||
dpi: int = DEFAULT_PDF_IMAGE_DPI,
|
||||
@@ -349,6 +381,7 @@ def load_images_from_pdf_bytes_range(
|
||||
timeout: int | None = None,
|
||||
threads: int | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""在持久进程池中批量渲染指定闭区间页面。"""
|
||||
if end_page_id < start_page_id:
|
||||
return []
|
||||
|
||||
@@ -500,6 +533,7 @@ def load_images_from_pdf_core(
|
||||
end_page_id: int | None = None,
|
||||
image_type: Literal["pil_img", "base64_img"] = ImageType.PIL,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""在当前进程中依次渲染 PDF 页面范围。"""
|
||||
images_list = []
|
||||
pdf_doc = None
|
||||
try:
|
||||
@@ -527,7 +561,50 @@ def load_images_from_pdf_core(
|
||||
return images_list
|
||||
|
||||
|
||||
def _model_input_bbox(item: dict[str, Any]) -> IntBBox:
|
||||
"""把模型输入项的 bbox 向外取整为裁图整数坐标。"""
|
||||
bbox = item["bbox"]
|
||||
assert bbox is not None
|
||||
xmin, ymin, xmax, ymax = [float(value) for value in bbox]
|
||||
return math.floor(xmin), math.floor(ymin), math.ceil(xmax), math.ceil(ymax)
|
||||
|
||||
|
||||
def crop_img(
|
||||
input_res: dict[str, Any],
|
||||
input_img: Image.Image | np.ndarray,
|
||||
crop_paste_x: int = 0,
|
||||
crop_paste_y: int = 0,
|
||||
) -> tuple[Image.Image | np.ndarray, list[int]]:
|
||||
"""按模型 bbox 裁图并在四周补白,返回坐标回投所需参数。"""
|
||||
crop_xmin, crop_ymin, crop_xmax, crop_ymax = _model_input_bbox(input_res)
|
||||
crop_new_width = crop_xmax - crop_xmin + crop_paste_x * 2
|
||||
crop_new_height = crop_ymax - crop_ymin + crop_paste_y * 2
|
||||
if isinstance(input_img, np.ndarray):
|
||||
return_image: Image.Image | np.ndarray = np.ones((crop_new_height, crop_new_width, 3), dtype=np.uint8) * 255
|
||||
cropped_img = input_img[crop_ymin:crop_ymax, crop_xmin:crop_xmax]
|
||||
return_image[
|
||||
crop_paste_y : crop_paste_y + (crop_ymax - crop_ymin),
|
||||
crop_paste_x : crop_paste_x + (crop_xmax - crop_xmin),
|
||||
] = cropped_img
|
||||
else:
|
||||
return_image = Image.new("RGB", (crop_new_width, crop_new_height), "white")
|
||||
cropped_img = input_img.crop((crop_xmin, crop_ymin, crop_xmax, crop_ymax))
|
||||
return_image.paste(cropped_img, (crop_paste_x, crop_paste_y))
|
||||
useful_list = [
|
||||
crop_paste_x,
|
||||
crop_paste_y,
|
||||
crop_xmin,
|
||||
crop_ymin,
|
||||
crop_xmax,
|
||||
crop_ymax,
|
||||
crop_new_width,
|
||||
crop_new_height,
|
||||
]
|
||||
return return_image, useful_list
|
||||
|
||||
|
||||
def get_crop_img(bbox: BBox, pil_img: Image.Image, scale: float = 2.0) -> Image.Image:
|
||||
"""按缩放 bbox 裁剪 Pillow 图像。"""
|
||||
scale_bbox = normalize_to_int_bbox([float(v) * scale for v in bbox])
|
||||
if scale_bbox is None:
|
||||
return pil_img.crop((0, 0, 0, 0))
|
||||
@@ -535,6 +612,7 @@ def get_crop_img(bbox: BBox, pil_img: Image.Image, scale: float = 2.0) -> Image.
|
||||
|
||||
|
||||
def get_crop_np_img(bbox: BBox, input_img: Image.Image | np.ndarray, scale: float = 2.0) -> np.ndarray:
|
||||
"""按缩放 bbox 裁剪 Pillow 或 NumPy 图像并返回数组。"""
|
||||
if isinstance(input_img, Image.Image):
|
||||
np_img = np.asarray(input_img)
|
||||
elif isinstance(input_img, np.ndarray):
|
||||
@@ -553,28 +631,13 @@ def get_crop_np_img(bbox: BBox, input_img: Image.Image | np.ndarray, scale: floa
|
||||
return np_img[scale_bbox[1] : scale_bbox[3], scale_bbox[0] : scale_bbox[2]]
|
||||
|
||||
|
||||
def images_bytes_to_pdf_bytes(image_bytes: bytes) -> bytes:
|
||||
# 内存缓冲区
|
||||
pdf_buffer = BytesIO()
|
||||
|
||||
# 载入并转换所有图像为 RGB 模式
|
||||
image = Image.open(BytesIO(image_bytes))
|
||||
# 根据 EXIF 信息自动转正(处理手机拍摄的带 Orientation 标记的图片)
|
||||
image = ImageOps.exif_transpose(image) or image
|
||||
# 只在必要时转换
|
||||
if image.mode != "RGB":
|
||||
image = image.convert("RGB")
|
||||
|
||||
# 第一张图保存为 PDF,其余追加
|
||||
image.save(
|
||||
pdf_buffer,
|
||||
format="PDF",
|
||||
resolution=DEFAULT_PDF_IMAGE_DPI,
|
||||
quality=95,
|
||||
subsampling=0,
|
||||
)
|
||||
|
||||
# 获取 PDF bytes 并重置指针(可选)
|
||||
pdf_bytes = pdf_buffer.getvalue()
|
||||
pdf_buffer.close()
|
||||
return pdf_bytes
|
||||
__all__ = [
|
||||
"ImageType",
|
||||
"crop_img",
|
||||
"get_crop_img",
|
||||
"get_crop_np_img",
|
||||
"load_images_from_pdf_bytes_range",
|
||||
"load_images_from_pdf_core",
|
||||
"pdf_page_to_image",
|
||||
"shutdown_pdf_render_executor",
|
||||
]
|
||||
@@ -10,9 +10,9 @@ from typing import Any
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
|
||||
from mineru.types import BBox, BlockType
|
||||
from mineru.utils.bbox_utils import normalize_to_int_bbox
|
||||
from mineru.utils.pdf_image_tools import get_crop_np_img
|
||||
from ....types import BBox, BlockType
|
||||
from ....utils.geometry import normalize_to_int_bbox
|
||||
from .images import get_crop_np_img
|
||||
|
||||
from .geometry import _normalize_layout_bbox_to_unit, _normalize_page_size
|
||||
from .constants import (
|
||||
@@ -24,7 +24,7 @@ from .constants import (
|
||||
|
||||
def _load_vlm_runtime() -> dict[str, Any]:
|
||||
"""按需加载 VLM runtime 组件,确保只有 high/extra_high 路径触发 VLM 依赖。"""
|
||||
from mineru.model.vlm.runtime import (
|
||||
from ....model.vlm.runtime import (
|
||||
ModelSingleton,
|
||||
_get_model_async,
|
||||
_maybe_enable_serial_execution,
|
||||
|
||||
@@ -8,9 +8,9 @@ import re
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from mineru.types import BBox, BlockType, RAW_PHONETIC
|
||||
from mineru.utils.bbox_utils import calculate_overlap_area_2_minbox_area_ratio
|
||||
from mineru.utils.text_utils import full_to_half_exclude_marks
|
||||
from ....types import BBox, BlockType, RAW_PHONETIC
|
||||
from ....utils.geometry import calculate_overlap_area_2_minbox_area_ratio
|
||||
from ....utils.text import full_to_half_exclude_marks
|
||||
|
||||
from .constants import (
|
||||
LAYOUT_TITLE_SPLIT_OVERLAP_THRESHOLD,
|
||||
|
||||
@@ -10,19 +10,13 @@ import cv2
|
||||
import numpy as np
|
||||
from tqdm import tqdm
|
||||
|
||||
from mineru.backend.local_model_runtime import HybridLocalModelContext
|
||||
from mineru.types import BlockType
|
||||
from mineru.utils.bbox_utils import normalize_to_int_bbox
|
||||
from mineru.utils.model_utils import crop_img
|
||||
from mineru.utils.ocr_utils import (
|
||||
OcrConfidence,
|
||||
get_adjusted_mfdetrec_res,
|
||||
get_ocr_result_list,
|
||||
mask_formula_regions_for_ocr_det,
|
||||
merge_det_boxes,
|
||||
sorted_boxes,
|
||||
update_det_boxes,
|
||||
)
|
||||
from ....model.runtime.hybrid import HybridLocalModelContext
|
||||
from ....types import BlockType
|
||||
from ....utils.geometry import normalize_to_int_bbox
|
||||
from .images import crop_img
|
||||
from ....model.ocr.geometry import merge_det_boxes, sorted_boxes, update_det_boxes
|
||||
from ....model.ocr.image import mask_formula_regions_for_ocr_det
|
||||
from ....model.ocr.results import OcrConfidence, get_adjusted_mfdetrec_res, get_ocr_result_list
|
||||
|
||||
from .constants import (
|
||||
BATCH_RATIO,
|
||||
|
||||
@@ -3,22 +3,19 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import time
|
||||
from typing import Any, cast
|
||||
|
||||
from mineru.backend.analysis.contracts import AnalysisResult, AnalyzeEffort, ParseMode, ResolvedParseMode
|
||||
from mineru.backend.local_model_runtime import HybridLocalModelContextSingleton
|
||||
from mineru.utils.engine_utils import get_vlm_engine
|
||||
from mineru.utils.model_utils import clean_memory
|
||||
from mineru.utils.pdf_document import PDFDocument
|
||||
from ..contracts import AnalysisResult, AnalyzeEffort, ParseMode, ResolvedParseMode
|
||||
from ....model.runtime.hybrid import HybridLocalModelContextSingleton
|
||||
from ....model.runtime.memory import clean_memory
|
||||
from ....model.vlm.selector import get_vlm_engine
|
||||
from ....model.flash.pdf.document import PDFDocument
|
||||
|
||||
from .layout import _load_vlm_runtime
|
||||
from .normalization import _normalize_pdf_model_list
|
||||
from .window import process_pdf_windows
|
||||
|
||||
os.environ["PYTORCH_ENABLE_MPS_FALLBACK"] = "1"
|
||||
|
||||
_SUPPORTED_PDF_EFFORTS = {"flash", "medium", "high", "xhigh"}
|
||||
|
||||
|
||||
@@ -29,6 +26,10 @@ def analyze_pdf(
|
||||
image_analysis: bool = True,
|
||||
) -> AnalysisResult:
|
||||
"""生产 PDF model-list,并返回最终路由元数据和精确推理耗时。"""
|
||||
# 只在真实 PDF 分析开始时配置 MPS 回退,避免 import backend 修改进程环境。
|
||||
import os
|
||||
|
||||
os.environ.setdefault("PYTORCH_ENABLE_MPS_FALLBACK", "1")
|
||||
if effort not in _SUPPORTED_PDF_EFFORTS:
|
||||
raise ValueError(f"Unsupported analyze effort: {effort}")
|
||||
|
||||
|
||||
@@ -13,22 +13,22 @@ import cv2
|
||||
import numpy as np
|
||||
from loguru import logger
|
||||
|
||||
from ...local_model_runtime import HybridLocalModelContext, run_ocr_inference
|
||||
from ....model.model_types import AtomicModelName
|
||||
from ....model.runtime.hybrid import HybridLocalModelContext, run_ocr_inference
|
||||
from ....model.runtime.contracts import AtomicModelName
|
||||
from ....types import RAW_ALGORITHM, RAW_FORMULA_NUMBER, RAW_PHONETIC, BBox, BlockType
|
||||
from ....utils.bbox_utils import (
|
||||
from ....utils.geometry import (
|
||||
calculate_overlap_area_in_bbox1_area_ratio,
|
||||
normalize_to_int_bbox,
|
||||
)
|
||||
from ....utils.native_pdf_table import (
|
||||
from ....model.flash.pdf.table_recovery import (
|
||||
NativeTableInput,
|
||||
coerce_native_table_rectangles,
|
||||
coerce_native_table_rules,
|
||||
recover_native_pdf_table,
|
||||
)
|
||||
from ....utils.ocr_utils import mask_formula_regions_for_ocr_det
|
||||
from ....utils.pdf_document import PDFPage, get_lines_from_chars
|
||||
from ....utils.spatial_text import project_ocr_table_text
|
||||
from ....model.ocr.image import mask_formula_regions_for_ocr_det
|
||||
from ....model.flash.pdf.document import PDFPage, get_lines_from_chars
|
||||
from ....model.flash.pdf.spatial_text import project_ocr_table_text
|
||||
|
||||
from .constants import (
|
||||
BATCH_RATIO,
|
||||
|
||||
@@ -1,2 +1,4 @@
|
||||
# Copyright (c) Opendatalab. All rights reserved.
|
||||
"""PDF 原生文本、OCR Span 和内容重建。"""
|
||||
|
||||
__all__: list[str] = []
|
||||
|
||||
@@ -9,18 +9,19 @@ import numpy as np
|
||||
from PIL import Image
|
||||
from pdftext.schema import Char
|
||||
|
||||
from mineru.backend.local_model_runtime import HybridLocalModelContext, run_ocr_inference
|
||||
from mineru.types import BBox, BlockType, ContentType
|
||||
from mineru.utils.language import detect_lang
|
||||
from mineru.utils.ocr_utils import OcrConfidence, rotate_vertical_crop_if_needed
|
||||
from mineru.utils.pdf_document import PDFPage, get_lines_from_chars
|
||||
from mineru.utils.pdf_text_styles import (
|
||||
from .....model.runtime.hybrid import HybridLocalModelContext, run_ocr_inference
|
||||
from .....types import BBox, BlockType, ContentType
|
||||
from .....utils.language import detect_lang
|
||||
from .....model.ocr.image import rotate_vertical_crop_if_needed
|
||||
from .....model.ocr.results import OcrConfidence
|
||||
from .....model.flash.pdf.document import PDFPage, get_lines_from_chars
|
||||
from .....model.flash.pdf.text_styles import (
|
||||
PDFTextLinkLine,
|
||||
PDFTextStyleLine,
|
||||
apply_pdf_text_links,
|
||||
apply_pdf_text_styles,
|
||||
)
|
||||
from mineru.utils.text_utils import merge_text_line_contents
|
||||
from .....utils.text import merge_text_line_contents
|
||||
|
||||
from ..constants import (
|
||||
CODE_CONTENT_BLOCK_TYPES,
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from .....types import ContentType
|
||||
from .....utils.ocr_utils import _is_overlaps_x_exceeds_threshold, _is_overlaps_y_exceeds_threshold
|
||||
from .....model.ocr.geometry import _is_overlaps_x_exceeds_threshold, _is_overlaps_y_exceeds_threshold
|
||||
from .models import _AnalyzeLine, _AnalyzeSpan
|
||||
|
||||
VERTICAL_SPAN_HEIGHT_TO_WIDTH_RATIO_THRESHOLD = 2
|
||||
|
||||
@@ -17,10 +17,10 @@ from loguru import logger
|
||||
from pdftext.schema import Char
|
||||
|
||||
from .....types import BBox, BlockType, ContentType
|
||||
from .....utils.image_utils import calculate_contrast
|
||||
from .....utils.pdf_document import PDFPage, get_lines_from_chars
|
||||
from .....utils.pdf_image_tools import get_crop_img
|
||||
from mineru.utils.bbox_utils import calculate_overlap_area_in_bbox1_area_ratio
|
||||
from .....utils.image import calculate_contrast
|
||||
from .....model.flash.pdf.document import PDFPage, get_lines_from_chars
|
||||
from ..images import get_crop_img
|
||||
from .....utils.geometry import calculate_overlap_area_in_bbox1_area_ratio
|
||||
from .models import _AnalyzeSpan
|
||||
|
||||
MAX_NATIVE_TEXT_CHARS_PER_PAGE = 65535
|
||||
|
||||
@@ -7,8 +7,8 @@ from typing import Any, Sequence
|
||||
|
||||
from pdftext.schema import Char
|
||||
|
||||
from mineru.utils.pdf_document import PDFPage, get_lines_from_chars
|
||||
from mineru.utils.pdf_text_styles import (
|
||||
from .....model.flash.pdf.document import PDFPage, get_lines_from_chars
|
||||
from .....model.flash.pdf.text_styles import (
|
||||
PDFTextLinkLine,
|
||||
PDFTextStyleLine,
|
||||
detect_pdf_text_link_lines,
|
||||
@@ -30,7 +30,7 @@ def build_pdf_native_visual_lines_and_styles(
|
||||
"""一次读取当前页字符,构造视觉 run、文本样式和超链接证据。"""
|
||||
|
||||
# 延迟导入避免 Hybrid 模块初始化时提前加载完整 Flash PDF 流水线。
|
||||
from mineru.model.flash.native_pdf.native_text import _build_native_line_items
|
||||
from .....model.flash.pdf.native_text import _build_native_line_items
|
||||
|
||||
chars = page_chars if page_chars is not None else pdf_page.get_chars()
|
||||
line_items = _build_native_line_items(
|
||||
|
||||
@@ -9,8 +9,8 @@ from typing import Any
|
||||
import numpy as np
|
||||
from loguru import logger
|
||||
|
||||
from mineru.types import BBox, BlockType
|
||||
from mineru.utils.bbox_utils import (
|
||||
from ....types import BBox, BlockType
|
||||
from ....utils.geometry import (
|
||||
calculate_overlap_area_2_minbox_area_ratio,
|
||||
calculate_overlap_area_in_bbox1_area_ratio,
|
||||
)
|
||||
|
||||
@@ -9,10 +9,9 @@ from typing import Any, Literal
|
||||
import numpy as np
|
||||
from loguru import logger
|
||||
|
||||
from mineru.backend.local_model_runtime import HybridLocalModelContext
|
||||
from mineru.utils.config_reader import get_processing_window_size
|
||||
from mineru.utils.pdf_document import PDFDocument, PDFPage
|
||||
from mineru.utils.pdf_image_tools import load_images_from_pdf_bytes_range
|
||||
from ....model.runtime.hybrid import HybridLocalModelContext
|
||||
from ....model.flash.pdf.document import PDFDocument, PDFPage
|
||||
from .images import load_images_from_pdf_bytes_range
|
||||
|
||||
from ..contracts import AnalyzeEffort
|
||||
from .constants import (
|
||||
@@ -61,6 +60,20 @@ from .text.content import (
|
||||
)
|
||||
|
||||
|
||||
def _configured_window_size(default: int = 64) -> int:
|
||||
"""读取 PDF 分析窗口大小,非法配置回退到正整数默认值。"""
|
||||
import os
|
||||
|
||||
raw_value = os.getenv("MINERU_PROCESSING_WINDOW_SIZE")
|
||||
if raw_value is None:
|
||||
return default
|
||||
try:
|
||||
return max(1, int(raw_value))
|
||||
except ValueError:
|
||||
logger.warning(f"Invalid MINERU_PROCESSING_WINDOW_SIZE value: {raw_value}, use default {default}")
|
||||
return default
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _ProcessingWindow:
|
||||
"""记录单个 Hybrid 处理窗口的页码范围。"""
|
||||
@@ -278,11 +291,11 @@ def process_pdf_windows(
|
||||
model_list: list[list[dict[str, Any]]] = []
|
||||
if flash_txt_mode:
|
||||
# Flash 先对整份 PDF 生成完整 model_list,不依赖页面渲染和处理窗口。
|
||||
from mineru.model.flash import PdfModel
|
||||
from ....model.flash import PdfModel
|
||||
|
||||
model_list = PdfModel().predict(document)
|
||||
|
||||
configured_window_size = get_processing_window_size(default=64)
|
||||
configured_window_size = _configured_window_size(default=64)
|
||||
windows = _build_processing_windows(page_count, configured_window_size)
|
||||
_log_processing_window_plan(page_count, configured_window_size, len(windows))
|
||||
|
||||
@@ -362,9 +375,7 @@ def process_pdf_windows(
|
||||
if effort == "medium":
|
||||
window_model_list = vl_style_layout_blocks
|
||||
elif effort == "high":
|
||||
high_vlm_blocks, accepted_native_tables = _split_native_high_table_blocks(
|
||||
vl_style_layout_blocks
|
||||
)
|
||||
high_vlm_blocks, accepted_native_tables = _split_native_high_table_blocks(vl_style_layout_blocks)
|
||||
high_vlm_results = vlm_predictor.batch_extract_with_layout(
|
||||
images=images_pil_list,
|
||||
blocks_list=high_vlm_blocks,
|
||||
|
||||
@@ -8,13 +8,10 @@ from typing import cast
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from mineru.backend.analysis.contracts import AnalyzeEffort, OfficeSuffix, ParseMode
|
||||
from mineru.backend.analysis.office import analyze_office
|
||||
from mineru.backend.analysis.pdf.pipeline import analyze_pdf
|
||||
from mineru.backend.postprocess.document import model_json_to_middle_json
|
||||
from mineru.config import config
|
||||
from mineru.types import FILE_SUFFIXES, FileSuffix, MiddleJson, ModelJson
|
||||
from mineru.version import __version__ as mineru_version
|
||||
from .analysis.contracts import AnalyzeEffort, OfficeSuffix, ParseMode
|
||||
from ..config import config
|
||||
from ..types import FILE_SUFFIXES, FileSuffix, MiddleJson, ModelJson
|
||||
from ..version import __version__ as mineru_version
|
||||
|
||||
_SUPPORTED_ANALYZE_EFFORTS = {"flash", "medium", "high", "xhigh"}
|
||||
|
||||
@@ -43,6 +40,8 @@ def doc_analyze(
|
||||
raise ValueError(f"Unsupported analyze effort: {effort}")
|
||||
|
||||
if file_suffix == "pdf":
|
||||
from .analysis.pdf.pipeline import analyze_pdf
|
||||
|
||||
result = analyze_pdf(
|
||||
file_bytes,
|
||||
effort=effort,
|
||||
@@ -50,6 +49,8 @@ def doc_analyze(
|
||||
image_analysis=image_analysis,
|
||||
)
|
||||
else:
|
||||
from .analysis.office import analyze_office
|
||||
|
||||
result = analyze_office(file_bytes, cast(OfficeSuffix, file_suffix))
|
||||
|
||||
_log_infer_performance(file_suffix, len(result.model_list), result.elapsed)
|
||||
@@ -61,6 +62,8 @@ def doc_analyze(
|
||||
parse_mode=result.parse_mode,
|
||||
mineru_version=mineru_version,
|
||||
)
|
||||
from .postprocess.document import model_json_to_middle_json
|
||||
|
||||
middle_json = model_json_to_middle_json(
|
||||
model_json,
|
||||
llm_aided_config=config.llm_aided,
|
||||
|
||||
@@ -1,2 +1,4 @@
|
||||
# Copyright (c) Opendatalab. All rights reserved.
|
||||
"""将严格 ModelJson 转换并增强为 MiddleJson 的后处理层。"""
|
||||
|
||||
__all__: list[str] = []
|
||||
|
||||
@@ -3,8 +3,8 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from mineru.config import LLMAidedConfig
|
||||
from mineru.types import MiddleJson, ModelJson
|
||||
from ...config import LLMAidedConfig
|
||||
from ...types import MiddleJson, ModelJson
|
||||
|
||||
from .llm_aided import apply_llm_aided_postprocess
|
||||
from .pages import model_json_to_pages
|
||||
|
||||
@@ -8,8 +8,8 @@ import re
|
||||
from dataclasses import dataclass
|
||||
from typing import TypeAlias, Union
|
||||
|
||||
from mineru.utils.language import detect_lang
|
||||
from mineru.utils.text_utils import CJK_LANGS, resolve_text_line_boundary
|
||||
from ...utils.language import detect_lang
|
||||
from ...utils.text import CJK_LANGS, resolve_text_line_boundary
|
||||
|
||||
_INLINE_START_RE = re.compile(
|
||||
r"<(?P<tag>eq|text|hyperlink|sup|sub|strong|b|em|i|s|u)(?P<attrs>\s[^<>]*?)?>",
|
||||
@@ -6,8 +6,8 @@ from __future__ import annotations
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from mineru.types import BlockType
|
||||
from mineru.utils.bbox_utils import calculate_overlap_area_in_bbox1_area_ratio
|
||||
from ...types import BlockType
|
||||
from ...utils.geometry import calculate_overlap_area_in_bbox1_area_ratio
|
||||
|
||||
from .visual import _bbox_for_calculation
|
||||
|
||||
|
||||
@@ -7,8 +7,8 @@ import asyncio
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from mineru.config import LLMAidedConfig
|
||||
from mineru.types import MiddleJson
|
||||
from ...config import LLMAidedConfig
|
||||
from ...types import MiddleJson
|
||||
|
||||
from .llm_client import LLMAidedClient
|
||||
from .table_merge.llm_cell_merge import apply_llm_cross_page_cell_merge
|
||||
|
||||
@@ -12,7 +12,7 @@ import json_repair
|
||||
from loguru import logger
|
||||
from openai import AsyncOpenAI
|
||||
|
||||
from mineru.config import LLMAidedConfig
|
||||
from ...config import LLMAidedConfig
|
||||
|
||||
ValidatedResult = TypeVar("ValidatedResult")
|
||||
|
||||
|
||||
@@ -6,16 +6,16 @@ from __future__ import annotations
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from mineru.types import (
|
||||
from ...types import (
|
||||
RAW_ALGORITHM,
|
||||
RAW_CAPTION,
|
||||
RAW_FOOTNOTE,
|
||||
BlockType,
|
||||
VISUAL_MAIN_TYPES,
|
||||
)
|
||||
from mineru.utils.guess_suffix_or_lang import guess_language_by_text
|
||||
from ...utils.language import guess_code_language
|
||||
|
||||
from mineru.utils.text_utils import clean_isolated_formula
|
||||
from ...utils.text import clean_isolated_formula
|
||||
|
||||
from .content import clean_content, code_content_clean
|
||||
from .lists import (
|
||||
@@ -167,7 +167,7 @@ def _annotate_code_languages(code_blocks: list[BlockDict]) -> None:
|
||||
continue
|
||||
for sub_block in code_block["content"]:
|
||||
if sub_block.get("type") == BlockType.CODE_BODY:
|
||||
code_block["guess_lang"] = guess_language_by_text(sub_block.get("content", ""))
|
||||
code_block["guess_lang"] = guess_code_language(sub_block.get("content", ""))
|
||||
break
|
||||
|
||||
|
||||
|
||||
@@ -8,11 +8,11 @@ from typing import Any
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from mineru.backend.postprocess.lists import fix_office_index_title_blocks, fix_office_paragraph_titles
|
||||
from mineru.backend.postprocess.page_blocks import process_page_blocks
|
||||
from mineru.backend.postprocess.paragraphs import merge_para_text_blocks
|
||||
from mineru.backend.postprocess.table_merge import merge_table
|
||||
from mineru.types import ModelJson, PageInfo
|
||||
from .lists import fix_office_index_title_blocks, fix_office_paragraph_titles
|
||||
from .page_blocks import process_page_blocks
|
||||
from .paragraphs import merge_para_text_blocks
|
||||
from .table_merge import merge_table
|
||||
from ...types import ModelJson, PageInfo
|
||||
|
||||
PAGE_INFO_LIST_ADAPTER = TypeAdapter(list[PageInfo])
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ from typing import Any
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from mineru.types import BlockType
|
||||
from ....types import BlockType
|
||||
|
||||
from .html import _build_front_cache, _scan_rows
|
||||
from .models import (
|
||||
|
||||
@@ -7,7 +7,7 @@ from copy import deepcopy
|
||||
|
||||
from bs4 import Tag
|
||||
|
||||
from mineru.types import BlockType
|
||||
from ....types import BlockType
|
||||
|
||||
from .blocks import (
|
||||
_build_post_body_child_index,
|
||||
|
||||
@@ -5,7 +5,7 @@ from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from mineru.types import BlockType
|
||||
from ....types import BlockType
|
||||
|
||||
from .blocks import _get_or_create_table_state
|
||||
from .models import BlockDict, PageInfoDict, TableMergeState
|
||||
|
||||
@@ -7,7 +7,7 @@ from typing import Any
|
||||
|
||||
from bs4 import BeautifulSoup, Tag
|
||||
|
||||
from mineru.utils.text_utils import full_to_half
|
||||
from ....utils.text import full_to_half
|
||||
|
||||
from .models import (
|
||||
MAX_HEADER_ROWS,
|
||||
|
||||
@@ -8,7 +8,7 @@ import json
|
||||
from dataclasses import dataclass
|
||||
from typing import Literal, cast
|
||||
|
||||
from mineru.types import PageInfo, TableBlock, TableBodyBlock
|
||||
from ....types import PageInfo, TableBlock, TableBodyBlock
|
||||
|
||||
from .html import build_row_rendered_cell_segments, build_table_state_from_html
|
||||
from .structure import _expand_header_count_by_rowspan, detect_table_headers
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# Copyright (c) Opendatalab. All rights reserved.
|
||||
"""跨页表格延续文本与 caption 的轻量判定规则。"""
|
||||
|
||||
from mineru.utils.text_utils import full_to_half
|
||||
from ....utils.text import full_to_half
|
||||
|
||||
CONTINUATION_END_MARKERS = [
|
||||
"(续)",
|
||||
|
||||
@@ -7,7 +7,7 @@ from typing import Any
|
||||
|
||||
from bs4 import Tag
|
||||
|
||||
from mineru.types import BlockType
|
||||
from ....types import BlockType
|
||||
|
||||
from .blocks import (
|
||||
_bbox_for_calculation,
|
||||
|
||||
@@ -7,8 +7,8 @@ import asyncio
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from mineru.types import DocTitleBlock, PageInfo, ParagraphTitleBlock
|
||||
from mineru.render._internal.common.inline import inline_plain_text, parse_inline_content
|
||||
from ...types import DocTitleBlock, PageInfo, ParagraphTitleBlock
|
||||
from .inline import inline_plain_text, parse_inline_content
|
||||
|
||||
from .llm_client import LLMAidedClient
|
||||
|
||||
|
||||
@@ -13,8 +13,8 @@ from ...types import (
|
||||
VISUAL_RELATION_IGNORED_TYPES,
|
||||
VISUAL_TYPE_MAPPING,
|
||||
)
|
||||
from mineru.utils.bbox_utils import bbox_center_distance, bbox_distance
|
||||
from mineru.types import RAW_CAPTION, RAW_FOOTNOTE
|
||||
from ...utils.geometry import bbox_center_distance, bbox_distance
|
||||
from ...types import RAW_CAPTION, RAW_FOOTNOTE
|
||||
from .table_merge.rules import is_table_continuation_text
|
||||
|
||||
INLINE_CAPTION_FRAGMENT_TYPES = {BlockType.TEXT, RAW_FOOTNOTE}
|
||||
|
||||
@@ -9,7 +9,7 @@ from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from ...filetypes import OFFICE_EXTENSIONS
|
||||
from ...utils.pdf_document import PDFDocument
|
||||
from ...model.flash.pdf.document import PDFDocument
|
||||
|
||||
# Optional office doc support
|
||||
try:
|
||||
|
||||
@@ -39,9 +39,9 @@ from ..types import (
|
||||
_iter_child_blocks,
|
||||
select_highest_cached_tier,
|
||||
)
|
||||
from ..utils.model_registry import ModelRepo, model_repos_for_tier
|
||||
from ..utils.models_download_utils import verify_model_repo
|
||||
from ..utils.pdf_document import PDFDocument
|
||||
from ..model.download import verify_model_repo
|
||||
from ..model.registry import ModelRepo, model_repos_for_tier
|
||||
from ..model.flash.pdf.document import PDFDocument
|
||||
from ..version import __version__
|
||||
from .background.parse_server_health import get_health, get_managed_parse_server_tier
|
||||
from .base import AsyncDoclibInterface
|
||||
@@ -2078,7 +2078,7 @@ def _page_markdown_blocks(
|
||||
short_id: str = "",
|
||||
tier: Tier = "standard",
|
||||
) -> list[tuple[int, str]]:
|
||||
from mineru.config import config
|
||||
from ..config import config
|
||||
|
||||
result: list[tuple[int, str]] = []
|
||||
image_renderer = _make_doclib_image_renderer(
|
||||
|
||||
@@ -1394,7 +1394,7 @@ class ParseService:
|
||||
if not pages:
|
||||
continue
|
||||
|
||||
from mineru.config import config
|
||||
from ...config import config
|
||||
from ...render._internal.markdown.blocks import render_single_block
|
||||
|
||||
delimiters = config.render.latex_delimiters
|
||||
|
||||
@@ -8,7 +8,7 @@ from typing import Literal, TypeAlias
|
||||
|
||||
from ..filetypes import PARSEABLE_EXTENSIONS
|
||||
from ..parser.base import ParseResult
|
||||
from ..render.writer import FileBasedDataWriter
|
||||
from ..parser.writer import FileBasedDataWriter
|
||||
from ..types import Tier
|
||||
from ..utils.image_payload import validate_image_sidecar_path
|
||||
|
||||
|
||||
@@ -10,10 +10,10 @@ from typing import Any, Literal
|
||||
|
||||
from ..render import render_markdown, render_structured_content
|
||||
from ..render.contracts import ImageRenderer, RenderMode
|
||||
from ..render.writer import DataWriter
|
||||
from .writer import DataWriter
|
||||
from ..types import FileSuffix, MiddleJson, PageInfo
|
||||
from ..utils.image_payload import ImagePayloadCache
|
||||
from ..utils.pdf_document import PDFDocument
|
||||
from ..model.flash.pdf.document import PDFDocument
|
||||
|
||||
MIDDLE_JSON_SCHEMA_VERSION: str = "2.0"
|
||||
_LEGACY_SCHEMA_VERSION: str = "1.0"
|
||||
|
||||
@@ -13,7 +13,7 @@ from ..backend.analyze import aio_doc_analyze, doc_analyze
|
||||
from ..errors import InvalidRequestError
|
||||
from ..filetypes import IMAGE_EXTENSIONS
|
||||
from ..types import FILE_SUFFIXES, FileSuffix, MiddleJson, ModelJson, PageInfo, Tier
|
||||
from ..utils.backend_options import effort_for_tier
|
||||
from .tier import effort_for_tier
|
||||
from .base import DocumentParser, ParseResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -111,8 +111,8 @@ class MinerUParser(DocumentParser):
|
||||
)
|
||||
|
||||
def _prepare_input(self, path: Path, page_range: str = "") -> _PreparedInput:
|
||||
from ..utils.guess_suffix_or_lang import guess_suffix_by_path
|
||||
from ..utils.pdf_document import PDFDocument
|
||||
from .file_type import guess_suffix_by_path
|
||||
from ..model.flash.pdf.document import PDFDocument
|
||||
|
||||
file_name = path.stem
|
||||
file_bytes = path.read_bytes()
|
||||
@@ -163,8 +163,8 @@ class MinerUParser(DocumentParser):
|
||||
if suffix != "pdf":
|
||||
return file_bytes, None, None
|
||||
|
||||
from ..utils.pdf_document import PDFDocument
|
||||
from ..utils.pdf_page_id import parse_page_range
|
||||
from ..model.flash.pdf.document import PDFDocument
|
||||
from .page_range import parse_page_range
|
||||
|
||||
doc = PDFDocument(file_bytes)
|
||||
page_indices = parse_page_range(page_range, doc.page_count)
|
||||
@@ -178,7 +178,7 @@ class MinerUParser(DocumentParser):
|
||||
if page_indices == list(range(doc.page_count)):
|
||||
return file_bytes, None, None
|
||||
|
||||
from ..utils.pdfium_guard import safe_rewrite_pdf_bytes_with_pdfium_result
|
||||
from ..model.flash.pdf.pdfium import safe_rewrite_pdf_bytes_with_pdfium_result
|
||||
|
||||
rewrite_result = safe_rewrite_pdf_bytes_with_pdfium_result(file_bytes, page_indices=page_indices)
|
||||
if rewrite_result.used_original:
|
||||
|
||||
@@ -1 +1,4 @@
|
||||
# Copyright (c) Opendatalab. All rights reserved.
|
||||
"""MinerU renderer 的非公共实现模块。"""
|
||||
|
||||
__all__: list[str] = []
|
||||
|
||||
@@ -1 +1,4 @@
|
||||
# Copyright (c) Opendatalab. All rights reserved.
|
||||
"""各输出格式共用的渲染语义与规划实现。"""
|
||||
|
||||
__all__: list[str] = []
|
||||
|
||||
@@ -5,7 +5,7 @@ from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from mineru.render._internal.common.inline import inline_plain_text, parse_inline_content
|
||||
from ....backend.postprocess.inline import inline_plain_text, parse_inline_content
|
||||
|
||||
_INDEX_ROMAN_RE = re.compile(r"[ivxlcdm]+", re.IGNORECASE)
|
||||
|
||||
|
||||
@@ -7,8 +7,8 @@ from dataclasses import dataclass
|
||||
import re
|
||||
from typing import Literal, TypeAlias
|
||||
|
||||
from mineru.render._internal.common.inline import inline_plain_text, parse_inline_content
|
||||
from mineru.types import BlockType, ListBlock
|
||||
from ....backend.postprocess.inline import inline_plain_text, parse_inline_content
|
||||
from ....types import BlockType, ListBlock
|
||||
|
||||
ListItemKind: TypeAlias = Literal["unordered", "ordered", "explicit", "none"]
|
||||
OrderedListStyle: TypeAlias = Literal["decimal", "lower-alpha", "upper-alpha", "lower-roman", "upper-roman"]
|
||||
|
||||
@@ -5,9 +5,9 @@ from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from mineru.backend.postprocess.table_merge import merge_table_content
|
||||
from mineru.render.contracts import RenderMode
|
||||
from mineru.types import (
|
||||
from ....backend.postprocess.table_merge import merge_table_content
|
||||
from ...contracts import RenderMode
|
||||
from ....types import (
|
||||
PAGE_AUXILIARY_BLOCK_TYPES,
|
||||
BlockType,
|
||||
ContinuableTextBlockBase,
|
||||
|
||||
@@ -1 +1,4 @@
|
||||
# Copyright (c) Opendatalab. All rights reserved.
|
||||
"""DOCX renderer 的内部实现。"""
|
||||
|
||||
__all__: list[str] = []
|
||||
|
||||
@@ -11,9 +11,9 @@ from urllib.parse import urlsplit
|
||||
|
||||
from PIL import Image, UnidentifiedImageError
|
||||
|
||||
from mineru.render.contracts import AssetResolver
|
||||
from mineru.types import ImagePayloadBlock
|
||||
from mineru.utils.image_payload import parse_image_data_uri_strict, validate_image_sidecar_path
|
||||
from ...contracts import AssetResolver
|
||||
from ....types import ImagePayloadBlock
|
||||
from ....utils.image_payload import parse_image_data_uri_strict, validate_image_sidecar_path
|
||||
|
||||
_PIL_FORMAT_EXTENSIONS = {
|
||||
"BMP": "bmp",
|
||||
|
||||
@@ -17,7 +17,7 @@ from docx.shared import Pt, RGBColor
|
||||
from lxml import etree
|
||||
from loguru import logger
|
||||
|
||||
from mineru.render._internal.common.inline import (
|
||||
from ....backend.postprocess.inline import (
|
||||
InlineEquation,
|
||||
InlineLink,
|
||||
InlineNode,
|
||||
@@ -27,7 +27,7 @@ from mineru.render._internal.common.inline import (
|
||||
join_inline_contents,
|
||||
parse_inline_content,
|
||||
)
|
||||
from mineru.render._internal.docx.math import DocxFormulaError, latex_to_omml
|
||||
from .math import DocxFormulaError, latex_to_omml
|
||||
|
||||
_BOOKMARK_SAFE_RE = re.compile(r"[^A-Za-z0-9_]+")
|
||||
_BOOKMARK_MAX_LENGTH = 40
|
||||
|
||||
@@ -0,0 +1,852 @@
|
||||
# Copyright (c) Opendatalab. All rights reserved.
|
||||
"""严格 MiddleJson 到可编辑 DOCX bytes 的公共渲染实现。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable
|
||||
from io import BytesIO
|
||||
import re
|
||||
|
||||
from bs4 import BeautifulSoup, NavigableString, Tag
|
||||
from docx import Document
|
||||
from docx.document import Document as DocumentType
|
||||
from docx.enum.table import WD_CELL_VERTICAL_ALIGNMENT
|
||||
from docx.enum.text import WD_ALIGN_PARAGRAPH, WD_TAB_ALIGNMENT
|
||||
from docx.oxml import OxmlElement
|
||||
from docx.oxml.ns import qn
|
||||
from docx.shared import Emu, Mm, Pt, Twips
|
||||
from docx.table import _Cell
|
||||
from docx.text.paragraph import Paragraph
|
||||
from lxml import etree
|
||||
from loguru import logger
|
||||
|
||||
from ..common.index import strip_index_page_tail
|
||||
from ....backend.postprocess.inline import (
|
||||
InlineEquation,
|
||||
InlineLink,
|
||||
InlineNode,
|
||||
InlineStyled,
|
||||
InlineText,
|
||||
parse_inline_content,
|
||||
)
|
||||
from ..common.list_items import parse_list_item_marker
|
||||
from ..common.planner import PlannedBlock, build_render_plan
|
||||
from .assets import (
|
||||
DocxAssetError,
|
||||
PreparedImage,
|
||||
prepare_block_image,
|
||||
prepare_html_image,
|
||||
)
|
||||
from .inline import (
|
||||
BookmarkRegistry,
|
||||
InlineRenderContext,
|
||||
append_inline_content,
|
||||
append_inline_nodes,
|
||||
append_internal_link,
|
||||
append_joined_inline_contents,
|
||||
sanitize_xml_text,
|
||||
)
|
||||
from .math import DocxFormulaError, latex_to_omml, split_formula_tag
|
||||
from .styles import (
|
||||
AUXILIARY_STYLE,
|
||||
BODY_STYLE,
|
||||
CAPTION_STYLE,
|
||||
CODE_STYLE,
|
||||
FOOTNOTE_STYLE,
|
||||
FORMULA_FALLBACK_STYLE,
|
||||
SPATIAL_TABLE_STYLE,
|
||||
configure_document,
|
||||
usable_width_emu,
|
||||
usable_width_twips,
|
||||
)
|
||||
from .table import DocxTableError, NestedTableWriter, materialize_docx_tables
|
||||
from ...contracts import AssetResolver, RenderMode
|
||||
from ...docx import DocxRenderError
|
||||
from ....types import (
|
||||
PAGE_AUXILIARY_BLOCK_TYPES,
|
||||
RAW_ALGORITHM,
|
||||
BlockBase,
|
||||
BlockType,
|
||||
BBox,
|
||||
ChartAnnotationBlock,
|
||||
ChartBlock,
|
||||
ChartBodyBlock,
|
||||
CodeAnnotationBlock,
|
||||
CodeBlock,
|
||||
CodeBodyBlock,
|
||||
DocTitleBlock,
|
||||
EquationBlock,
|
||||
ImageAnnotationBlock,
|
||||
ImageBlock,
|
||||
ImageBodyBlock,
|
||||
ImagePayloadBlock,
|
||||
IndexBlock,
|
||||
ListBlock,
|
||||
MiddleJson,
|
||||
PageAuxTextBlock,
|
||||
ParagraphTitleBlock,
|
||||
RefTextBlock,
|
||||
TableAnnotationBlock,
|
||||
TableBlock,
|
||||
TableBodyBlock,
|
||||
TextBlock,
|
||||
TitleBlockBase,
|
||||
)
|
||||
|
||||
_HTML_TABLE_RE = re.compile(r"<table\b", re.IGNORECASE)
|
||||
_ANNOTATION_CAPTION_TYPES = {
|
||||
BlockType.IMAGE_CAPTION,
|
||||
BlockType.TABLE_CAPTION,
|
||||
BlockType.CHART_CAPTION,
|
||||
BlockType.CODE_CAPTION,
|
||||
}
|
||||
_ANNOTATION_FOOTNOTE_TYPES = {
|
||||
BlockType.IMAGE_FOOTNOTE,
|
||||
BlockType.TABLE_FOOTNOTE,
|
||||
BlockType.CHART_FOOTNOTE,
|
||||
BlockType.CODE_FOOTNOTE,
|
||||
}
|
||||
|
||||
|
||||
class _DocxRenderer:
|
||||
"""持有单次 DOCX 渲染所需的 document、素材解析器和书签状态。"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
middle_json: MiddleJson,
|
||||
*,
|
||||
mode: RenderMode,
|
||||
asset_resolver: AssetResolver | None,
|
||||
) -> None:
|
||||
"""初始化 renderer,并预注册全部标题 anchor。"""
|
||||
self.middle_json = middle_json
|
||||
self.mode = mode
|
||||
self.asset_resolver = asset_resolver
|
||||
self.document = Document()
|
||||
configure_document(self.document)
|
||||
self.bookmarks = BookmarkRegistry(_iter_document_anchors(middle_json))
|
||||
self.usable_width_emu = usable_width_emu(self.document)
|
||||
self.usable_width_twips = usable_width_twips(self.document)
|
||||
|
||||
def render(self) -> bytes:
|
||||
"""执行逐页 visitor,并把 python-docx document 序列化为 bytes。"""
|
||||
planned_pages = build_render_plan(self.middle_json, self.mode)
|
||||
for page_position, planned_blocks in enumerate(planned_pages):
|
||||
if self.mode is RenderMode.FULL and page_position > 0:
|
||||
self.document.add_page_break()
|
||||
for planned in planned_blocks:
|
||||
if planned.removed:
|
||||
continue
|
||||
if self.mode is RenderMode.DEFAULT and planned.block.type in PAGE_AUXILIARY_BLOCK_TYPES:
|
||||
continue
|
||||
self._render_planned_block(planned)
|
||||
|
||||
output = BytesIO()
|
||||
self.document.save(output)
|
||||
return output.getvalue()
|
||||
|
||||
def _render_planned_block(self, planned: PlannedBlock) -> None:
|
||||
"""按严格 PageBlock 具体类型分派 Word 写入逻辑。"""
|
||||
block = planned.block
|
||||
context = self._context(planned.page_idx, block)
|
||||
if isinstance(block, (TextBlock, RefTextBlock)):
|
||||
paragraph = self.document.add_paragraph(style=BODY_STYLE)
|
||||
append_joined_inline_contents(
|
||||
paragraph,
|
||||
planned.text_contents or [block.content],
|
||||
context=context,
|
||||
)
|
||||
return
|
||||
if isinstance(block, (DocTitleBlock, ParagraphTitleBlock)):
|
||||
self._render_title(block, context)
|
||||
return
|
||||
if isinstance(block, PageAuxTextBlock):
|
||||
paragraph = self.document.add_paragraph(style=AUXILIARY_STYLE)
|
||||
append_inline_content(paragraph, block.content, context=context)
|
||||
return
|
||||
if isinstance(block, EquationBlock):
|
||||
self._render_equation(block, context)
|
||||
return
|
||||
if isinstance(block, ListBlock):
|
||||
self._render_list(block, context, depth=0)
|
||||
return
|
||||
if isinstance(block, IndexBlock):
|
||||
self._render_index(block, context, depth=0)
|
||||
return
|
||||
if isinstance(block, ImageBlock):
|
||||
self._render_image_block(block, context)
|
||||
return
|
||||
if isinstance(block, TableBlock):
|
||||
self._render_table_block(block, context)
|
||||
return
|
||||
if isinstance(block, ChartBlock):
|
||||
self._render_chart_block(block, context)
|
||||
return
|
||||
if isinstance(block, CodeBlock):
|
||||
self._render_code_block(block, context)
|
||||
return
|
||||
raise TypeError(f"Unsupported PageBlock type: {type(block).__name__}")
|
||||
|
||||
def _context(self, page_idx: int, block: BlockBase) -> InlineRenderContext:
|
||||
"""为当前 block 构造行内渲染上下文。"""
|
||||
return InlineRenderContext(
|
||||
bookmarks=self.bookmarks,
|
||||
page_idx=page_idx,
|
||||
block_index=block.index,
|
||||
block_type=block.type,
|
||||
)
|
||||
|
||||
def _render_title(
|
||||
self,
|
||||
block: DocTitleBlock | ParagraphTitleBlock,
|
||||
context: InlineRenderContext,
|
||||
) -> None:
|
||||
"""用 Heading 1–6 写严格标题,并在段落内容外添加 bookmark。"""
|
||||
level = min(max(block.level, 1), 9)
|
||||
paragraph = self.document.add_paragraph(style=f"Heading {level}")
|
||||
append_inline_content(paragraph, block.content, context=context)
|
||||
self.bookmarks.attach(paragraph, block.anchor)
|
||||
|
||||
def _render_equation(self, block: EquationBlock, context: InlineRenderContext) -> None:
|
||||
"""优先写可编辑 OMML,失败时按图片、可见 LaTeX 的顺序回退。"""
|
||||
content = block.content.strip()
|
||||
if content:
|
||||
formula, tag = split_formula_tag(content)
|
||||
try:
|
||||
math_element = latex_to_omml(formula, display=True)
|
||||
self._append_display_formula(math_element, tag, context=context)
|
||||
return
|
||||
except DocxFormulaError as exc:
|
||||
logger.warning("DOCX display formula fallback: {} ({})", exc, context.location())
|
||||
|
||||
if block.image_base64 is not None or block.image_path is not None:
|
||||
try:
|
||||
self._append_block_image(block, context=context, alt_text="formula")
|
||||
return
|
||||
except DocxRenderError:
|
||||
if not content:
|
||||
raise
|
||||
logger.warning("DOCX formula image fallback failed; preserving LaTeX ({})", context.location())
|
||||
|
||||
if content:
|
||||
paragraph = self.document.add_paragraph(style=FORMULA_FALLBACK_STYLE)
|
||||
paragraph.add_run(sanitize_xml_text(content, context=context))
|
||||
return
|
||||
raise self._render_error("Equation does not contain usable LaTeX or image", context)
|
||||
|
||||
def _append_display_formula(
|
||||
self,
|
||||
math_element: etree._Element,
|
||||
tag: str | None,
|
||||
*,
|
||||
context: InlineRenderContext,
|
||||
) -> None:
|
||||
"""把块公式居中写入;带 tag 时使用中心与右对齐 tab。"""
|
||||
if tag is None:
|
||||
properties = OxmlElement("m:oMathParaPr")
|
||||
justification = OxmlElement("m:jc")
|
||||
justification.set(qn("m:val"), "center")
|
||||
properties.append(justification)
|
||||
math_element.insert(0, properties)
|
||||
paragraph = self.document.add_paragraph(style=BODY_STYLE)
|
||||
paragraph.paragraph_format.space_before = Pt(4)
|
||||
paragraph.paragraph_format.space_after = Pt(5)
|
||||
paragraph._p.append(math_element)
|
||||
return
|
||||
|
||||
equation = math_element.find(qn("m:oMath"))
|
||||
if equation is None:
|
||||
raise ValueError("Display OMML does not contain m:oMath")
|
||||
math_element.remove(equation)
|
||||
paragraph = self.document.add_paragraph(style=BODY_STYLE)
|
||||
paragraph.paragraph_format.space_before = Pt(4)
|
||||
paragraph.paragraph_format.space_after = Pt(5)
|
||||
paragraph.paragraph_format.tab_stops.add_tab_stop(
|
||||
Twips(self.usable_width_twips // 2),
|
||||
WD_TAB_ALIGNMENT.CENTER,
|
||||
)
|
||||
paragraph.paragraph_format.tab_stops.add_tab_stop(
|
||||
Twips(self.usable_width_twips),
|
||||
WD_TAB_ALIGNMENT.RIGHT,
|
||||
)
|
||||
paragraph.add_run().add_tab()
|
||||
paragraph._p.append(equation)
|
||||
paragraph.add_run().add_tab()
|
||||
rendered_tag = tag if tag.startswith("(") and tag.endswith(")") else f"({tag})"
|
||||
paragraph.add_run(sanitize_xml_text(rendered_tag, context=context))
|
||||
|
||||
def _render_list(
|
||||
self,
|
||||
block: ListBlock,
|
||||
context: InlineRenderContext,
|
||||
*,
|
||||
depth: int,
|
||||
) -> None:
|
||||
"""保留叶子原 marker,并仅用缩进表达递归列表层级。"""
|
||||
for child in block.content:
|
||||
if isinstance(child, ListBlock):
|
||||
self._render_list(child, context, depth=depth + 1)
|
||||
continue
|
||||
paragraph = self.document.add_paragraph(style=BODY_STYLE)
|
||||
item = parse_list_item_marker(child.content)
|
||||
if item.marker is None:
|
||||
paragraph.paragraph_format.left_indent = Mm(depth * 6)
|
||||
else:
|
||||
paragraph.paragraph_format.left_indent = Mm((depth + 1) * 6)
|
||||
paragraph.paragraph_format.first_line_indent = Mm(-6)
|
||||
append_inline_content(paragraph, child.content, context=context)
|
||||
|
||||
def _render_index(
|
||||
self,
|
||||
block: IndexBlock,
|
||||
context: InlineRenderContext,
|
||||
*,
|
||||
depth: int,
|
||||
) -> None:
|
||||
"""递归输出目录项,并把标题叶子链接到预注册 bookmark。"""
|
||||
for child in block.content:
|
||||
if isinstance(child, IndexBlock):
|
||||
self._render_index(child, context, depth=depth + 1)
|
||||
continue
|
||||
content = strip_index_page_tail(child.content)
|
||||
nodes = parse_inline_content(content)
|
||||
if not nodes:
|
||||
continue
|
||||
paragraph = self.document.add_paragraph(style=BODY_STYLE)
|
||||
paragraph.paragraph_format.left_indent = Mm((depth + 1) * 6)
|
||||
paragraph.paragraph_format.first_line_indent = Mm(-4)
|
||||
paragraph.add_run("• ")
|
||||
anchor = child.anchor if isinstance(child, TitleBlockBase) else None
|
||||
append_internal_link(paragraph, nodes, anchor=anchor, context=context)
|
||||
|
||||
def _render_image_block(self, block: ImageBlock, context: InlineRenderContext) -> None:
|
||||
"""按原始子块顺序写图片主体、caption 与 footnote。"""
|
||||
for child in block.content:
|
||||
if isinstance(child, ImageBodyBlock):
|
||||
alt_text = _plain_html_text(child.content) or block.sub_type or "image"
|
||||
self._append_block_image(child, context=context, alt_text=alt_text)
|
||||
elif isinstance(child, ImageAnnotationBlock):
|
||||
self._render_annotation(child, context)
|
||||
else:
|
||||
raise TypeError(f"Unsupported image child: {type(child).__name__}")
|
||||
|
||||
def _render_table_block(self, block: TableBlock, context: InlineRenderContext) -> None:
|
||||
"""按原始子块顺序写原生 HTML table 或预格式空间投影文本。"""
|
||||
for child in block.content:
|
||||
if isinstance(child, TableBodyBlock):
|
||||
if _HTML_TABLE_RE.search(child.content):
|
||||
try:
|
||||
self._append_html_tables(child.content, context=context, depth=0)
|
||||
except DocxRenderError as exc:
|
||||
logger.warning("DOCX HTML table fallback: {}", exc)
|
||||
if child.image_base64 is None and child.image_path is None:
|
||||
raise self._render_error(
|
||||
"HTML table cannot be materialized and has no image fallback",
|
||||
context,
|
||||
) from exc
|
||||
self._append_block_image(child, context=context, alt_text="table")
|
||||
else:
|
||||
if child.content and not child.content.isspace():
|
||||
paragraph = self.document.add_paragraph(style=SPATIAL_TABLE_STYLE)
|
||||
paragraph.add_run(sanitize_xml_text(child.content, context=context))
|
||||
elif child.image_base64 is not None or child.image_path is not None:
|
||||
self._append_block_image(child, context=context, alt_text="table")
|
||||
else:
|
||||
raise self._render_error(
|
||||
"Spatial table does not contain text content or image",
|
||||
context,
|
||||
)
|
||||
elif isinstance(child, TableAnnotationBlock):
|
||||
self._render_annotation(child, context)
|
||||
else:
|
||||
raise TypeError(f"Unsupported table child: {type(child).__name__}")
|
||||
|
||||
def _render_chart_block(self, block: ChartBlock, context: InlineRenderContext) -> None:
|
||||
"""先写图表图片,再把 HTML 结构化数据追加为原生表格。"""
|
||||
for child in block.content:
|
||||
if isinstance(child, ChartBodyBlock):
|
||||
has_image = child.image_base64 is not None or child.image_path is not None
|
||||
if has_image:
|
||||
self._append_block_image(
|
||||
child,
|
||||
context=context,
|
||||
alt_text=_plain_html_text(child.content) or block.sub_type or "chart",
|
||||
)
|
||||
if _HTML_TABLE_RE.search(child.content):
|
||||
try:
|
||||
self._append_html_tables(child.content, context=context, depth=0)
|
||||
except DocxRenderError as exc:
|
||||
if not has_image:
|
||||
raise
|
||||
logger.warning("DOCX chart data table omitted after image fallback: {}", exc)
|
||||
elif child.content.strip() and not has_image:
|
||||
paragraph = self.document.add_paragraph(style=BODY_STYLE)
|
||||
append_inline_content(paragraph, child.content, context=context)
|
||||
elif not has_image and not child.content.strip():
|
||||
raise self._render_error("Chart does not contain image or structured content", context)
|
||||
elif isinstance(child, ChartAnnotationBlock):
|
||||
self._render_annotation(child, context)
|
||||
else:
|
||||
raise TypeError(f"Unsupported chart child: {type(child).__name__}")
|
||||
|
||||
def _render_code_block(self, block: CodeBlock, context: InlineRenderContext) -> None:
|
||||
"""按原始子块顺序写代码或算法主体及说明。"""
|
||||
for child in block.content:
|
||||
if isinstance(child, CodeBodyBlock):
|
||||
paragraph = self.document.add_paragraph(style=CODE_STYLE)
|
||||
if block.sub_type == BlockType.CODE:
|
||||
paragraph.add_run(sanitize_xml_text(child.content, context=context))
|
||||
elif block.sub_type == RAW_ALGORITHM:
|
||||
append_inline_content(paragraph, child.content, context=context)
|
||||
else:
|
||||
raise ValueError(f"Unsupported code subtype: {block.sub_type}")
|
||||
elif isinstance(child, CodeAnnotationBlock):
|
||||
self._render_annotation(child, context)
|
||||
else:
|
||||
raise TypeError(f"Unsupported code child: {type(child).__name__}")
|
||||
|
||||
def _render_annotation(
|
||||
self,
|
||||
block: ImageAnnotationBlock | TableAnnotationBlock | ChartAnnotationBlock | CodeAnnotationBlock,
|
||||
context: InlineRenderContext,
|
||||
) -> None:
|
||||
"""根据说明类型选择 Caption 或 Footnote 样式并写入行内内容。"""
|
||||
if block.type in _ANNOTATION_CAPTION_TYPES:
|
||||
style = CAPTION_STYLE
|
||||
elif block.type in _ANNOTATION_FOOTNOTE_TYPES:
|
||||
style = FOOTNOTE_STYLE
|
||||
else:
|
||||
raise TypeError(f"Unsupported annotation type: {block.type}")
|
||||
paragraph = self.document.add_paragraph(style=style)
|
||||
append_inline_content(paragraph, block.content, context=context)
|
||||
|
||||
def _append_block_image(
|
||||
self,
|
||||
block: ImagePayloadBlock,
|
||||
*,
|
||||
context: InlineRenderContext,
|
||||
alt_text: str,
|
||||
) -> None:
|
||||
"""安全加载 block 图片,并按 bbox/自然尺寸限制到可用页面范围。"""
|
||||
try:
|
||||
prepared = prepare_block_image(block, self.asset_resolver)
|
||||
except DocxAssetError as exc:
|
||||
raise self._render_error(str(exc), context) from exc
|
||||
self._append_prepared_image(
|
||||
prepared,
|
||||
bbox=block.bbox,
|
||||
alt_text=alt_text,
|
||||
max_width_emu=self.usable_width_emu,
|
||||
context=context,
|
||||
)
|
||||
|
||||
def _append_prepared_image(
|
||||
self,
|
||||
prepared: PreparedImage,
|
||||
*,
|
||||
bbox: BBox | None,
|
||||
alt_text: str,
|
||||
max_width_emu: int,
|
||||
context: InlineRenderContext,
|
||||
paragraph: Paragraph | None = None,
|
||||
) -> None:
|
||||
"""把准备好的图片按比例插入居中段落并写入 alt description。"""
|
||||
target_paragraph = paragraph or self.document.add_paragraph(style=BODY_STYLE)
|
||||
target_paragraph.alignment = WD_ALIGN_PARAGRAPH.CENTER
|
||||
natural_width = round(prepared.width_px / 96 * 914400)
|
||||
desired_width = natural_width
|
||||
if bbox is not None:
|
||||
desired_width = round(max_width_emu * max(0.0, min(1.0, bbox[2] - bbox[0])))
|
||||
minimum_width = min(int(Mm(5)), max_width_emu)
|
||||
desired_width = max(minimum_width, min(desired_width, max_width_emu))
|
||||
|
||||
usable_height = int(Mm(257))
|
||||
rendered_height = round(desired_width * prepared.height_px / max(prepared.width_px, 1))
|
||||
if rendered_height > usable_height:
|
||||
desired_width = round(usable_height * prepared.width_px / max(prepared.height_px, 1))
|
||||
|
||||
shape = target_paragraph.add_run().add_picture(
|
||||
BytesIO(prepared.data),
|
||||
width=Emu(max(1, desired_width)),
|
||||
)
|
||||
description = sanitize_xml_text(
|
||||
re.sub(r"\s+", " ", alt_text).strip()[:2048],
|
||||
context=context,
|
||||
)
|
||||
if description:
|
||||
shape._inline.docPr.set("descr", description)
|
||||
|
||||
def _append_html_tables(
|
||||
self,
|
||||
markup: str,
|
||||
*,
|
||||
context: InlineRenderContext,
|
||||
depth: int,
|
||||
container: DocumentType | _Cell | None = None,
|
||||
) -> None:
|
||||
"""物化 HTML 表格,并通过共享行内/素材逻辑填充 origin cell。"""
|
||||
if depth >= 4:
|
||||
raise self._render_error("Nested HTML table depth exceeds 4", context)
|
||||
target_container = container or self.document
|
||||
target_xml = target_container._element.body if isinstance(target_container, DocumentType) else target_container._tc
|
||||
existing_child_count = len(target_xml)
|
||||
insertion_index = existing_child_count - 1 if isinstance(target_container, DocumentType) else existing_child_count
|
||||
existing_relationship_ids = set(self.document.part.rels)
|
||||
|
||||
def fill_cell(cell: _Cell, source: Tag, write_nested: NestedTableWriter) -> None:
|
||||
"""填充一个 origin cell,并在遇到 nested table 时调用递归 writer。"""
|
||||
self._fill_html_cell(
|
||||
cell,
|
||||
source,
|
||||
write_nested=write_nested,
|
||||
context=context,
|
||||
)
|
||||
|
||||
try:
|
||||
materialize_docx_tables(
|
||||
target_container,
|
||||
markup,
|
||||
width_twips=self.usable_width_twips,
|
||||
fill_cell=fill_cell,
|
||||
)
|
||||
except DocxRenderError:
|
||||
self._rollback_table_materialization(
|
||||
target_xml,
|
||||
existing_child_count=existing_child_count,
|
||||
insertion_index=insertion_index,
|
||||
existing_relationship_ids=existing_relationship_ids,
|
||||
)
|
||||
raise
|
||||
except DocxTableError as exc:
|
||||
self._rollback_table_materialization(
|
||||
target_xml,
|
||||
existing_child_count=existing_child_count,
|
||||
insertion_index=insertion_index,
|
||||
existing_relationship_ids=existing_relationship_ids,
|
||||
)
|
||||
raise self._render_error(str(exc), context) from exc
|
||||
|
||||
def _rollback_table_materialization(
|
||||
self,
|
||||
target_xml: etree._Element,
|
||||
*,
|
||||
existing_child_count: int,
|
||||
insertion_index: int,
|
||||
existing_relationship_ids: set[str],
|
||||
) -> None:
|
||||
"""回滚本轮表格新增的 XML 根节点和 relationship,避免残留半成品。"""
|
||||
while len(target_xml) > existing_child_count:
|
||||
target_xml.remove(target_xml[insertion_index])
|
||||
for relationship_id in tuple(self.document.part.rels):
|
||||
if relationship_id not in existing_relationship_ids:
|
||||
self.document.part.drop_rel(relationship_id)
|
||||
|
||||
def _fill_html_cell(
|
||||
self,
|
||||
cell: _Cell,
|
||||
source: Tag,
|
||||
*,
|
||||
write_nested: NestedTableWriter,
|
||||
context: InlineRenderContext,
|
||||
) -> None:
|
||||
"""按源顺序填充单元格文本、图片和任意包装层中的嵌套表格。"""
|
||||
cell.text = ""
|
||||
cell.vertical_alignment = WD_CELL_VERTICAL_ALIGNMENT.CENTER
|
||||
_set_cell_margins(cell, top=100, start=120, bottom=100, end=120)
|
||||
max_image_width_emu = _cell_content_width_emu(cell, horizontal_margin_twips=240)
|
||||
paragraph = cell.paragraphs[0]
|
||||
paragraph.paragraph_format.space_after = Pt(0)
|
||||
paragraph.paragraph_format.line_spacing = 1.0
|
||||
|
||||
for child in source.children:
|
||||
if not isinstance(child, (NavigableString, Tag)):
|
||||
continue
|
||||
if isinstance(child, Tag) and child.name.lower() in {"p", "div"} and _paragraph_has_content(paragraph):
|
||||
paragraph = cell.add_paragraph()
|
||||
paragraph.paragraph_format.space_after = Pt(0)
|
||||
paragraph.paragraph_format.line_spacing = 1.0
|
||||
styles = ("bold",) if source.name == "th" else ()
|
||||
paragraph, _ = self._append_html_node(
|
||||
paragraph,
|
||||
child,
|
||||
cell=cell,
|
||||
write_nested=write_nested,
|
||||
context=context,
|
||||
inherited_styles=styles,
|
||||
max_image_width_emu=max_image_width_emu,
|
||||
)
|
||||
|
||||
def _append_html_node(
|
||||
self,
|
||||
paragraph: Paragraph,
|
||||
node: NavigableString | Tag,
|
||||
*,
|
||||
cell: _Cell,
|
||||
write_nested: NestedTableWriter,
|
||||
context: InlineRenderContext,
|
||||
inherited_styles: tuple[str, ...],
|
||||
max_image_width_emu: int,
|
||||
) -> tuple[Paragraph, bool]:
|
||||
"""按 HTML 源顺序写入单元格行内文本、样式、公式和图片。"""
|
||||
if isinstance(node, NavigableString):
|
||||
text = str(node)
|
||||
if not text:
|
||||
return paragraph, False
|
||||
nodes: list[InlineNode] = [InlineText(text)]
|
||||
if inherited_styles:
|
||||
nodes = [InlineStyled(nodes, inherited_styles)]
|
||||
append_inline_nodes(paragraph, nodes, context=context)
|
||||
return paragraph, bool(text.strip())
|
||||
|
||||
name = node.name.lower()
|
||||
if name == "table":
|
||||
write_nested(node)
|
||||
trailing_paragraph = cell.paragraphs[-1]
|
||||
trailing_paragraph.paragraph_format.space_after = Pt(0)
|
||||
trailing_paragraph.paragraph_format.line_spacing = 1.0
|
||||
return trailing_paragraph, True
|
||||
if name == "img":
|
||||
self._append_html_image(
|
||||
paragraph,
|
||||
str(node.get("src", "")),
|
||||
context=context,
|
||||
alt_text=str(node.get("alt", "")),
|
||||
max_width_emu=max_image_width_emu,
|
||||
)
|
||||
return paragraph, True
|
||||
if name == "br":
|
||||
paragraph.add_run().add_break()
|
||||
return paragraph, True
|
||||
if name == "eq":
|
||||
latex = node.get_text().strip()
|
||||
nodes = [InlineEquation(latex)] if latex else []
|
||||
if inherited_styles and nodes:
|
||||
nodes = [InlineStyled(nodes, inherited_styles)]
|
||||
append_inline_nodes(paragraph, nodes, context=context)
|
||||
return paragraph, bool(nodes)
|
||||
if name == "a":
|
||||
children = _html_inline_nodes(node)
|
||||
if inherited_styles and children:
|
||||
children = [InlineStyled(children, inherited_styles)]
|
||||
append_inline_nodes(
|
||||
paragraph,
|
||||
[InlineLink(children, str(node.get("href", "")).strip())],
|
||||
context=context,
|
||||
)
|
||||
return paragraph, bool(children)
|
||||
if name in {"ul", "ol"}:
|
||||
start_value = node.get("start", "1")
|
||||
try:
|
||||
ordered_number = int(str(start_value))
|
||||
except ValueError:
|
||||
ordered_number = 1
|
||||
rendered = False
|
||||
for item in node.find_all("li", recursive=False):
|
||||
if _paragraph_has_content(paragraph) or rendered:
|
||||
paragraph.add_run().add_break()
|
||||
marker = f"{ordered_number}. " if name == "ol" else "- "
|
||||
paragraph.add_run(marker)
|
||||
if name == "ol":
|
||||
ordered_number += 1
|
||||
for child in item.children:
|
||||
if isinstance(child, (NavigableString, Tag)):
|
||||
paragraph, child_rendered = self._append_html_node(
|
||||
paragraph,
|
||||
child,
|
||||
cell=cell,
|
||||
write_nested=write_nested,
|
||||
context=context,
|
||||
inherited_styles=inherited_styles,
|
||||
max_image_width_emu=max_image_width_emu,
|
||||
)
|
||||
rendered = child_rendered or rendered
|
||||
return paragraph, rendered
|
||||
if name == "li":
|
||||
rendered = False
|
||||
for child in node.children:
|
||||
if isinstance(child, (NavigableString, Tag)):
|
||||
paragraph, child_rendered = self._append_html_node(
|
||||
paragraph,
|
||||
child,
|
||||
cell=cell,
|
||||
write_nested=write_nested,
|
||||
context=context,
|
||||
inherited_styles=inherited_styles,
|
||||
max_image_width_emu=max_image_width_emu,
|
||||
)
|
||||
rendered = child_rendered or rendered
|
||||
return paragraph, rendered
|
||||
if name == "text":
|
||||
nodes = parse_inline_content(str(node))
|
||||
if inherited_styles and nodes:
|
||||
nodes = [InlineStyled(nodes, inherited_styles)]
|
||||
append_inline_nodes(paragraph, nodes, context=context)
|
||||
return paragraph, bool(nodes)
|
||||
|
||||
style = {
|
||||
"strong": "bold",
|
||||
"b": "bold",
|
||||
"em": "italic",
|
||||
"i": "italic",
|
||||
"u": "underline",
|
||||
"s": "strikethrough",
|
||||
"sup": "superscript",
|
||||
"sub": "subscript",
|
||||
}.get(name)
|
||||
styles = tuple(dict.fromkeys((*inherited_styles, style))) if style else inherited_styles
|
||||
rendered = False
|
||||
for child in node.children:
|
||||
if isinstance(child, (NavigableString, Tag)):
|
||||
paragraph, child_rendered = self._append_html_node(
|
||||
paragraph,
|
||||
child,
|
||||
cell=cell,
|
||||
write_nested=write_nested,
|
||||
context=context,
|
||||
inherited_styles=styles,
|
||||
max_image_width_emu=max_image_width_emu,
|
||||
)
|
||||
rendered = child_rendered or rendered
|
||||
return paragraph, rendered
|
||||
|
||||
def _append_html_image(
|
||||
self,
|
||||
paragraph: Paragraph,
|
||||
source: str,
|
||||
*,
|
||||
context: InlineRenderContext,
|
||||
alt_text: str,
|
||||
max_width_emu: int,
|
||||
) -> None:
|
||||
"""安全加载表格单元格 img,并限制到紧凑的单元格宽度。"""
|
||||
try:
|
||||
prepared = prepare_html_image(source, self.asset_resolver)
|
||||
except DocxAssetError as exc:
|
||||
raise self._render_error(str(exc), context) from exc
|
||||
self._append_prepared_image(
|
||||
prepared,
|
||||
bbox=None,
|
||||
alt_text=alt_text or "table image",
|
||||
max_width_emu=min(self.usable_width_emu, max_width_emu),
|
||||
context=context,
|
||||
paragraph=paragraph,
|
||||
)
|
||||
|
||||
def _render_error(self, message: str, context: InlineRenderContext) -> DocxRenderError:
|
||||
"""用当前 context 构造带 page/block 定位的公共异常。"""
|
||||
return DocxRenderError(
|
||||
message,
|
||||
page_idx=context.page_idx,
|
||||
block_index=context.block_index,
|
||||
block_type=context.block_type,
|
||||
)
|
||||
|
||||
|
||||
def render_docx(
|
||||
middle_json: MiddleJson,
|
||||
*,
|
||||
mode: RenderMode = RenderMode.DEFAULT,
|
||||
asset_resolver: AssetResolver | None = None,
|
||||
) -> bytes:
|
||||
"""把严格 MiddleJson 无副作用地渲染为完整 DOCX bytes。"""
|
||||
if not isinstance(middle_json, MiddleJson):
|
||||
raise TypeError("render_docx expects a MiddleJson instance")
|
||||
if not isinstance(mode, RenderMode):
|
||||
raise TypeError("mode must be a RenderMode value")
|
||||
return _DocxRenderer(
|
||||
middle_json,
|
||||
mode=mode,
|
||||
asset_resolver=asset_resolver,
|
||||
).render()
|
||||
|
||||
|
||||
def _iter_document_anchors(middle_json: MiddleJson) -> Iterable[str]:
|
||||
"""按文档顺序遍历实际会写入 bookmark 的正文标题 anchor。"""
|
||||
for page in middle_json.pages:
|
||||
for block in page.blocks:
|
||||
if isinstance(block, TitleBlockBase) and block.anchor:
|
||||
yield block.anchor
|
||||
|
||||
|
||||
def _plain_html_text(content: str) -> str:
|
||||
"""把图片或图表 body 内容收敛为适合 alt description 的纯文本。"""
|
||||
if not content.strip():
|
||||
return ""
|
||||
return BeautifulSoup(content, "html.parser").get_text(" ", strip=True)
|
||||
|
||||
|
||||
def _html_inline_nodes(node: Tag) -> list[InlineNode]:
|
||||
"""把表格单元格中的安全 HTML 行内标签转换为共享 InlineNode。"""
|
||||
nodes: list[InlineNode] = []
|
||||
for child in node.children:
|
||||
if isinstance(child, NavigableString):
|
||||
if str(child):
|
||||
nodes.append(InlineText(str(child)))
|
||||
continue
|
||||
if not isinstance(child, Tag):
|
||||
continue
|
||||
name = child.name.lower()
|
||||
children = _html_inline_nodes(child)
|
||||
if name == "eq":
|
||||
latex = child.get_text().strip()
|
||||
if latex:
|
||||
nodes.append(InlineEquation(latex))
|
||||
elif name in {"strong", "b"}:
|
||||
nodes.append(InlineStyled(children, ("bold",)))
|
||||
elif name in {"em", "i"}:
|
||||
nodes.append(InlineStyled(children, ("italic",)))
|
||||
elif name == "u":
|
||||
nodes.append(InlineStyled(children, ("underline",)))
|
||||
elif name == "s":
|
||||
nodes.append(InlineStyled(children, ("strikethrough",)))
|
||||
elif name == "sup":
|
||||
nodes.append(InlineStyled(children, ("superscript",)))
|
||||
elif name == "sub":
|
||||
nodes.append(InlineStyled(children, ("subscript",)))
|
||||
elif name == "a":
|
||||
nodes.append(InlineLink(children, str(child.get("href", "")).strip()))
|
||||
elif name == "br":
|
||||
nodes.append(InlineText("\n"))
|
||||
elif name == "text":
|
||||
nodes.extend(parse_inline_content(str(child)))
|
||||
elif name not in {"img", "table"}:
|
||||
nodes.extend(children)
|
||||
return nodes
|
||||
|
||||
|
||||
def _set_cell_margins(
|
||||
cell: _Cell,
|
||||
*,
|
||||
top: int,
|
||||
start: int,
|
||||
bottom: int,
|
||||
end: int,
|
||||
) -> None:
|
||||
"""用 DXA 为 Word 单元格写入四边内边距。"""
|
||||
cell_properties = cell._tc.get_or_add_tcPr()
|
||||
margins = cell_properties.find(qn("w:tcMar"))
|
||||
if margins is None:
|
||||
margins = OxmlElement("w:tcMar")
|
||||
cell_properties.append(margins)
|
||||
for side, value in (("top", top), ("start", start), ("bottom", bottom), ("end", end)):
|
||||
element = margins.find(qn(f"w:{side}"))
|
||||
if element is None:
|
||||
element = OxmlElement(f"w:{side}")
|
||||
margins.append(element)
|
||||
element.set(qn("w:w"), str(value))
|
||||
element.set(qn("w:type"), "dxa")
|
||||
|
||||
|
||||
def _cell_content_width_emu(cell: _Cell, *, horizontal_margin_twips: int) -> int:
|
||||
"""从当前物理或合并单元格 tcW 计算扣除左右内边距后的可用宽度。"""
|
||||
cell_width = cell._tc.get_or_add_tcPr().get_or_add_tcW().w
|
||||
width_twips = int(cell_width) if cell_width is not None else 1
|
||||
return int(Twips(max(1, width_twips - horizontal_margin_twips)))
|
||||
|
||||
|
||||
def _paragraph_has_content(paragraph: Paragraph) -> bool:
|
||||
"""判断段落是否包含 pPr 之外的 run、公式、链接或 drawing。"""
|
||||
return any(child.tag != qn("w:pPr") for child in paragraph._p)
|
||||
|
||||
|
||||
__all__ = ["render_docx"]
|
||||
@@ -1 +1,4 @@
|
||||
# Copyright (c) Opendatalab. All rights reserved.
|
||||
"""HTML renderer 的内部实现。"""
|
||||
|
||||
__all__: list[str] = []
|
||||
|
||||
@@ -9,7 +9,7 @@ import re
|
||||
from dataclasses import dataclass
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from mineru.render._internal.common.inline import (
|
||||
from ....backend.postprocess.inline import (
|
||||
InlineEquation,
|
||||
InlineLink,
|
||||
InlineNode,
|
||||
@@ -19,25 +19,19 @@ from mineru.render._internal.common.inline import (
|
||||
join_inline_contents,
|
||||
parse_inline_content,
|
||||
)
|
||||
from mineru.render._internal.html.sanitizer import sanitize_link_url
|
||||
from .sanitizer import sanitize_link_url
|
||||
|
||||
|
||||
_AUTOLINK_TLD_PATTERN = r"(?:[A-Za-z]{2,63}|xn--[A-Za-z0-9-]{2,59})"
|
||||
_AUTOLINK_DOMAIN_PATTERN = (
|
||||
rf"(?:[A-Za-z0-9](?:[A-Za-z0-9-]{{0,61}}[A-Za-z0-9])?\.){{1,10}}{_AUTOLINK_TLD_PATTERN}"
|
||||
)
|
||||
_AUTOLINK_DOMAIN_PATTERN = rf"(?:[A-Za-z0-9](?:[A-Za-z0-9-]{{0,61}}[A-Za-z0-9])?\.){{1,10}}{_AUTOLINK_TLD_PATTERN}"
|
||||
_AUTOLINK_EMAIL_EDGE_PATTERN = r"[A-Za-z0-9!#$%&'*+/=?^_{|}~-]"
|
||||
_AUTOLINK_EMAIL_PATTERN = (
|
||||
rf"{_AUTOLINK_EMAIL_EDGE_PATTERN}"
|
||||
rf"(?:[A-Za-z0-9.!#$%&'*+/=?^_{{|}}~-]{{0,62}}{_AUTOLINK_EMAIL_EDGE_PATTERN})?"
|
||||
rf"@{_AUTOLINK_DOMAIN_PATTERN}"
|
||||
)
|
||||
_AUTOLINK_EXPLICIT_HOST_PATTERN = (
|
||||
rf"(?:{_AUTOLINK_DOMAIN_PATTERN}|localhost|(?:\d{{1,3}}\.){{3}}\d{{1,3}}|\[[0-9A-Fa-f:.]+\])"
|
||||
)
|
||||
_AUTOLINK_URL_TAIL_PATTERN = (
|
||||
r"(?::\d{1,5})?(?:[/?#](?:(?!https?://)[A-Za-z0-9._~:/?#\[\]@!$&()*+,;=%-])*)?"
|
||||
)
|
||||
_AUTOLINK_EXPLICIT_HOST_PATTERN = rf"(?:{_AUTOLINK_DOMAIN_PATTERN}|localhost|(?:\d{{1,3}}\.){{3}}\d{{1,3}}|\[[0-9A-Fa-f:.]+\])"
|
||||
_AUTOLINK_URL_TAIL_PATTERN = r"(?::\d{1,5})?(?:[/?#](?:(?!https?://)[A-Za-z0-9._~:/?#\[\]@!$&()*+,;=%-])*)?"
|
||||
_AUTOLINK_RE = re.compile(
|
||||
rf"(?:(?P<email>(?<![A-Za-z0-9._%+@-]){_AUTOLINK_EMAIL_PATTERN})"
|
||||
rf"|(?P<explicit>https?://{_AUTOLINK_EXPLICIT_HOST_PATTERN}{_AUTOLINK_URL_TAIL_PATTERN})"
|
||||
|
||||
@@ -0,0 +1,941 @@
|
||||
# Copyright (c) Opendatalab. All rights reserved.
|
||||
"""严格 MiddleJson 到 MinerU 风格 HTML 的公共渲染实现。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import re
|
||||
from functools import lru_cache
|
||||
from importlib import resources
|
||||
from urllib.parse import quote
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
from loguru import logger
|
||||
|
||||
from ..common.index import strip_index_page_tail
|
||||
from ....backend.postprocess.inline import inline_plain_text, parse_inline_content
|
||||
from ..common.list_items import ListItem, parse_list_item_marker, reference_list_needs_bullets
|
||||
from ..common.planner import PlannedBlock, build_render_plan
|
||||
from .inline import (
|
||||
HtmlInlineResult,
|
||||
render_inline_content_html,
|
||||
render_inline_nodes_html,
|
||||
render_joined_inline_contents_html,
|
||||
render_math_html,
|
||||
)
|
||||
from .sanitizer import (
|
||||
is_supported_html_markup,
|
||||
sanitize_html_fragment,
|
||||
sanitize_image_source,
|
||||
)
|
||||
from .table import looks_like_gfm_table, render_gfm_table_html
|
||||
from ...contracts import RenderMode
|
||||
from ....types import (
|
||||
PAGE_AUXILIARY_BLOCK_TYPES,
|
||||
RAW_ALGORITHM,
|
||||
BlockType,
|
||||
ChartAnnotationBlock,
|
||||
ChartBlock,
|
||||
ChartBodyBlock,
|
||||
CodeAnnotationBlock,
|
||||
CodeBlock,
|
||||
CodeBodyBlock,
|
||||
DocTitleBlock,
|
||||
EquationBlock,
|
||||
ImageAnnotationBlock,
|
||||
ImageBlock,
|
||||
ImageBodyBlock,
|
||||
IndexBlock,
|
||||
ListBlock,
|
||||
MiddleJson,
|
||||
PageAuxTextBlock,
|
||||
ParagraphTitleBlock,
|
||||
RefTextBlock,
|
||||
TableAnnotationBlock,
|
||||
TableBlock,
|
||||
TableBodyBlock,
|
||||
TextBlock,
|
||||
TitleBlockBase,
|
||||
)
|
||||
|
||||
_STYLE_RESOURCE_NAME = "mineru.min.css"
|
||||
_MATHJAX_URL = "https://cdn.jsdelivr.net/npm/mathjax@4.1.2/tex-chtml.js"
|
||||
_MATHJAX_INTEGRITY = "sha384-zAhQQhdaMeHsMProNntGGg6nOUVcfuF9F22C3d1qJ9NZAVzCplXk1X85D2O5iufn"
|
||||
_PRISM_CORE_URL = "https://cdn.jsdelivr.net/npm/prismjs@1.30.0/components/prism-core.min.js"
|
||||
_PRISM_CORE_INTEGRITY = "sha384-zLRFO4dwowZvh8kzutOb5AWhH7f39HeJp+N7PtHF1SQtTBnifRx0AtmvTYs3F4YV"
|
||||
_PRISM_AUTOLOADER_URL = "https://cdn.jsdelivr.net/npm/prismjs@1.30.0/plugins/autoloader/prism-autoloader.min.js"
|
||||
_PRISM_AUTOLOADER_INTEGRITY = "sha384-Uq05+JLko69eOiPr39ta9bh7kld5PKZoU+fF7g0EXTAriEollhZ+DrN8Q/Oi8J2Q"
|
||||
_PRISM_LANGUAGES_PATH = "https://cdn.jsdelivr.net/npm/prismjs@1.30.0/components/"
|
||||
_MERMAID_URL = "https://cdn.jsdelivr.net/npm/mermaid@11.16.1/dist/mermaid.min.js"
|
||||
_MERMAID_INTEGRITY = "sha384-aBQXj4hK6Jm05i7aQAsUV3bLdSUrHX1BGYfMB0166TtWt/RRaw+h0Eelme9OCOvy"
|
||||
_MERMAID_MAX_TEXT_SIZE = 50_000
|
||||
_MERMAID_FLOWCHART_HEADER_RE = re.compile(
|
||||
r"(?:graph|flowchart)\s+(?:TB|TD|BT|RL|LR)\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_SAFE_PRISM_LANGUAGE_RE = re.compile(r"[a-z0-9][a-z0-9-]{0,31}")
|
||||
_BLOCKED_PRISM_LANGUAGE_NAMES = {"constructor", "prototype"}
|
||||
_PRISM_LANGUAGE_ALIASES = {
|
||||
"c#": "csharp",
|
||||
"c++": "cpp",
|
||||
"html": "markup",
|
||||
"js": "javascript",
|
||||
"py": "python",
|
||||
"sh": "bash",
|
||||
"shell": "bash",
|
||||
"none": "",
|
||||
"text": "",
|
||||
"txt": "",
|
||||
"ts": "typescript",
|
||||
"xml": "markup",
|
||||
"yml": "yaml",
|
||||
}
|
||||
|
||||
|
||||
class _HtmlRenderer:
|
||||
"""维护一次 HTML 渲染所需的 anchor、公式与代码资源状态。"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
middle_json: MiddleJson,
|
||||
*,
|
||||
mode: RenderMode,
|
||||
asset_base_url: str,
|
||||
standalone: bool,
|
||||
document_title: str | None,
|
||||
) -> None:
|
||||
"""保存严格输入和调用选项,并预收集真实正文标题 anchor。"""
|
||||
self.middle_json = middle_json
|
||||
self.mode = mode
|
||||
self.asset_base_url = asset_base_url
|
||||
self.standalone = standalone
|
||||
self.document_title = document_title
|
||||
self.anchor_targets = _collect_title_anchor_ids(middle_json)
|
||||
self.emitted_anchors: set[str] = set()
|
||||
self.has_math = False
|
||||
self.has_prism = False
|
||||
self.has_mermaid = False
|
||||
|
||||
def render(self) -> str:
|
||||
"""生成 fragment article,并按需包装完整 HTML 文档。"""
|
||||
planned_pages = build_render_plan(self.middle_json, self.mode)
|
||||
if self.mode is RenderMode.FULL:
|
||||
body = self._render_full_pages(planned_pages)
|
||||
else:
|
||||
body = self._render_default_pages(planned_pages)
|
||||
article = f'<article class="mineru-document mineru-document--{self.mode.value}">\n{body}\n</article>'
|
||||
if not self.standalone:
|
||||
return article
|
||||
# 最终依赖以真实 DOM 为准,避免回退分支或普通文本中的 class 字样误触发。
|
||||
article_soup = BeautifulSoup(article, "html.parser")
|
||||
self.has_math = article_soup.select_one(".mineru-math") is not None
|
||||
self.has_prism = any(
|
||||
any(str(class_name).startswith("language-") for class_name in code.get("class", []))
|
||||
for code in article_soup.find_all("code")
|
||||
if code.get_text()
|
||||
)
|
||||
self.has_mermaid = article_soup.select_one(".mineru-flowchart") is not None
|
||||
return self._render_standalone(article)
|
||||
|
||||
def _render_default_pages(self, planned_pages: list[list[PlannedBlock]]) -> str:
|
||||
"""把 DEFAULT 计划展平为无页面边界的连续阅读内容。"""
|
||||
rendered: list[str] = []
|
||||
for page in planned_pages:
|
||||
for planned in page:
|
||||
block = self._render_planned_block(planned)
|
||||
if block:
|
||||
rendered.append(block)
|
||||
return "\n".join(rendered)
|
||||
|
||||
def _render_full_pages(self, planned_pages: list[list[PlannedBlock]]) -> str:
|
||||
"""为 FULL 计划保留所有页面 section 和相邻页面分隔。"""
|
||||
sections: list[str] = []
|
||||
for page in planned_pages:
|
||||
page_idx = page[0].page_idx if page else None
|
||||
if page_idx is None:
|
||||
page_position = len(sections)
|
||||
page_idx = self.middle_json.pages[page_position].page_idx
|
||||
blocks = [rendered for planned in page if (rendered := self._render_planned_block(planned))]
|
||||
rendered_blocks = "\n".join(blocks)
|
||||
sections.append(f'<section class="mineru-page" data-page-idx="{page_idx}">\n{rendered_blocks}\n</section>')
|
||||
return '\n<hr class="mineru-page-break" aria-hidden="true">\n'.join(sections)
|
||||
|
||||
def _render_planned_block(self, planned: PlannedBlock) -> str:
|
||||
"""过滤计划块、分派类型 visitor,并添加稳定来源元数据。"""
|
||||
if planned.removed:
|
||||
return ""
|
||||
block = planned.block
|
||||
if self.mode is RenderMode.DEFAULT and block.type in PAGE_AUXILIARY_BLOCK_TYPES:
|
||||
return ""
|
||||
content = self._render_block_content(planned)
|
||||
if not content or not content.strip():
|
||||
return ""
|
||||
attrs = [
|
||||
'class="mineru-block"',
|
||||
f'data-page-idx="{planned.page_idx}"',
|
||||
f'data-block-type="{html.escape(str(block.type), quote=True)}"',
|
||||
]
|
||||
if block.index is not None:
|
||||
attrs.append(f'data-block-index="{block.index}"')
|
||||
return f"<div {' '.join(attrs)}>\n{content}\n</div>"
|
||||
|
||||
def _render_block_content(self, planned: PlannedBlock) -> str:
|
||||
"""按严格 PageBlock 具体类型返回对应语义 HTML。"""
|
||||
block = planned.block
|
||||
if isinstance(block, TextBlock):
|
||||
rendered = render_joined_inline_contents_html(planned.text_contents or [block.content])
|
||||
self._observe_inline(rendered)
|
||||
return f'<p class="mineru-text">{rendered.html}</p>' if rendered.html else ""
|
||||
if isinstance(block, RefTextBlock):
|
||||
rendered = render_joined_inline_contents_html(planned.text_contents or [block.content])
|
||||
self._observe_inline(rendered)
|
||||
return f'<p class="mineru-ref-text">{rendered.html}</p>' if rendered.html else ""
|
||||
if isinstance(block, (DocTitleBlock, ParagraphTitleBlock)):
|
||||
return self._render_title(block)
|
||||
if isinstance(block, PageAuxTextBlock):
|
||||
rendered = render_inline_content_html(block.content)
|
||||
self._observe_inline(rendered)
|
||||
class_name = html.escape(str(block.type).replace("_", "-"), quote=True)
|
||||
return f'<div class="mineru-page-aux mineru-page-aux--{class_name}">{rendered.html}</div>' if rendered.html else ""
|
||||
if isinstance(block, EquationBlock):
|
||||
return self._render_equation(block)
|
||||
if isinstance(block, ListBlock):
|
||||
return self._render_list(block)
|
||||
if isinstance(block, IndexBlock):
|
||||
return self._render_index(block)
|
||||
if isinstance(block, ImageBlock):
|
||||
return self._render_image_block(block)
|
||||
if isinstance(block, TableBlock):
|
||||
return self._render_table_block(block)
|
||||
if isinstance(block, ChartBlock):
|
||||
return self._render_chart_block(block)
|
||||
if isinstance(block, CodeBlock):
|
||||
return self._render_code_block(block)
|
||||
raise TypeError(f"Unsupported PageBlock type: {type(block).__name__}")
|
||||
|
||||
def _render_title(self, block: DocTitleBlock | ParagraphTitleBlock) -> str:
|
||||
"""渲染标题,并保证正文 anchor 只在首次出现时生成 id。"""
|
||||
rendered = render_inline_content_html(block.content)
|
||||
self._observe_inline(rendered)
|
||||
if not rendered.html:
|
||||
return ""
|
||||
level = min(max(block.level, 1), 6)
|
||||
attrs = [f'class="mineru-heading mineru-heading--{level}"', f'data-heading-level="{block.level}"']
|
||||
anchor = _anchor_key(block.anchor)
|
||||
if anchor:
|
||||
if anchor not in self.emitted_anchors:
|
||||
anchor_id = self.anchor_targets.get(anchor)
|
||||
if anchor_id:
|
||||
attrs.append(f'id="{html.escape(anchor_id, quote=True)}"')
|
||||
self.emitted_anchors.add(anchor)
|
||||
else:
|
||||
logger.warning("Duplicate HTML title anchor omitted: {!r}", anchor)
|
||||
return f"<h{level} {' '.join(attrs)}>{rendered.html}</h{level}>"
|
||||
|
||||
def _render_equation(self, block: EquationBlock) -> str:
|
||||
"""优先输出行间公式,空 LaTeX 时回退到安全图片来源。"""
|
||||
if block.content.strip():
|
||||
rendered = render_math_html(block.content, display=True)
|
||||
self._observe_inline(rendered)
|
||||
return rendered.html
|
||||
source = self._safe_block_image_source(block)
|
||||
return self._render_image(source, alt="formula", class_name="mineru-equation-image") if source else ""
|
||||
|
||||
def _render_list(self, block: ListBlock) -> str:
|
||||
"""按直属 marker 类型渲染一层列表,并递归挂接嵌套列表。"""
|
||||
parsed_leaves = [
|
||||
parse_list_item_marker(child.content)
|
||||
for child in block.content
|
||||
if not isinstance(child, ListBlock) and child.content.strip()
|
||||
]
|
||||
add_reference_bullets = reference_list_needs_bullets(block)
|
||||
container_tag, list_type, class_name = _classify_list(parsed_leaves, add_reference_bullets)
|
||||
items: list[dict[str, object]] = []
|
||||
expected_value: int | None = None
|
||||
|
||||
for child in block.content:
|
||||
if isinstance(child, ListBlock):
|
||||
nested = self._render_list(child)
|
||||
if not nested:
|
||||
continue
|
||||
if not items:
|
||||
items.append({"content": "", "attrs": ' class="mineru-list-item--orphan"', "nested": []})
|
||||
nested_items = items[-1]["nested"]
|
||||
assert isinstance(nested_items, list)
|
||||
nested_items.append(nested)
|
||||
continue
|
||||
|
||||
parsed = parse_list_item_marker(child.content)
|
||||
item_content, marker = _list_item_content(
|
||||
parsed,
|
||||
add_reference_bullets,
|
||||
explicit_markers=class_name == "mineru-list--explicit",
|
||||
)
|
||||
rendered = render_inline_content_html(item_content)
|
||||
self._observe_inline(rendered)
|
||||
if not rendered.html and not marker:
|
||||
items.append({"content": "", "attrs": ' class="mineru-list-item--markerless"', "nested": []})
|
||||
continue
|
||||
|
||||
attrs: list[str] = []
|
||||
if class_name == "mineru-list--explicit":
|
||||
attrs.append('class="mineru-list-item--explicit"')
|
||||
if container_tag == "ol" and parsed.kind == "ordered" and parsed.value is not None:
|
||||
if expected_value is None:
|
||||
expected_value = parsed.value
|
||||
if parsed.value != expected_value:
|
||||
attrs.append(f'value="{parsed.value}"')
|
||||
expected_value = parsed.value + 1
|
||||
marker_html = (
|
||||
f'<span class="mineru-list-marker">{html.escape(_replace_html_controls(marker), quote=False)}</span>'
|
||||
if marker
|
||||
else ""
|
||||
)
|
||||
if class_name == "mineru-list--explicit" and not marker_html:
|
||||
marker_html = '<span class="mineru-list-marker"></span>'
|
||||
items.append(
|
||||
{
|
||||
"content": f'{marker_html}<span class="mineru-list-content">{rendered.html}</span>',
|
||||
"attrs": f" {' '.join(attrs)}" if attrs else "",
|
||||
"nested": [],
|
||||
}
|
||||
)
|
||||
|
||||
rendered_items: list[str] = []
|
||||
for item in items:
|
||||
nested = item["nested"]
|
||||
assert isinstance(nested, list)
|
||||
item_content = str(item["content"])
|
||||
if not item_content and not nested:
|
||||
continue
|
||||
rendered_items.append(f"<li{item['attrs']}>{item_content}{''.join(str(value) for value in nested)}</li>")
|
||||
if not rendered_items:
|
||||
return ""
|
||||
|
||||
container_attrs = [f'class="mineru-list {class_name}"']
|
||||
if list_type:
|
||||
container_attrs.append(f'type="{list_type}"')
|
||||
if container_tag == "ol" and parsed_leaves and parsed_leaves[0].value not in (None, 1):
|
||||
container_attrs.append(f'start="{parsed_leaves[0].value}"')
|
||||
return f"<{container_tag} {' '.join(container_attrs)}>{''.join(rendered_items)}</{container_tag}>"
|
||||
|
||||
def _render_index(self, block: IndexBlock) -> str:
|
||||
"""递归渲染目录,并仅链接到真实正文标题 anchor。"""
|
||||
items: list[dict[str, object]] = []
|
||||
for child in block.content:
|
||||
if isinstance(child, IndexBlock):
|
||||
nested = self._render_index_list(child)
|
||||
if not nested:
|
||||
continue
|
||||
if not items:
|
||||
items.append({"content": "", "orphan": True, "nested": []})
|
||||
nested_items = items[-1]["nested"]
|
||||
assert isinstance(nested_items, list)
|
||||
nested_items.append(nested)
|
||||
continue
|
||||
rendered = self._render_index_leaf(child)
|
||||
items.append({"content": rendered, "orphan": False, "nested": []})
|
||||
inner = _serialize_index_items(items)
|
||||
return f'<nav class="mineru-index" aria-label="Table of contents"><ul>{inner}</ul></nav>' if inner else ""
|
||||
|
||||
def _render_index_list(self, block: IndexBlock) -> str:
|
||||
"""渲染一个嵌套 IndexBlock 为可挂接到父项的 ul。"""
|
||||
items: list[dict[str, object]] = []
|
||||
for child in block.content:
|
||||
if isinstance(child, IndexBlock):
|
||||
nested = self._render_index_list(child)
|
||||
if nested:
|
||||
if not items:
|
||||
items.append({"content": "", "orphan": True, "nested": []})
|
||||
nested_items = items[-1]["nested"]
|
||||
assert isinstance(nested_items, list)
|
||||
nested_items.append(nested)
|
||||
continue
|
||||
rendered = self._render_index_leaf(child)
|
||||
items.append({"content": rendered, "orphan": False, "nested": []})
|
||||
inner = _serialize_index_items(items)
|
||||
return f"<ul>{inner}</ul>" if inner else ""
|
||||
|
||||
def _render_index_leaf(self, block: TextBlock | TitleBlockBase) -> str:
|
||||
"""渲染目录叶子,并在 anchor 命中正文标题时生成内部链接。"""
|
||||
content = strip_index_page_tail(block.content)
|
||||
rendered = render_inline_content_html(content)
|
||||
self._observe_inline(rendered)
|
||||
if not rendered.html:
|
||||
return ""
|
||||
anchor = _anchor_key(block.anchor) if isinstance(block, TitleBlockBase) else ""
|
||||
if anchor and (anchor_id := self.anchor_targets.get(anchor)):
|
||||
href = quote(anchor_id, safe="-._~")
|
||||
return f'<a href="#{href}">{rendered.html}</a>'
|
||||
return rendered.html
|
||||
|
||||
def _render_image_block(self, block: ImageBlock) -> str:
|
||||
"""按原始子块顺序渲染图片主体及重复说明。"""
|
||||
parts: list[str] = []
|
||||
for child in block.content:
|
||||
if isinstance(child, ImageBodyBlock):
|
||||
body = self._render_image_body(block, child)
|
||||
elif isinstance(child, ImageAnnotationBlock):
|
||||
body = self._render_annotation(child)
|
||||
else:
|
||||
raise TypeError(f"Unsupported image child: {type(child).__name__}")
|
||||
if body:
|
||||
parts.append(body)
|
||||
return f'<figure class="mineru-figure mineru-figure--image">{"".join(parts)}</figure>' if parts else ""
|
||||
|
||||
def _render_image_body(self, parent: ImageBlock, block: ImageBodyBlock) -> str:
|
||||
"""渲染图片,并把并存的识别内容放入原生 details。"""
|
||||
if parent.sub_type == "flowchart":
|
||||
mermaid_source = _extract_mermaid_flowchart_source(block.content)
|
||||
if mermaid_source is not None:
|
||||
return self._render_flowchart_body(block, mermaid_source)
|
||||
source = self._safe_block_image_source(block)
|
||||
content = self._render_embedded_content(block.content, linkify_text=parent.sub_type != "flowchart")
|
||||
alt = _plain_content_text(block.content) or parent.sub_type or "image"
|
||||
parts = [self._render_image(source, alt=alt, class_name="mineru-image")] if source else []
|
||||
if content.html:
|
||||
if source:
|
||||
parts.append(self._render_details(content.html, parent.sub_type or "image content"))
|
||||
else:
|
||||
parts.append(content.html)
|
||||
return "".join(parts)
|
||||
|
||||
def _render_flowchart_body(self, block: ImageBodyBlock, mermaid_source: str) -> str:
|
||||
"""输出 Mermaid canvas,并保留 raster 与源码两级失败回退。"""
|
||||
source = self._safe_block_image_source(block)
|
||||
escaped_source = html.escape(_replace_html_controls(mermaid_source), quote=False)
|
||||
fallback = self._render_image(source, alt="flowchart", class_name="mineru-flowchart-fallback") if source else ""
|
||||
fallback_class = " mineru-flowchart--has-raster" if fallback else ""
|
||||
details_open = "" if fallback else " open"
|
||||
return (
|
||||
f'<div class="mineru-flowchart{fallback_class}" data-mermaid-state="pending">'
|
||||
'<div class="mineru-flowchart-canvas" role="img" aria-label="flowchart"></div>'
|
||||
f"{fallback}</div>"
|
||||
f'<details class="mineru-details mineru-flowchart-details"{details_open}>'
|
||||
"<summary>flowchart source</summary>"
|
||||
f'<pre class="mineru-flowchart-source"><code>{escaped_source}</code></pre>'
|
||||
"</details>"
|
||||
)
|
||||
|
||||
def _render_table_block(self, block: TableBlock) -> str:
|
||||
"""按原始子块顺序渲染表格主体、标题与脚注。"""
|
||||
parts: list[str] = []
|
||||
for child in block.content:
|
||||
if isinstance(child, TableBodyBlock):
|
||||
body = self._render_table_body(child)
|
||||
elif isinstance(child, TableAnnotationBlock):
|
||||
body = self._render_annotation(child)
|
||||
else:
|
||||
raise TypeError(f"Unsupported table child: {type(child).__name__}")
|
||||
if body:
|
||||
parts.append(body)
|
||||
return f'<figure class="mineru-figure mineru-figure--table">{"".join(parts)}</figure>' if parts else ""
|
||||
|
||||
def _render_table_body(self, block: TableBodyBlock) -> str:
|
||||
"""优先保留安全 HTML table,空间文本和整体图片依次回退。"""
|
||||
normalized_content = _replace_html_controls(block.content)
|
||||
content = normalized_content.strip()
|
||||
if content:
|
||||
source_soup = BeautifulSoup(content, "html.parser")
|
||||
if source_soup.find("table") is not None:
|
||||
rendered = self._render_embedded_content(content)
|
||||
if rendered.html and _contains_usable_table(rendered.html):
|
||||
return rendered.html
|
||||
source = self._safe_block_image_source(block)
|
||||
if source:
|
||||
return self._render_image(source, alt="table", class_name="mineru-table-image")
|
||||
return _render_raw_fallback(content)
|
||||
return f'<pre class="mineru-table-text">{html.escape(normalized_content, quote=False)}</pre>'
|
||||
source = self._safe_block_image_source(block)
|
||||
return self._render_image(source, alt="table", class_name="mineru-table-image") if source else ""
|
||||
|
||||
def _render_chart_block(self, block: ChartBlock) -> str:
|
||||
"""按原始子块顺序渲染 chart 图片、结构内容及说明。"""
|
||||
parts: list[str] = []
|
||||
for child in block.content:
|
||||
if isinstance(child, ChartBodyBlock):
|
||||
body = self._render_chart_body(block, child)
|
||||
elif isinstance(child, ChartAnnotationBlock):
|
||||
body = self._render_annotation(child)
|
||||
else:
|
||||
raise TypeError(f"Unsupported chart child: {type(child).__name__}")
|
||||
if body:
|
||||
parts.append(body)
|
||||
return f'<figure class="mineru-figure mineru-figure--chart">{"".join(parts)}</figure>' if parts else ""
|
||||
|
||||
def _render_chart_body(self, parent: ChartBlock, block: ChartBodyBlock) -> str:
|
||||
"""渲染 chart 图片,并将并存结构内容放入 details。"""
|
||||
source = self._safe_block_image_source(block)
|
||||
content = self._render_chart_content(block.content)
|
||||
parts = [self._render_image(source, alt=parent.sub_type or "chart", class_name="mineru-chart-image")] if source else []
|
||||
if content.html:
|
||||
if source:
|
||||
parts.append(self._render_details(content.html, parent.sub_type or "chart content"))
|
||||
else:
|
||||
parts.append(content.html)
|
||||
return "".join(parts)
|
||||
|
||||
def _render_chart_content(self, content: str) -> HtmlInlineResult:
|
||||
"""按 HTML、严格 GFM 表格、普通行内内容的顺序渲染 chart content。"""
|
||||
normalized = content.strip()
|
||||
if not normalized:
|
||||
return HtmlInlineResult("")
|
||||
if is_supported_html_markup(normalized):
|
||||
return self._render_embedded_content(normalized)
|
||||
if looks_like_gfm_table(normalized):
|
||||
rendered = render_gfm_table_html(normalized)
|
||||
if rendered is None:
|
||||
return HtmlInlineResult(_render_raw_fallback(content))
|
||||
self._observe_inline(rendered)
|
||||
return rendered
|
||||
rendered = render_inline_content_html(content)
|
||||
self._observe_inline(rendered)
|
||||
return rendered
|
||||
|
||||
def _render_code_block(self, block: CodeBlock) -> str:
|
||||
"""按原始子块顺序渲染代码或算法及其说明。"""
|
||||
parts: list[str] = []
|
||||
for child in block.content:
|
||||
if isinstance(child, CodeBodyBlock):
|
||||
body = self._render_code_body(block, child)
|
||||
elif isinstance(child, CodeAnnotationBlock):
|
||||
body = self._render_annotation(child)
|
||||
else:
|
||||
raise TypeError(f"Unsupported code child: {type(child).__name__}")
|
||||
if body:
|
||||
parts.append(body)
|
||||
return f'<figure class="mineru-figure mineru-figure--code">{"".join(parts)}</figure>' if parts else ""
|
||||
|
||||
def _render_code_body(self, parent: CodeBlock, block: CodeBodyBlock) -> str:
|
||||
"""代码使用 Prism class,算法使用共享 inline AST 保留空白。"""
|
||||
if parent.sub_type == BlockType.CODE:
|
||||
escaped = html.escape(_replace_html_controls(block.content), quote=False)
|
||||
if not block.content:
|
||||
return '<pre class="mineru-code"><code></code></pre>'
|
||||
language = _normalize_prism_language(parent.guess_lang)
|
||||
class_attr = f' class="language-{language}"' if language else ""
|
||||
pre_class = f"mineru-code language-{language}" if language else "mineru-code"
|
||||
self.has_prism = bool(language) or self.has_prism
|
||||
return f'<pre class="{pre_class}"><code{class_attr}>{escaped}</code></pre>'
|
||||
if parent.sub_type == RAW_ALGORITHM:
|
||||
rendered = render_inline_nodes_html(
|
||||
parse_inline_content(block.content),
|
||||
linkify_text=False,
|
||||
separate_adjacent_math=True,
|
||||
preserve_newlines=True,
|
||||
)
|
||||
self._observe_inline(rendered)
|
||||
return f'<div class="mineru-algorithm">{rendered.html}</div>' if rendered.html else ""
|
||||
raise ValueError(f"Unsupported code subtype: {parent.sub_type}")
|
||||
|
||||
def _render_annotation(
|
||||
self,
|
||||
block: ImageAnnotationBlock | TableAnnotationBlock | ChartAnnotationBlock | CodeAnnotationBlock,
|
||||
) -> str:
|
||||
"""把视觉说明渲染为保持原类型的独立段落。"""
|
||||
rendered = render_inline_content_html(block.content)
|
||||
self._observe_inline(rendered)
|
||||
if not rendered.html:
|
||||
return ""
|
||||
is_caption = str(block.type).endswith("caption")
|
||||
role_class = "mineru-caption" if is_caption else "mineru-footnote"
|
||||
type_class = html.escape(str(block.type).replace("_", "-"), quote=True)
|
||||
return f'<p class="{role_class} {role_class}--{type_class}">{rendered.html}</p>'
|
||||
|
||||
def _render_embedded_content(self, content: str, *, linkify_text: bool = True) -> HtmlInlineResult:
|
||||
"""安全处理 body 富 HTML;普通内容走共享 inline AST。"""
|
||||
normalized = content.strip()
|
||||
if not normalized:
|
||||
return HtmlInlineResult("")
|
||||
if not is_supported_html_markup(normalized):
|
||||
rendered = render_inline_nodes_html(parse_inline_content(content), linkify_text=linkify_text)
|
||||
self._observe_inline(rendered)
|
||||
return rendered
|
||||
sanitized = sanitize_html_fragment(normalized, asset_base_url=self.asset_base_url)
|
||||
rendered = self._replace_safe_equations(sanitized)
|
||||
self._observe_inline(rendered)
|
||||
return rendered
|
||||
|
||||
def _replace_safe_equations(self, markup: str) -> HtmlInlineResult:
|
||||
"""把 sanitizer 保留的 eq 标签替换为可信 MathJax carrier。"""
|
||||
if not markup:
|
||||
return HtmlInlineResult("")
|
||||
soup = BeautifulSoup(markup, "html.parser")
|
||||
has_math = False
|
||||
for equation in list(soup.find_all("eq")):
|
||||
rendered = render_math_html(equation.get_text(), display=False)
|
||||
if not rendered.html:
|
||||
equation.decompose()
|
||||
continue
|
||||
replacement = BeautifulSoup(rendered.html, "html.parser")
|
||||
equation.replace_with(*list(replacement.contents))
|
||||
has_math = True
|
||||
return HtmlInlineResult(str(soup), has_math)
|
||||
|
||||
def _safe_block_image_source(self, block: EquationBlock | ImageBodyBlock | TableBodyBlock | ChartBodyBlock) -> str | None:
|
||||
"""解析 block 图片来源,并应用 HTML renderer 的 URL 安全策略。"""
|
||||
if block.image_path:
|
||||
return sanitize_image_source(block.image_path, asset_base_url=self.asset_base_url)
|
||||
if block.image_base64:
|
||||
return sanitize_image_source(block.image_base64)
|
||||
return None
|
||||
|
||||
def _render_image(self, source: str, *, alt: str, class_name: str) -> str:
|
||||
"""构造属性已转义的安全 img 元素。"""
|
||||
return (
|
||||
f'<img class="{class_name}" src="{html.escape(source, quote=True)}" '
|
||||
f'alt="{html.escape(_replace_html_controls(alt), quote=True)}">'
|
||||
)
|
||||
|
||||
def _render_details(self, content: str, summary: str) -> str:
|
||||
"""构造浏览器原生折叠详情,不引入额外脚本。"""
|
||||
return (
|
||||
'<details class="mineru-details">'
|
||||
f"<summary>{html.escape(_replace_html_controls(summary), quote=False)}</summary>{content}</details>"
|
||||
)
|
||||
|
||||
def _observe_inline(self, rendered: HtmlInlineResult) -> None:
|
||||
"""累计当前文档是否需要加载 MathJax。"""
|
||||
self.has_math = rendered.has_math or self.has_math
|
||||
|
||||
def _render_standalone(self, article: str) -> str:
|
||||
"""用固定 CSS 和按需 CDN 资源包装完整 HTML 文档。"""
|
||||
title = html.escape(
|
||||
_replace_html_controls(_resolve_document_title(self.middle_json, self.document_title)),
|
||||
quote=False,
|
||||
)
|
||||
styles = _load_html_styles()
|
||||
dependencies = []
|
||||
if self.has_math:
|
||||
dependencies.append(_mathjax_head())
|
||||
if self.has_prism:
|
||||
dependencies.append(_prism_head())
|
||||
if self.has_mermaid:
|
||||
dependencies.append(_mermaid_head())
|
||||
dependency_html = "\n".join(dependencies)
|
||||
return (
|
||||
"<!doctype html>\n"
|
||||
'<html lang="und">\n'
|
||||
"<head>\n"
|
||||
'<meta charset="utf-8">\n'
|
||||
'<meta name="viewport" content="width=device-width, initial-scale=1.0">\n'
|
||||
f"<title>{title}</title>\n"
|
||||
f"<style>{styles}</style>\n"
|
||||
f"{dependency_html}\n"
|
||||
"</head>\n"
|
||||
'<body class="mineru-html-body">\n'
|
||||
f"{article}\n"
|
||||
"</body>\n"
|
||||
"</html>"
|
||||
)
|
||||
|
||||
|
||||
def render_html(
|
||||
middle_json: MiddleJson,
|
||||
*,
|
||||
mode: RenderMode = RenderMode.DEFAULT,
|
||||
asset_base_url: str = "",
|
||||
standalone: bool = True,
|
||||
document_title: str | None = None,
|
||||
) -> str:
|
||||
"""把严格 MiddleJson 无副作用地渲染为 HTML 文档或单根 fragment。"""
|
||||
if not isinstance(middle_json, MiddleJson):
|
||||
raise TypeError("render_html expects a MiddleJson instance")
|
||||
if not isinstance(mode, RenderMode):
|
||||
raise TypeError("mode must be a RenderMode value")
|
||||
if not isinstance(asset_base_url, str):
|
||||
raise TypeError("asset_base_url must be a string")
|
||||
if not isinstance(standalone, bool):
|
||||
raise TypeError("standalone must be a bool")
|
||||
if document_title is not None and not isinstance(document_title, str):
|
||||
raise TypeError("document_title must be a string or None")
|
||||
return _HtmlRenderer(
|
||||
middle_json,
|
||||
mode=mode,
|
||||
asset_base_url=asset_base_url,
|
||||
standalone=standalone,
|
||||
document_title=document_title,
|
||||
).render()
|
||||
|
||||
|
||||
def _classify_list(items: list[ListItem], add_reference_bullets: bool) -> tuple[str, str | None, str]:
|
||||
"""根据直属 marker 选择原生列表类型或显式 marker 模式。"""
|
||||
if add_reference_bullets:
|
||||
return "ul", None, "mineru-list--reference"
|
||||
if items and all(item.kind == "unordered" for item in items):
|
||||
return "ul", None, "mineru-list--unordered"
|
||||
if items and all(item.kind == "ordered" for item in items):
|
||||
styles = {item.ordered_style for item in items}
|
||||
if len(styles) == 1:
|
||||
style = next(iter(styles))
|
||||
list_type = {
|
||||
"lower-alpha": "a",
|
||||
"upper-alpha": "A",
|
||||
"lower-roman": "i",
|
||||
"upper-roman": "I",
|
||||
}.get(style or "")
|
||||
return "ol", list_type, "mineru-list--ordered"
|
||||
if items and all(item.kind == "none" for item in items):
|
||||
return "ul", None, "mineru-list--unmarked"
|
||||
return "ul", None, "mineru-list--explicit"
|
||||
|
||||
|
||||
def _list_item_content(
|
||||
item: ListItem,
|
||||
add_reference_bullets: bool,
|
||||
*,
|
||||
explicit_markers: bool,
|
||||
) -> tuple[str, str | None]:
|
||||
"""决定一个列表项应剥离、保留还是显式显示源 marker。"""
|
||||
if add_reference_bullets:
|
||||
if item.kind == "unordered":
|
||||
return item.body, None
|
||||
original = f"{item.leading}{item.marker or ''}{item.separator}{item.body}"
|
||||
return original, None
|
||||
if explicit_markers:
|
||||
return item.body, item.marker
|
||||
if item.kind in {"unordered", "ordered"}:
|
||||
return item.body, None
|
||||
return item.body, item.marker
|
||||
|
||||
|
||||
def _serialize_index_items(items: list[dict[str, object]]) -> str:
|
||||
"""把目录项及其已挂接的嵌套 ul 序列化为 li。"""
|
||||
rendered: list[str] = []
|
||||
for item in items:
|
||||
nested = item["nested"]
|
||||
assert isinstance(nested, list)
|
||||
content = str(item["content"])
|
||||
if not content and not nested:
|
||||
continue
|
||||
class_attr = ' class="mineru-list-item--orphan"' if item["orphan"] else ""
|
||||
rendered.append(f"<li{class_attr}>{content}{''.join(str(value) for value in nested)}</li>")
|
||||
return "".join(rendered)
|
||||
|
||||
|
||||
def _collect_title_anchor_ids(middle_json: MiddleJson) -> dict[str, str]:
|
||||
"""为正文首次出现的非空 anchor 分配无空白且 document-wide 唯一的 HTML id。"""
|
||||
anchor_ids: dict[str, str] = {}
|
||||
used_ids: set[str] = set()
|
||||
for page in middle_json.pages:
|
||||
for block in page.blocks:
|
||||
if not isinstance(block, TitleBlockBase):
|
||||
continue
|
||||
anchor = _anchor_key(block.anchor)
|
||||
visible_title = inline_plain_text(parse_inline_content(block.content)).strip()
|
||||
if not anchor or not visible_title or anchor in anchor_ids:
|
||||
continue
|
||||
base_id = _html_anchor_base(anchor)
|
||||
candidate = base_id
|
||||
suffix = 2
|
||||
while candidate in used_ids:
|
||||
candidate = f"{base_id}-{suffix}"
|
||||
suffix += 1
|
||||
anchor_ids[anchor] = candidate
|
||||
used_ids.add(candidate)
|
||||
return anchor_ids
|
||||
|
||||
|
||||
def _plain_content_text(content: str) -> str:
|
||||
"""从 body 内容提取安全 img alt 所需的可见纯文本。"""
|
||||
if not content:
|
||||
return ""
|
||||
if is_supported_html_markup(content):
|
||||
return BeautifulSoup(content, "html.parser").get_text(" ", strip=True)
|
||||
return inline_plain_text(parse_inline_content(content)).strip()
|
||||
|
||||
|
||||
def _extract_mermaid_flowchart_source(content: str) -> str | None:
|
||||
"""提取受限 Mermaid flowchart fence,拒绝其他图类型和可改写配置的语法。"""
|
||||
normalized = _replace_html_controls(content).replace("\r\n", "\n").replace("\r", "\n").strip()
|
||||
if not normalized:
|
||||
return None
|
||||
lines = normalized.split("\n")
|
||||
if len(lines) < 3:
|
||||
return None
|
||||
opening = re.fullmatch(r"(?P<fence>\x60{3,})[ \t]*mermaid[ \t]*", lines[0], re.IGNORECASE)
|
||||
if opening is None:
|
||||
return None
|
||||
fence_length = len(opening.group("fence"))
|
||||
if re.fullmatch(rf"\x60{{{fence_length},}}[ \t]*", lines[-1]) is None:
|
||||
return None
|
||||
source = "\n".join(lines[1:-1]).strip("\n")
|
||||
if not source or len(source) > _MERMAID_MAX_TEXT_SIZE:
|
||||
return None
|
||||
if source.lstrip().startswith("---") or "%%{" in source:
|
||||
return None
|
||||
first_statement = next(
|
||||
(line.strip() for line in source.splitlines() if line.strip() and not line.lstrip().startswith("%%")),
|
||||
"",
|
||||
)
|
||||
if _MERMAID_FLOWCHART_HEADER_RE.match(first_statement) is None:
|
||||
return None
|
||||
return source
|
||||
|
||||
|
||||
def _contains_usable_table(markup: str) -> bool:
|
||||
"""确认清洗后的 markup 至少保留一个含单元格的 table。"""
|
||||
soup = BeautifulSoup(markup, "html.parser")
|
||||
return any(table.find(("td", "th")) is not None for table in soup.find_all("table"))
|
||||
|
||||
|
||||
def _render_raw_fallback(content: str) -> str:
|
||||
"""把无法安全结构化的原内容转义为可见 pre,避免静默丢失。"""
|
||||
return f'<pre class="mineru-raw-fallback">{html.escape(_replace_html_controls(content), quote=False)}</pre>'
|
||||
|
||||
|
||||
def _normalize_prism_language(language: str | None) -> str | None:
|
||||
"""把模型语言名归一化为不可构造路径穿越的 Prism component 名。"""
|
||||
normalized = (language or "").strip().lower()
|
||||
normalized = _PRISM_LANGUAGE_ALIASES.get(normalized, normalized.replace("_", "-"))
|
||||
if not normalized or normalized in _BLOCKED_PRISM_LANGUAGE_NAMES:
|
||||
return None
|
||||
return normalized if _SAFE_PRISM_LANGUAGE_RE.fullmatch(normalized) else None
|
||||
|
||||
|
||||
def _resolve_document_title(middle_json: MiddleJson, explicit_title: str | None) -> str:
|
||||
"""按显式参数、首个正文文档标题、固定回退值解析 HTML title。"""
|
||||
if explicit_title and explicit_title.strip():
|
||||
return explicit_title.strip()
|
||||
for page in middle_json.pages:
|
||||
for block in page.blocks:
|
||||
if isinstance(block, DocTitleBlock):
|
||||
title = inline_plain_text(parse_inline_content(block.content)).strip()
|
||||
if title:
|
||||
return title
|
||||
return "MinerU Document"
|
||||
|
||||
|
||||
def _anchor_key(anchor: str | None) -> str:
|
||||
"""保留 producer anchor 身份,仅去除首尾空白供标题与目录精确匹配。"""
|
||||
return (anchor or "").strip()
|
||||
|
||||
|
||||
def _html_anchor_base(anchor: str) -> str:
|
||||
"""把原始 anchor 转成无空白的 HTML id 基值,碰撞由 document registry 处理。"""
|
||||
return re.sub(r"\s+", "-", _replace_html_controls(anchor))
|
||||
|
||||
|
||||
def _replace_html_controls(content: str) -> str:
|
||||
"""替换 HTML 不允许的 C0 控制字符与不可编码的孤立 surrogate。"""
|
||||
return re.sub(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f\ud800-\udfff]", "\ufffd", content)
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _load_html_styles() -> str:
|
||||
"""从包资源读取并缓存压缩后的 MinerU 独立样式。"""
|
||||
root = resources.files("mineru").joinpath("resources", "html")
|
||||
return root.joinpath(_STYLE_RESOURCE_NAME).read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def _mathjax_head() -> str:
|
||||
"""返回固定、收紧公式输入能力的 MathJax 4.1.2 head 片段。"""
|
||||
return f"""<script>
|
||||
window.MathJax = {{
|
||||
loader: {{load: ['ui/safe']}},
|
||||
tex: {{
|
||||
inlineMath: [['\\\\(', '\\\\)']],
|
||||
displayMath: [['\\\\[', '\\\\]']],
|
||||
processEscapes: true,
|
||||
processEnvironments: false,
|
||||
processRefs: false,
|
||||
packages: {{'[-]': ['require']}}
|
||||
}},
|
||||
options: {{
|
||||
ignoreHtmlClass: 'mineru-document',
|
||||
processHtmlClass: 'mineru-math',
|
||||
enableMenu: false,
|
||||
enableEnrichment: false,
|
||||
safeOptions: {{allow: {{URLs: 'none', classes: 'none', cssIDs: 'none', styles: 'none'}}}}
|
||||
}},
|
||||
startup: {{
|
||||
ready: function () {{
|
||||
window.MathJax.startup.defaultReady();
|
||||
var options = window.MathJax.startup.document.options;
|
||||
options.enableMenu = false;
|
||||
options.enableEnrichment = false;
|
||||
}}
|
||||
}}
|
||||
}};
|
||||
</script>
|
||||
<script id="MathJax-script" defer src="{_MATHJAX_URL}"
|
||||
integrity="{_MATHJAX_INTEGRITY}" crossorigin="anonymous" referrerpolicy="no-referrer"></script>"""
|
||||
|
||||
|
||||
def _prism_head() -> str:
|
||||
"""返回固定 Prism core、Autoloader 和当前文档局部高亮脚本。"""
|
||||
return f"""<script defer data-manual src="{_PRISM_CORE_URL}"
|
||||
integrity="{_PRISM_CORE_INTEGRITY}" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
|
||||
<script defer src="{_PRISM_AUTOLOADER_URL}"
|
||||
integrity="{_PRISM_AUTOLOADER_INTEGRITY}" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
|
||||
<script>
|
||||
document.addEventListener('DOMContentLoaded', function () {{
|
||||
if (!window.Prism || !Prism.plugins || !Prism.plugins.autoloader) return;
|
||||
Prism.plugins.autoloader.languages_path = '{_PRISM_LANGUAGES_PATH}';
|
||||
var root = document.querySelector('.mineru-document');
|
||||
if (root) Prism.highlightAllUnder(root);
|
||||
}});
|
||||
</script>"""
|
||||
|
||||
|
||||
def _mermaid_head() -> str:
|
||||
"""返回固定 Mermaid 入口和逐图安全渲染、失败回退脚本。"""
|
||||
return f"""<script defer src="{_MERMAID_URL}"
|
||||
integrity="{_MERMAID_INTEGRITY}" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
|
||||
<script>
|
||||
function markMineruMermaidError(host) {{
|
||||
host.dataset.mermaidState = 'error';
|
||||
if (!host.classList.contains('mineru-flowchart--has-raster')) {{
|
||||
var details = host.nextElementSibling;
|
||||
if (details && details.classList.contains('mineru-flowchart-details')) details.open = true;
|
||||
}}
|
||||
}}
|
||||
document.addEventListener('DOMContentLoaded', function () {{
|
||||
var root = document.querySelector('.mineru-document');
|
||||
var hosts = root ? Array.from(root.querySelectorAll('.mineru-flowchart')) : [];
|
||||
if (!hosts.length) return;
|
||||
if (!window.mermaid) {{
|
||||
hosts.forEach(markMineruMermaidError);
|
||||
return;
|
||||
}}
|
||||
try {{
|
||||
window.mermaid.initialize({{
|
||||
startOnLoad: false,
|
||||
securityLevel: 'strict',
|
||||
suppressErrorRendering: true,
|
||||
theme: 'neutral',
|
||||
maxTextSize: 50000,
|
||||
maxEdges: 500,
|
||||
flowchart: {{htmlLabels: false, useMaxWidth: true}}
|
||||
}});
|
||||
}} catch (_error) {{
|
||||
hosts.forEach(markMineruMermaidError);
|
||||
return;
|
||||
}}
|
||||
(async function () {{
|
||||
for (var index = 0; index < hosts.length; index += 1) {{
|
||||
var host = hosts[index];
|
||||
var details = host.nextElementSibling;
|
||||
var source = details && details.querySelector('.mineru-flowchart-source code');
|
||||
var canvas = host.querySelector('.mineru-flowchart-canvas');
|
||||
if (!source || !canvas) {{
|
||||
markMineruMermaidError(host);
|
||||
continue;
|
||||
}}
|
||||
var renderId = 'mineru-mermaid-' + index;
|
||||
while (document.getElementById(renderId)) renderId += '-x';
|
||||
host.dataset.mermaidState = 'rendering';
|
||||
try {{
|
||||
var result = await window.mermaid.render(renderId, source.textContent);
|
||||
canvas.innerHTML = result.svg;
|
||||
host.dataset.mermaidState = 'rendered';
|
||||
}} catch (_error) {{
|
||||
markMineruMermaidError(host);
|
||||
}}
|
||||
}}
|
||||
}})();
|
||||
}});
|
||||
</script>"""
|
||||
|
||||
|
||||
__all__ = ["render_html"]
|
||||
@@ -10,7 +10,7 @@ from urllib.parse import quote, unquote, urlsplit
|
||||
import nh3
|
||||
from bs4 import BeautifulSoup, NavigableString, Tag
|
||||
|
||||
from mineru.utils.image_payload import parse_image_data_uri_strict
|
||||
from ....utils.image_payload import parse_image_data_uri_strict
|
||||
|
||||
|
||||
_ALLOWED_TAGS = {
|
||||
|
||||
@@ -5,7 +5,7 @@ from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from mineru.render._internal.html.inline import HtmlInlineResult, render_inline_content_html
|
||||
from .inline import HtmlInlineResult, render_inline_content_html
|
||||
|
||||
_SEPARATOR_CELL_RE = re.compile(r"^:?-{3,}:?$")
|
||||
|
||||
|
||||
@@ -1 +1,4 @@
|
||||
# Copyright (c) Opendatalab. All rights reserved.
|
||||
"""Markdown 与 Content List 共用的内部实现。"""
|
||||
|
||||
__all__: list[str] = []
|
||||
|
||||
@@ -6,7 +6,7 @@ from __future__ import annotations
|
||||
import re
|
||||
from urllib.parse import quote
|
||||
|
||||
from mineru.types import ImagePayloadBlock
|
||||
from ....types import ImagePayloadBlock
|
||||
|
||||
_HTML_IMAGE_SRC_RE = re.compile(
|
||||
r"(?P<prefix>\bsrc\s*=\s*)(?P<quote>[\"'])(?P<src>.*?)(?P=quote)",
|
||||
|
||||
@@ -6,13 +6,13 @@ from __future__ import annotations
|
||||
import html
|
||||
import re
|
||||
|
||||
from mineru.config import LatexDelimitersConfig
|
||||
from mineru.render._internal.common.index import strip_index_page_tail
|
||||
from mineru.render._internal.common.list_items import has_markdown_unordered_marker, reference_list_needs_bullets
|
||||
from mineru.render._internal.common.planner import PlannedBlock
|
||||
from mineru.render._internal.markdown.assets import build_markdown_image, resolve_image_source
|
||||
from mineru.render._internal.markdown.escaping import escape_standalone_marker_rule, escape_text_block_markdown_prefix
|
||||
from mineru.render._internal.markdown.inline import (
|
||||
from ....config import LatexDelimitersConfig
|
||||
from ..common.index import strip_index_page_tail
|
||||
from ..common.list_items import has_markdown_unordered_marker, reference_list_needs_bullets
|
||||
from ..common.planner import PlannedBlock
|
||||
from .assets import build_markdown_image, resolve_image_source
|
||||
from .escaping import escape_standalone_marker_rule, escape_text_block_markdown_prefix
|
||||
from .inline import (
|
||||
render_inline_content,
|
||||
render_internal_link,
|
||||
render_joined_inline_contents,
|
||||
@@ -23,7 +23,7 @@ from .table import (
|
||||
render_html_table,
|
||||
)
|
||||
from ...contracts import ImageRenderer
|
||||
from mineru.types import (
|
||||
from ....types import (
|
||||
RAW_ALGORITHM,
|
||||
BlockType,
|
||||
ChartAnnotationBlock,
|
||||
|
||||
@@ -6,8 +6,8 @@ from __future__ import annotations
|
||||
import html
|
||||
import re
|
||||
|
||||
from mineru.config import LatexDelimitersConfig
|
||||
from mineru.render._internal.common.inline import (
|
||||
from ....config import LatexDelimitersConfig
|
||||
from ....backend.postprocess.inline import (
|
||||
InlineEquation,
|
||||
InlineLink,
|
||||
InlineNode,
|
||||
@@ -17,7 +17,7 @@ from mineru.render._internal.common.inline import (
|
||||
join_inline_contents,
|
||||
parse_inline_content,
|
||||
)
|
||||
from mineru.render._internal.markdown.escaping import escape_conservative_markdown_text
|
||||
from .escaping import escape_conservative_markdown_text
|
||||
|
||||
_ANGLE_TEXT_RE = re.compile(r"<[^<>\n]*>")
|
||||
_SIMPLE_STYLE_WRAPPERS = {
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
# Copyright (c) Opendatalab. All rights reserved.
|
||||
"""严格 MiddleJson 到 Markdown 的公共渲染实现。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from ....config import LatexDelimitersConfig, config
|
||||
from ....types import PAGE_AUXILIARY_BLOCK_TYPES, MiddleJson
|
||||
from ...contracts import ImageRenderer, RenderMode
|
||||
from ..common.planner import PlannedBlock, build_render_plan
|
||||
from .blocks import render_planned_block
|
||||
|
||||
_PAGE_SEPARATOR = "\n\n---\n\n"
|
||||
|
||||
|
||||
def render_markdown(
|
||||
middle_json: MiddleJson,
|
||||
*,
|
||||
mode: RenderMode = RenderMode.DEFAULT,
|
||||
asset_base_url: str = "",
|
||||
image_renderer: ImageRenderer | None = None,
|
||||
) -> str:
|
||||
"""把严格 MiddleJson 纯函数式渲染为 Markdown 字符串。"""
|
||||
if not isinstance(middle_json, MiddleJson):
|
||||
raise TypeError("render_markdown expects a MiddleJson instance")
|
||||
if not isinstance(mode, RenderMode):
|
||||
raise TypeError("mode must be a RenderMode value")
|
||||
|
||||
delimiters = config.render.latex_delimiters
|
||||
planned_pages = build_render_plan(middle_json, mode)
|
||||
rendered_pages = [
|
||||
_render_page(
|
||||
page,
|
||||
mode=mode,
|
||||
delimiters=delimiters,
|
||||
asset_base_url=asset_base_url,
|
||||
image_renderer=image_renderer,
|
||||
)
|
||||
for page in planned_pages
|
||||
]
|
||||
if mode is RenderMode.FULL:
|
||||
return _PAGE_SEPARATOR.join(rendered_pages)
|
||||
return "\n\n".join(page for page in rendered_pages if page)
|
||||
|
||||
|
||||
def _render_page(
|
||||
planned_blocks: list[PlannedBlock],
|
||||
*,
|
||||
mode: RenderMode,
|
||||
delimiters: LatexDelimitersConfig,
|
||||
asset_base_url: str,
|
||||
image_renderer: ImageRenderer | None = None,
|
||||
) -> str:
|
||||
"""渲染单页逻辑块,并在默认模式中过滤重复页元素。"""
|
||||
rendered: list[str] = []
|
||||
for planned in planned_blocks:
|
||||
if planned.removed:
|
||||
continue
|
||||
if mode is RenderMode.DEFAULT and planned.block.type in PAGE_AUXILIARY_BLOCK_TYPES:
|
||||
continue
|
||||
text = render_planned_block(
|
||||
planned,
|
||||
delimiters=delimiters,
|
||||
asset_base_url=asset_base_url,
|
||||
image_renderer=image_renderer,
|
||||
)
|
||||
if text and text.strip():
|
||||
rendered.append(text.strip("\n"))
|
||||
return "\n\n".join(rendered)
|
||||
|
||||
|
||||
__all__ = ["render_markdown"]
|
||||
@@ -0,0 +1,4 @@
|
||||
# Copyright (c) Opendatalab. All rights reserved.
|
||||
"""Structured Content 私有渲染实现。"""
|
||||
|
||||
__all__: list[str] = []
|
||||
@@ -0,0 +1,181 @@
|
||||
# Copyright (c) Opendatalab. All rights reserved.
|
||||
"""严格 MiddleJson 到树形 Markdown structured_content 的公共渲染实现。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, TypeAlias
|
||||
|
||||
from ....config import LatexDelimitersConfig, config
|
||||
from ..markdown.blocks import (
|
||||
render_single_block,
|
||||
render_title_inline_content,
|
||||
render_visual_annotation,
|
||||
render_visual_body_content,
|
||||
)
|
||||
from ..markdown.assets import normalize_image_source, resolve_image_source
|
||||
from ..markdown.escaping import escape_standalone_marker_rule, escape_text_block_markdown_prefix
|
||||
from ....types import (
|
||||
BlockType,
|
||||
ChartAnnotationBlock,
|
||||
ChartBlock,
|
||||
ChartBodyBlock,
|
||||
CodeAnnotationBlock,
|
||||
CodeBlock,
|
||||
DocTitleBlock,
|
||||
EquationBlock,
|
||||
ImageAnnotationBlock,
|
||||
ImageBlock,
|
||||
ImageBodyBlock,
|
||||
ImagePayloadBlock,
|
||||
MiddleJson,
|
||||
PageBlock,
|
||||
ParagraphTitleBlock,
|
||||
TableAnnotationBlock,
|
||||
TableBlock,
|
||||
TableBodyBlock,
|
||||
)
|
||||
|
||||
VisualBlock: TypeAlias = ImageBlock | TableBlock | ChartBlock | CodeBlock
|
||||
VisualAnnotationBlock: TypeAlias = ImageAnnotationBlock | TableAnnotationBlock | ChartAnnotationBlock | CodeAnnotationBlock
|
||||
|
||||
_CAPTION_TYPES = {
|
||||
BlockType.IMAGE_CAPTION,
|
||||
BlockType.TABLE_CAPTION,
|
||||
BlockType.CHART_CAPTION,
|
||||
BlockType.CODE_CAPTION,
|
||||
}
|
||||
_FOOTNOTE_TYPES = {
|
||||
BlockType.IMAGE_FOOTNOTE,
|
||||
BlockType.TABLE_FOOTNOTE,
|
||||
BlockType.CHART_FOOTNOTE,
|
||||
BlockType.CODE_FOOTNOTE,
|
||||
}
|
||||
_REMOVED_BLOCK_FIELDS = {"content", "index", "guess_lang", "image_path", "image_base64"}
|
||||
|
||||
|
||||
def render_structured_content(
|
||||
middle_json: MiddleJson,
|
||||
*,
|
||||
asset_base_url: str = "",
|
||||
) -> dict[str, Any]:
|
||||
"""把严格 MiddleJson 无副作用地渲染为树形 Markdown structured_content。"""
|
||||
if not isinstance(middle_json, MiddleJson):
|
||||
raise TypeError("render_structured_content expects a MiddleJson instance")
|
||||
|
||||
delimiters = config.render.latex_delimiters
|
||||
document_fields = middle_json.model_dump(
|
||||
mode="json",
|
||||
exclude={"pages"},
|
||||
exclude_defaults=True,
|
||||
)
|
||||
pages = [
|
||||
{
|
||||
"page_idx": page.page_idx,
|
||||
"blocks": [
|
||||
_render_content_block(
|
||||
block,
|
||||
delimiters=delimiters,
|
||||
asset_base_url=asset_base_url,
|
||||
)
|
||||
for block in page.blocks
|
||||
],
|
||||
}
|
||||
for page in middle_json.pages
|
||||
]
|
||||
return {"pages": pages, **document_fields}
|
||||
|
||||
|
||||
def _render_content_block(
|
||||
block: PageBlock,
|
||||
*,
|
||||
delimiters: LatexDelimitersConfig,
|
||||
asset_base_url: str,
|
||||
) -> dict[str, Any]:
|
||||
"""保留父块元数据,并把 block 内容收敛为 Markdown 字符串。"""
|
||||
payload = block.model_dump(
|
||||
mode="json",
|
||||
exclude=_REMOVED_BLOCK_FIELDS,
|
||||
exclude_defaults=True,
|
||||
)
|
||||
if isinstance(block, (DocTitleBlock, ParagraphTitleBlock)):
|
||||
content = render_title_inline_content(block, delimiters)
|
||||
payload["content"] = escape_standalone_marker_rule(escape_text_block_markdown_prefix(content))
|
||||
return payload
|
||||
if isinstance(block, EquationBlock):
|
||||
payload["content"] = block.content.strip()
|
||||
image_source = _resolve_content_image_source(block, asset_base_url)
|
||||
if image_source is not None:
|
||||
payload["image_source"] = image_source
|
||||
return payload
|
||||
if not isinstance(block, (ImageBlock, TableBlock, ChartBlock, CodeBlock)):
|
||||
payload["content"] = render_single_block(
|
||||
block,
|
||||
delimiters=delimiters,
|
||||
asset_base_url=asset_base_url,
|
||||
)
|
||||
return payload
|
||||
|
||||
payload["content"] = render_visual_body_content(
|
||||
block,
|
||||
delimiters=delimiters,
|
||||
asset_base_url=asset_base_url,
|
||||
)
|
||||
payload["captions"] = _render_annotation_group(block, _CAPTION_TYPES, delimiters)
|
||||
payload["footnotes"] = _render_annotation_group(block, _FOOTNOTE_TYPES, delimiters)
|
||||
image_source = _resolve_visual_image_source(block, asset_base_url)
|
||||
if image_source is not None:
|
||||
payload["image_source"] = image_source
|
||||
return payload
|
||||
|
||||
|
||||
def _render_annotation_group(
|
||||
block: VisualBlock,
|
||||
accepted_types: set[str],
|
||||
delimiters: LatexDelimitersConfig,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""按源 index 稳定排序视觉说明,并保留可用 bbox 与 Markdown 内容。"""
|
||||
annotations: list[tuple[int, VisualAnnotationBlock]] = []
|
||||
for position, child in enumerate(block.content):
|
||||
if (
|
||||
isinstance(
|
||||
child,
|
||||
(ImageAnnotationBlock, TableAnnotationBlock, ChartAnnotationBlock, CodeAnnotationBlock),
|
||||
)
|
||||
and child.type in accepted_types
|
||||
):
|
||||
annotations.append((position, child))
|
||||
annotations.sort(key=_annotation_sort_key)
|
||||
rendered_annotations: list[dict[str, Any]] = []
|
||||
for _, child in annotations:
|
||||
annotation_payload: dict[str, Any] = {}
|
||||
if child.bbox is not None:
|
||||
annotation_payload["bbox"] = list(child.bbox)
|
||||
annotation_payload["content"] = render_visual_annotation(child, delimiters)
|
||||
rendered_annotations.append(annotation_payload)
|
||||
return rendered_annotations
|
||||
|
||||
|
||||
def _annotation_sort_key(
|
||||
item: tuple[int, VisualAnnotationBlock],
|
||||
) -> tuple[bool, int, int]:
|
||||
"""让有 index 的说明升序优先,缺失 index 的说明稳定排在末尾。"""
|
||||
position, block = item
|
||||
return block.index is None, block.index if block.index is not None else 0, position
|
||||
|
||||
|
||||
def _resolve_visual_image_source(block: VisualBlock, asset_base_url: str) -> str | None:
|
||||
"""解析视觉 body 实际选择的图片来源,并返回安全的 Markdown 地址。"""
|
||||
for child in block.content:
|
||||
if not isinstance(child, (ImageBodyBlock, TableBodyBlock, ChartBodyBlock)):
|
||||
continue
|
||||
return _resolve_content_image_source(child, asset_base_url)
|
||||
return None
|
||||
|
||||
|
||||
def _resolve_content_image_source(block: ImagePayloadBlock, asset_base_url: str) -> str | None:
|
||||
"""把图片载荷收敛为 structured_content 中唯一且安全的 image_source。"""
|
||||
source = resolve_image_source(block, asset_base_url)
|
||||
return normalize_image_source(source) if source else None
|
||||
|
||||
|
||||
__all__ = ["render_structured_content"]
|
||||
@@ -5,7 +5,7 @@ from __future__ import annotations
|
||||
|
||||
from typing import Any, Literal, overload
|
||||
|
||||
from mineru.types import MiddleJson
|
||||
from ..types import MiddleJson
|
||||
|
||||
from .contracts import (
|
||||
DocxRenderOptions,
|
||||
|
||||
+6
-835
@@ -1,110 +1,10 @@
|
||||
# Copyright (c) Opendatalab. All rights reserved.
|
||||
"""严格 MiddleJson 到可编辑 DOCX bytes 的公共渲染实现。"""
|
||||
"""严格 MiddleJson 到 DOCX 的轻量公共门面与稳定异常。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable
|
||||
from io import BytesIO
|
||||
import re
|
||||
|
||||
from bs4 import BeautifulSoup, NavigableString, Tag
|
||||
from docx import Document
|
||||
from docx.document import Document as DocumentType
|
||||
from docx.enum.table import WD_CELL_VERTICAL_ALIGNMENT
|
||||
from docx.enum.text import WD_ALIGN_PARAGRAPH, WD_TAB_ALIGNMENT
|
||||
from docx.oxml import OxmlElement
|
||||
from docx.oxml.ns import qn
|
||||
from docx.shared import Emu, Mm, Pt, Twips
|
||||
from docx.table import _Cell
|
||||
from docx.text.paragraph import Paragraph
|
||||
from lxml import etree
|
||||
from loguru import logger
|
||||
|
||||
from mineru.render._internal.common.index import strip_index_page_tail
|
||||
from mineru.render._internal.common.inline import (
|
||||
InlineEquation,
|
||||
InlineLink,
|
||||
InlineNode,
|
||||
InlineStyled,
|
||||
InlineText,
|
||||
parse_inline_content,
|
||||
)
|
||||
from mineru.render._internal.common.list_items import parse_list_item_marker
|
||||
from mineru.render._internal.common.planner import PlannedBlock, build_render_plan
|
||||
from mineru.render._internal.docx.assets import (
|
||||
DocxAssetError,
|
||||
PreparedImage,
|
||||
prepare_block_image,
|
||||
prepare_html_image,
|
||||
)
|
||||
from mineru.render._internal.docx.inline import (
|
||||
BookmarkRegistry,
|
||||
InlineRenderContext,
|
||||
append_inline_content,
|
||||
append_inline_nodes,
|
||||
append_internal_link,
|
||||
append_joined_inline_contents,
|
||||
sanitize_xml_text,
|
||||
)
|
||||
from mineru.render._internal.docx.math import DocxFormulaError, latex_to_omml, split_formula_tag
|
||||
from mineru.render._internal.docx.styles import (
|
||||
AUXILIARY_STYLE,
|
||||
BODY_STYLE,
|
||||
CAPTION_STYLE,
|
||||
CODE_STYLE,
|
||||
FOOTNOTE_STYLE,
|
||||
FORMULA_FALLBACK_STYLE,
|
||||
SPATIAL_TABLE_STYLE,
|
||||
configure_document,
|
||||
usable_width_emu,
|
||||
usable_width_twips,
|
||||
)
|
||||
from mineru.render._internal.docx.table import DocxTableError, NestedTableWriter, materialize_docx_tables
|
||||
from mineru.render.contracts import AssetResolver, RenderMode
|
||||
from mineru.types import (
|
||||
PAGE_AUXILIARY_BLOCK_TYPES,
|
||||
RAW_ALGORITHM,
|
||||
BlockBase,
|
||||
BlockType,
|
||||
BBox,
|
||||
ChartAnnotationBlock,
|
||||
ChartBlock,
|
||||
ChartBodyBlock,
|
||||
CodeAnnotationBlock,
|
||||
CodeBlock,
|
||||
CodeBodyBlock,
|
||||
DocTitleBlock,
|
||||
EquationBlock,
|
||||
ImageAnnotationBlock,
|
||||
ImageBlock,
|
||||
ImageBodyBlock,
|
||||
ImagePayloadBlock,
|
||||
IndexBlock,
|
||||
ListBlock,
|
||||
MiddleJson,
|
||||
PageAuxTextBlock,
|
||||
ParagraphTitleBlock,
|
||||
RefTextBlock,
|
||||
TableAnnotationBlock,
|
||||
TableBlock,
|
||||
TableBodyBlock,
|
||||
TextBlock,
|
||||
TitleBlockBase,
|
||||
)
|
||||
|
||||
_HTML_TABLE_RE = re.compile(r"<table\b", re.IGNORECASE)
|
||||
_ANNOTATION_CAPTION_TYPES = {
|
||||
BlockType.IMAGE_CAPTION,
|
||||
BlockType.TABLE_CAPTION,
|
||||
BlockType.CHART_CAPTION,
|
||||
BlockType.CODE_CAPTION,
|
||||
}
|
||||
_ANNOTATION_FOOTNOTE_TYPES = {
|
||||
BlockType.IMAGE_FOOTNOTE,
|
||||
BlockType.TABLE_FOOTNOTE,
|
||||
BlockType.CHART_FOOTNOTE,
|
||||
BlockType.CODE_FOOTNOTE,
|
||||
}
|
||||
from ..types import MiddleJson
|
||||
from .contracts import AssetResolver, RenderMode
|
||||
|
||||
|
||||
class DocxRenderError(RuntimeError):
|
||||
@@ -125,745 +25,16 @@ class DocxRenderError(RuntimeError):
|
||||
super().__init__(f"{message} (page_idx={page_idx}, block_index={block_index}, block_type={block_type})")
|
||||
|
||||
|
||||
class _DocxRenderer:
|
||||
"""持有单次 DOCX 渲染所需的 document、素材解析器和书签状态。"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
middle_json: MiddleJson,
|
||||
*,
|
||||
mode: RenderMode,
|
||||
asset_resolver: AssetResolver | None,
|
||||
) -> None:
|
||||
"""初始化 renderer,并预注册全部标题 anchor。"""
|
||||
self.middle_json = middle_json
|
||||
self.mode = mode
|
||||
self.asset_resolver = asset_resolver
|
||||
self.document = Document()
|
||||
configure_document(self.document)
|
||||
self.bookmarks = BookmarkRegistry(_iter_document_anchors(middle_json))
|
||||
self.usable_width_emu = usable_width_emu(self.document)
|
||||
self.usable_width_twips = usable_width_twips(self.document)
|
||||
|
||||
def render(self) -> bytes:
|
||||
"""执行逐页 visitor,并把 python-docx document 序列化为 bytes。"""
|
||||
planned_pages = build_render_plan(self.middle_json, self.mode)
|
||||
for page_position, planned_blocks in enumerate(planned_pages):
|
||||
if self.mode is RenderMode.FULL and page_position > 0:
|
||||
self.document.add_page_break()
|
||||
for planned in planned_blocks:
|
||||
if planned.removed:
|
||||
continue
|
||||
if self.mode is RenderMode.DEFAULT and planned.block.type in PAGE_AUXILIARY_BLOCK_TYPES:
|
||||
continue
|
||||
self._render_planned_block(planned)
|
||||
|
||||
output = BytesIO()
|
||||
self.document.save(output)
|
||||
return output.getvalue()
|
||||
|
||||
def _render_planned_block(self, planned: PlannedBlock) -> None:
|
||||
"""按严格 PageBlock 具体类型分派 Word 写入逻辑。"""
|
||||
block = planned.block
|
||||
context = self._context(planned.page_idx, block)
|
||||
if isinstance(block, (TextBlock, RefTextBlock)):
|
||||
paragraph = self.document.add_paragraph(style=BODY_STYLE)
|
||||
append_joined_inline_contents(
|
||||
paragraph,
|
||||
planned.text_contents or [block.content],
|
||||
context=context,
|
||||
)
|
||||
return
|
||||
if isinstance(block, (DocTitleBlock, ParagraphTitleBlock)):
|
||||
self._render_title(block, context)
|
||||
return
|
||||
if isinstance(block, PageAuxTextBlock):
|
||||
paragraph = self.document.add_paragraph(style=AUXILIARY_STYLE)
|
||||
append_inline_content(paragraph, block.content, context=context)
|
||||
return
|
||||
if isinstance(block, EquationBlock):
|
||||
self._render_equation(block, context)
|
||||
return
|
||||
if isinstance(block, ListBlock):
|
||||
self._render_list(block, context, depth=0)
|
||||
return
|
||||
if isinstance(block, IndexBlock):
|
||||
self._render_index(block, context, depth=0)
|
||||
return
|
||||
if isinstance(block, ImageBlock):
|
||||
self._render_image_block(block, context)
|
||||
return
|
||||
if isinstance(block, TableBlock):
|
||||
self._render_table_block(block, context)
|
||||
return
|
||||
if isinstance(block, ChartBlock):
|
||||
self._render_chart_block(block, context)
|
||||
return
|
||||
if isinstance(block, CodeBlock):
|
||||
self._render_code_block(block, context)
|
||||
return
|
||||
raise TypeError(f"Unsupported PageBlock type: {type(block).__name__}")
|
||||
|
||||
def _context(self, page_idx: int, block: BlockBase) -> InlineRenderContext:
|
||||
"""为当前 block 构造行内渲染上下文。"""
|
||||
return InlineRenderContext(
|
||||
bookmarks=self.bookmarks,
|
||||
page_idx=page_idx,
|
||||
block_index=block.index,
|
||||
block_type=block.type,
|
||||
)
|
||||
|
||||
def _render_title(
|
||||
self,
|
||||
block: DocTitleBlock | ParagraphTitleBlock,
|
||||
context: InlineRenderContext,
|
||||
) -> None:
|
||||
"""用 Heading 1–6 写严格标题,并在段落内容外添加 bookmark。"""
|
||||
level = min(max(block.level, 1), 9)
|
||||
paragraph = self.document.add_paragraph(style=f"Heading {level}")
|
||||
append_inline_content(paragraph, block.content, context=context)
|
||||
self.bookmarks.attach(paragraph, block.anchor)
|
||||
|
||||
def _render_equation(self, block: EquationBlock, context: InlineRenderContext) -> None:
|
||||
"""优先写可编辑 OMML,失败时按图片、可见 LaTeX 的顺序回退。"""
|
||||
content = block.content.strip()
|
||||
if content:
|
||||
formula, tag = split_formula_tag(content)
|
||||
try:
|
||||
math_element = latex_to_omml(formula, display=True)
|
||||
self._append_display_formula(math_element, tag, context=context)
|
||||
return
|
||||
except DocxFormulaError as exc:
|
||||
logger.warning("DOCX display formula fallback: {} ({})", exc, context.location())
|
||||
|
||||
if block.image_base64 is not None or block.image_path is not None:
|
||||
try:
|
||||
self._append_block_image(block, context=context, alt_text="formula")
|
||||
return
|
||||
except DocxRenderError:
|
||||
if not content:
|
||||
raise
|
||||
logger.warning("DOCX formula image fallback failed; preserving LaTeX ({})", context.location())
|
||||
|
||||
if content:
|
||||
paragraph = self.document.add_paragraph(style=FORMULA_FALLBACK_STYLE)
|
||||
paragraph.add_run(sanitize_xml_text(content, context=context))
|
||||
return
|
||||
raise self._render_error("Equation does not contain usable LaTeX or image", context)
|
||||
|
||||
def _append_display_formula(
|
||||
self,
|
||||
math_element: etree._Element,
|
||||
tag: str | None,
|
||||
*,
|
||||
context: InlineRenderContext,
|
||||
) -> None:
|
||||
"""把块公式居中写入;带 tag 时使用中心与右对齐 tab。"""
|
||||
if tag is None:
|
||||
properties = OxmlElement("m:oMathParaPr")
|
||||
justification = OxmlElement("m:jc")
|
||||
justification.set(qn("m:val"), "center")
|
||||
properties.append(justification)
|
||||
math_element.insert(0, properties)
|
||||
paragraph = self.document.add_paragraph(style=BODY_STYLE)
|
||||
paragraph.paragraph_format.space_before = Pt(4)
|
||||
paragraph.paragraph_format.space_after = Pt(5)
|
||||
paragraph._p.append(math_element)
|
||||
return
|
||||
|
||||
equation = math_element.find(qn("m:oMath"))
|
||||
if equation is None:
|
||||
raise ValueError("Display OMML does not contain m:oMath")
|
||||
math_element.remove(equation)
|
||||
paragraph = self.document.add_paragraph(style=BODY_STYLE)
|
||||
paragraph.paragraph_format.space_before = Pt(4)
|
||||
paragraph.paragraph_format.space_after = Pt(5)
|
||||
paragraph.paragraph_format.tab_stops.add_tab_stop(
|
||||
Twips(self.usable_width_twips // 2),
|
||||
WD_TAB_ALIGNMENT.CENTER,
|
||||
)
|
||||
paragraph.paragraph_format.tab_stops.add_tab_stop(
|
||||
Twips(self.usable_width_twips),
|
||||
WD_TAB_ALIGNMENT.RIGHT,
|
||||
)
|
||||
paragraph.add_run().add_tab()
|
||||
paragraph._p.append(equation)
|
||||
paragraph.add_run().add_tab()
|
||||
rendered_tag = tag if tag.startswith("(") and tag.endswith(")") else f"({tag})"
|
||||
paragraph.add_run(sanitize_xml_text(rendered_tag, context=context))
|
||||
|
||||
def _render_list(
|
||||
self,
|
||||
block: ListBlock,
|
||||
context: InlineRenderContext,
|
||||
*,
|
||||
depth: int,
|
||||
) -> None:
|
||||
"""保留叶子原 marker,并仅用缩进表达递归列表层级。"""
|
||||
for child in block.content:
|
||||
if isinstance(child, ListBlock):
|
||||
self._render_list(child, context, depth=depth + 1)
|
||||
continue
|
||||
paragraph = self.document.add_paragraph(style=BODY_STYLE)
|
||||
item = parse_list_item_marker(child.content)
|
||||
if item.marker is None:
|
||||
paragraph.paragraph_format.left_indent = Mm(depth * 6)
|
||||
else:
|
||||
paragraph.paragraph_format.left_indent = Mm((depth + 1) * 6)
|
||||
paragraph.paragraph_format.first_line_indent = Mm(-6)
|
||||
append_inline_content(paragraph, child.content, context=context)
|
||||
|
||||
def _render_index(
|
||||
self,
|
||||
block: IndexBlock,
|
||||
context: InlineRenderContext,
|
||||
*,
|
||||
depth: int,
|
||||
) -> None:
|
||||
"""递归输出目录项,并把标题叶子链接到预注册 bookmark。"""
|
||||
for child in block.content:
|
||||
if isinstance(child, IndexBlock):
|
||||
self._render_index(child, context, depth=depth + 1)
|
||||
continue
|
||||
content = strip_index_page_tail(child.content)
|
||||
nodes = parse_inline_content(content)
|
||||
if not nodes:
|
||||
continue
|
||||
paragraph = self.document.add_paragraph(style=BODY_STYLE)
|
||||
paragraph.paragraph_format.left_indent = Mm((depth + 1) * 6)
|
||||
paragraph.paragraph_format.first_line_indent = Mm(-4)
|
||||
paragraph.add_run("• ")
|
||||
anchor = child.anchor if isinstance(child, TitleBlockBase) else None
|
||||
append_internal_link(paragraph, nodes, anchor=anchor, context=context)
|
||||
|
||||
def _render_image_block(self, block: ImageBlock, context: InlineRenderContext) -> None:
|
||||
"""按原始子块顺序写图片主体、caption 与 footnote。"""
|
||||
for child in block.content:
|
||||
if isinstance(child, ImageBodyBlock):
|
||||
alt_text = _plain_html_text(child.content) or block.sub_type or "image"
|
||||
self._append_block_image(child, context=context, alt_text=alt_text)
|
||||
elif isinstance(child, ImageAnnotationBlock):
|
||||
self._render_annotation(child, context)
|
||||
else:
|
||||
raise TypeError(f"Unsupported image child: {type(child).__name__}")
|
||||
|
||||
def _render_table_block(self, block: TableBlock, context: InlineRenderContext) -> None:
|
||||
"""按原始子块顺序写原生 HTML table 或预格式空间投影文本。"""
|
||||
for child in block.content:
|
||||
if isinstance(child, TableBodyBlock):
|
||||
if _HTML_TABLE_RE.search(child.content):
|
||||
try:
|
||||
self._append_html_tables(child.content, context=context, depth=0)
|
||||
except DocxRenderError as exc:
|
||||
logger.warning("DOCX HTML table fallback: {}", exc)
|
||||
if child.image_base64 is None and child.image_path is None:
|
||||
raise self._render_error(
|
||||
"HTML table cannot be materialized and has no image fallback",
|
||||
context,
|
||||
) from exc
|
||||
self._append_block_image(child, context=context, alt_text="table")
|
||||
else:
|
||||
if child.content and not child.content.isspace():
|
||||
paragraph = self.document.add_paragraph(style=SPATIAL_TABLE_STYLE)
|
||||
paragraph.add_run(sanitize_xml_text(child.content, context=context))
|
||||
elif child.image_base64 is not None or child.image_path is not None:
|
||||
self._append_block_image(child, context=context, alt_text="table")
|
||||
else:
|
||||
raise self._render_error(
|
||||
"Spatial table does not contain text content or image",
|
||||
context,
|
||||
)
|
||||
elif isinstance(child, TableAnnotationBlock):
|
||||
self._render_annotation(child, context)
|
||||
else:
|
||||
raise TypeError(f"Unsupported table child: {type(child).__name__}")
|
||||
|
||||
def _render_chart_block(self, block: ChartBlock, context: InlineRenderContext) -> None:
|
||||
"""先写图表图片,再把 HTML 结构化数据追加为原生表格。"""
|
||||
for child in block.content:
|
||||
if isinstance(child, ChartBodyBlock):
|
||||
has_image = child.image_base64 is not None or child.image_path is not None
|
||||
if has_image:
|
||||
self._append_block_image(
|
||||
child,
|
||||
context=context,
|
||||
alt_text=_plain_html_text(child.content) or block.sub_type or "chart",
|
||||
)
|
||||
if _HTML_TABLE_RE.search(child.content):
|
||||
try:
|
||||
self._append_html_tables(child.content, context=context, depth=0)
|
||||
except DocxRenderError as exc:
|
||||
if not has_image:
|
||||
raise
|
||||
logger.warning("DOCX chart data table omitted after image fallback: {}", exc)
|
||||
elif child.content.strip() and not has_image:
|
||||
paragraph = self.document.add_paragraph(style=BODY_STYLE)
|
||||
append_inline_content(paragraph, child.content, context=context)
|
||||
elif not has_image and not child.content.strip():
|
||||
raise self._render_error("Chart does not contain image or structured content", context)
|
||||
elif isinstance(child, ChartAnnotationBlock):
|
||||
self._render_annotation(child, context)
|
||||
else:
|
||||
raise TypeError(f"Unsupported chart child: {type(child).__name__}")
|
||||
|
||||
def _render_code_block(self, block: CodeBlock, context: InlineRenderContext) -> None:
|
||||
"""按原始子块顺序写代码或算法主体及说明。"""
|
||||
for child in block.content:
|
||||
if isinstance(child, CodeBodyBlock):
|
||||
paragraph = self.document.add_paragraph(style=CODE_STYLE)
|
||||
if block.sub_type == BlockType.CODE:
|
||||
paragraph.add_run(sanitize_xml_text(child.content, context=context))
|
||||
elif block.sub_type == RAW_ALGORITHM:
|
||||
append_inline_content(paragraph, child.content, context=context)
|
||||
else:
|
||||
raise ValueError(f"Unsupported code subtype: {block.sub_type}")
|
||||
elif isinstance(child, CodeAnnotationBlock):
|
||||
self._render_annotation(child, context)
|
||||
else:
|
||||
raise TypeError(f"Unsupported code child: {type(child).__name__}")
|
||||
|
||||
def _render_annotation(
|
||||
self,
|
||||
block: ImageAnnotationBlock | TableAnnotationBlock | ChartAnnotationBlock | CodeAnnotationBlock,
|
||||
context: InlineRenderContext,
|
||||
) -> None:
|
||||
"""根据说明类型选择 Caption 或 Footnote 样式并写入行内内容。"""
|
||||
if block.type in _ANNOTATION_CAPTION_TYPES:
|
||||
style = CAPTION_STYLE
|
||||
elif block.type in _ANNOTATION_FOOTNOTE_TYPES:
|
||||
style = FOOTNOTE_STYLE
|
||||
else:
|
||||
raise TypeError(f"Unsupported annotation type: {block.type}")
|
||||
paragraph = self.document.add_paragraph(style=style)
|
||||
append_inline_content(paragraph, block.content, context=context)
|
||||
|
||||
def _append_block_image(
|
||||
self,
|
||||
block: ImagePayloadBlock,
|
||||
*,
|
||||
context: InlineRenderContext,
|
||||
alt_text: str,
|
||||
) -> None:
|
||||
"""安全加载 block 图片,并按 bbox/自然尺寸限制到可用页面范围。"""
|
||||
try:
|
||||
prepared = prepare_block_image(block, self.asset_resolver)
|
||||
except DocxAssetError as exc:
|
||||
raise self._render_error(str(exc), context) from exc
|
||||
self._append_prepared_image(
|
||||
prepared,
|
||||
bbox=block.bbox,
|
||||
alt_text=alt_text,
|
||||
max_width_emu=self.usable_width_emu,
|
||||
context=context,
|
||||
)
|
||||
|
||||
def _append_prepared_image(
|
||||
self,
|
||||
prepared: PreparedImage,
|
||||
*,
|
||||
bbox: BBox | None,
|
||||
alt_text: str,
|
||||
max_width_emu: int,
|
||||
context: InlineRenderContext,
|
||||
paragraph: Paragraph | None = None,
|
||||
) -> None:
|
||||
"""把准备好的图片按比例插入居中段落并写入 alt description。"""
|
||||
target_paragraph = paragraph or self.document.add_paragraph(style=BODY_STYLE)
|
||||
target_paragraph.alignment = WD_ALIGN_PARAGRAPH.CENTER
|
||||
natural_width = round(prepared.width_px / 96 * 914400)
|
||||
desired_width = natural_width
|
||||
if bbox is not None:
|
||||
desired_width = round(max_width_emu * max(0.0, min(1.0, bbox[2] - bbox[0])))
|
||||
minimum_width = min(int(Mm(5)), max_width_emu)
|
||||
desired_width = max(minimum_width, min(desired_width, max_width_emu))
|
||||
|
||||
usable_height = int(Mm(257))
|
||||
rendered_height = round(desired_width * prepared.height_px / max(prepared.width_px, 1))
|
||||
if rendered_height > usable_height:
|
||||
desired_width = round(usable_height * prepared.width_px / max(prepared.height_px, 1))
|
||||
|
||||
shape = target_paragraph.add_run().add_picture(
|
||||
BytesIO(prepared.data),
|
||||
width=Emu(max(1, desired_width)),
|
||||
)
|
||||
description = sanitize_xml_text(
|
||||
re.sub(r"\s+", " ", alt_text).strip()[:2048],
|
||||
context=context,
|
||||
)
|
||||
if description:
|
||||
shape._inline.docPr.set("descr", description)
|
||||
|
||||
def _append_html_tables(
|
||||
self,
|
||||
markup: str,
|
||||
*,
|
||||
context: InlineRenderContext,
|
||||
depth: int,
|
||||
container: DocumentType | _Cell | None = None,
|
||||
) -> None:
|
||||
"""物化 HTML 表格,并通过共享行内/素材逻辑填充 origin cell。"""
|
||||
if depth >= 4:
|
||||
raise self._render_error("Nested HTML table depth exceeds 4", context)
|
||||
target_container = container or self.document
|
||||
target_xml = target_container._element.body if isinstance(target_container, DocumentType) else target_container._tc
|
||||
existing_child_count = len(target_xml)
|
||||
insertion_index = existing_child_count - 1 if isinstance(target_container, DocumentType) else existing_child_count
|
||||
existing_relationship_ids = set(self.document.part.rels)
|
||||
|
||||
def fill_cell(cell: _Cell, source: Tag, write_nested: NestedTableWriter) -> None:
|
||||
"""填充一个 origin cell,并在遇到 nested table 时调用递归 writer。"""
|
||||
self._fill_html_cell(
|
||||
cell,
|
||||
source,
|
||||
write_nested=write_nested,
|
||||
context=context,
|
||||
)
|
||||
|
||||
try:
|
||||
materialize_docx_tables(
|
||||
target_container,
|
||||
markup,
|
||||
width_twips=self.usable_width_twips,
|
||||
fill_cell=fill_cell,
|
||||
)
|
||||
except DocxRenderError:
|
||||
self._rollback_table_materialization(
|
||||
target_xml,
|
||||
existing_child_count=existing_child_count,
|
||||
insertion_index=insertion_index,
|
||||
existing_relationship_ids=existing_relationship_ids,
|
||||
)
|
||||
raise
|
||||
except DocxTableError as exc:
|
||||
self._rollback_table_materialization(
|
||||
target_xml,
|
||||
existing_child_count=existing_child_count,
|
||||
insertion_index=insertion_index,
|
||||
existing_relationship_ids=existing_relationship_ids,
|
||||
)
|
||||
raise self._render_error(str(exc), context) from exc
|
||||
|
||||
def _rollback_table_materialization(
|
||||
self,
|
||||
target_xml: etree._Element,
|
||||
*,
|
||||
existing_child_count: int,
|
||||
insertion_index: int,
|
||||
existing_relationship_ids: set[str],
|
||||
) -> None:
|
||||
"""回滚本轮表格新增的 XML 根节点和 relationship,避免残留半成品。"""
|
||||
while len(target_xml) > existing_child_count:
|
||||
target_xml.remove(target_xml[insertion_index])
|
||||
for relationship_id in tuple(self.document.part.rels):
|
||||
if relationship_id not in existing_relationship_ids:
|
||||
self.document.part.drop_rel(relationship_id)
|
||||
|
||||
def _fill_html_cell(
|
||||
self,
|
||||
cell: _Cell,
|
||||
source: Tag,
|
||||
*,
|
||||
write_nested: NestedTableWriter,
|
||||
context: InlineRenderContext,
|
||||
) -> None:
|
||||
"""按源顺序填充单元格文本、图片和任意包装层中的嵌套表格。"""
|
||||
cell.text = ""
|
||||
cell.vertical_alignment = WD_CELL_VERTICAL_ALIGNMENT.CENTER
|
||||
_set_cell_margins(cell, top=100, start=120, bottom=100, end=120)
|
||||
max_image_width_emu = _cell_content_width_emu(cell, horizontal_margin_twips=240)
|
||||
paragraph = cell.paragraphs[0]
|
||||
paragraph.paragraph_format.space_after = Pt(0)
|
||||
paragraph.paragraph_format.line_spacing = 1.0
|
||||
|
||||
for child in source.children:
|
||||
if not isinstance(child, (NavigableString, Tag)):
|
||||
continue
|
||||
if isinstance(child, Tag) and child.name.lower() in {"p", "div"} and _paragraph_has_content(paragraph):
|
||||
paragraph = cell.add_paragraph()
|
||||
paragraph.paragraph_format.space_after = Pt(0)
|
||||
paragraph.paragraph_format.line_spacing = 1.0
|
||||
styles = ("bold",) if source.name == "th" else ()
|
||||
paragraph, _ = self._append_html_node(
|
||||
paragraph,
|
||||
child,
|
||||
cell=cell,
|
||||
write_nested=write_nested,
|
||||
context=context,
|
||||
inherited_styles=styles,
|
||||
max_image_width_emu=max_image_width_emu,
|
||||
)
|
||||
|
||||
def _append_html_node(
|
||||
self,
|
||||
paragraph: Paragraph,
|
||||
node: NavigableString | Tag,
|
||||
*,
|
||||
cell: _Cell,
|
||||
write_nested: NestedTableWriter,
|
||||
context: InlineRenderContext,
|
||||
inherited_styles: tuple[str, ...],
|
||||
max_image_width_emu: int,
|
||||
) -> tuple[Paragraph, bool]:
|
||||
"""按 HTML 源顺序写入单元格行内文本、样式、公式和图片。"""
|
||||
if isinstance(node, NavigableString):
|
||||
text = str(node)
|
||||
if not text:
|
||||
return paragraph, False
|
||||
nodes: list[InlineNode] = [InlineText(text)]
|
||||
if inherited_styles:
|
||||
nodes = [InlineStyled(nodes, inherited_styles)]
|
||||
append_inline_nodes(paragraph, nodes, context=context)
|
||||
return paragraph, bool(text.strip())
|
||||
|
||||
name = node.name.lower()
|
||||
if name == "table":
|
||||
write_nested(node)
|
||||
trailing_paragraph = cell.paragraphs[-1]
|
||||
trailing_paragraph.paragraph_format.space_after = Pt(0)
|
||||
trailing_paragraph.paragraph_format.line_spacing = 1.0
|
||||
return trailing_paragraph, True
|
||||
if name == "img":
|
||||
self._append_html_image(
|
||||
paragraph,
|
||||
str(node.get("src", "")),
|
||||
context=context,
|
||||
alt_text=str(node.get("alt", "")),
|
||||
max_width_emu=max_image_width_emu,
|
||||
)
|
||||
return paragraph, True
|
||||
if name == "br":
|
||||
paragraph.add_run().add_break()
|
||||
return paragraph, True
|
||||
if name == "eq":
|
||||
latex = node.get_text().strip()
|
||||
nodes = [InlineEquation(latex)] if latex else []
|
||||
if inherited_styles and nodes:
|
||||
nodes = [InlineStyled(nodes, inherited_styles)]
|
||||
append_inline_nodes(paragraph, nodes, context=context)
|
||||
return paragraph, bool(nodes)
|
||||
if name == "a":
|
||||
children = _html_inline_nodes(node)
|
||||
if inherited_styles and children:
|
||||
children = [InlineStyled(children, inherited_styles)]
|
||||
append_inline_nodes(
|
||||
paragraph,
|
||||
[InlineLink(children, str(node.get("href", "")).strip())],
|
||||
context=context,
|
||||
)
|
||||
return paragraph, bool(children)
|
||||
if name in {"ul", "ol"}:
|
||||
start_value = node.get("start", "1")
|
||||
try:
|
||||
ordered_number = int(str(start_value))
|
||||
except ValueError:
|
||||
ordered_number = 1
|
||||
rendered = False
|
||||
for item in node.find_all("li", recursive=False):
|
||||
if _paragraph_has_content(paragraph) or rendered:
|
||||
paragraph.add_run().add_break()
|
||||
marker = f"{ordered_number}. " if name == "ol" else "- "
|
||||
paragraph.add_run(marker)
|
||||
if name == "ol":
|
||||
ordered_number += 1
|
||||
for child in item.children:
|
||||
if isinstance(child, (NavigableString, Tag)):
|
||||
paragraph, child_rendered = self._append_html_node(
|
||||
paragraph,
|
||||
child,
|
||||
cell=cell,
|
||||
write_nested=write_nested,
|
||||
context=context,
|
||||
inherited_styles=inherited_styles,
|
||||
max_image_width_emu=max_image_width_emu,
|
||||
)
|
||||
rendered = child_rendered or rendered
|
||||
return paragraph, rendered
|
||||
if name == "li":
|
||||
rendered = False
|
||||
for child in node.children:
|
||||
if isinstance(child, (NavigableString, Tag)):
|
||||
paragraph, child_rendered = self._append_html_node(
|
||||
paragraph,
|
||||
child,
|
||||
cell=cell,
|
||||
write_nested=write_nested,
|
||||
context=context,
|
||||
inherited_styles=inherited_styles,
|
||||
max_image_width_emu=max_image_width_emu,
|
||||
)
|
||||
rendered = child_rendered or rendered
|
||||
return paragraph, rendered
|
||||
if name == "text":
|
||||
nodes = parse_inline_content(str(node))
|
||||
if inherited_styles and nodes:
|
||||
nodes = [InlineStyled(nodes, inherited_styles)]
|
||||
append_inline_nodes(paragraph, nodes, context=context)
|
||||
return paragraph, bool(nodes)
|
||||
|
||||
style = {
|
||||
"strong": "bold",
|
||||
"b": "bold",
|
||||
"em": "italic",
|
||||
"i": "italic",
|
||||
"u": "underline",
|
||||
"s": "strikethrough",
|
||||
"sup": "superscript",
|
||||
"sub": "subscript",
|
||||
}.get(name)
|
||||
styles = tuple(dict.fromkeys((*inherited_styles, style))) if style else inherited_styles
|
||||
rendered = False
|
||||
for child in node.children:
|
||||
if isinstance(child, (NavigableString, Tag)):
|
||||
paragraph, child_rendered = self._append_html_node(
|
||||
paragraph,
|
||||
child,
|
||||
cell=cell,
|
||||
write_nested=write_nested,
|
||||
context=context,
|
||||
inherited_styles=styles,
|
||||
max_image_width_emu=max_image_width_emu,
|
||||
)
|
||||
rendered = child_rendered or rendered
|
||||
return paragraph, rendered
|
||||
|
||||
def _append_html_image(
|
||||
self,
|
||||
paragraph: Paragraph,
|
||||
source: str,
|
||||
*,
|
||||
context: InlineRenderContext,
|
||||
alt_text: str,
|
||||
max_width_emu: int,
|
||||
) -> None:
|
||||
"""安全加载表格单元格 img,并限制到紧凑的单元格宽度。"""
|
||||
try:
|
||||
prepared = prepare_html_image(source, self.asset_resolver)
|
||||
except DocxAssetError as exc:
|
||||
raise self._render_error(str(exc), context) from exc
|
||||
self._append_prepared_image(
|
||||
prepared,
|
||||
bbox=None,
|
||||
alt_text=alt_text or "table image",
|
||||
max_width_emu=min(self.usable_width_emu, max_width_emu),
|
||||
context=context,
|
||||
paragraph=paragraph,
|
||||
)
|
||||
|
||||
def _render_error(self, message: str, context: InlineRenderContext) -> DocxRenderError:
|
||||
"""用当前 context 构造带 page/block 定位的公共异常。"""
|
||||
return DocxRenderError(
|
||||
message,
|
||||
page_idx=context.page_idx,
|
||||
block_index=context.block_index,
|
||||
block_type=context.block_type,
|
||||
)
|
||||
|
||||
|
||||
def render_docx(
|
||||
middle_json: MiddleJson,
|
||||
*,
|
||||
mode: RenderMode = RenderMode.DEFAULT,
|
||||
asset_resolver: AssetResolver | None = None,
|
||||
) -> bytes:
|
||||
"""把严格 MiddleJson 无副作用地渲染为完整 DOCX bytes。"""
|
||||
if not isinstance(middle_json, MiddleJson):
|
||||
raise TypeError("render_docx expects a MiddleJson instance")
|
||||
if not isinstance(mode, RenderMode):
|
||||
raise TypeError("mode must be a RenderMode value")
|
||||
return _DocxRenderer(
|
||||
middle_json,
|
||||
mode=mode,
|
||||
asset_resolver=asset_resolver,
|
||||
).render()
|
||||
"""惰性加载 DOCX 实现并渲染严格 MiddleJson。"""
|
||||
from ._internal.docx.renderer import render_docx as _render_docx
|
||||
|
||||
|
||||
def _iter_document_anchors(middle_json: MiddleJson) -> Iterable[str]:
|
||||
"""按文档顺序遍历实际会写入 bookmark 的正文标题 anchor。"""
|
||||
for page in middle_json.pages:
|
||||
for block in page.blocks:
|
||||
if isinstance(block, TitleBlockBase) and block.anchor:
|
||||
yield block.anchor
|
||||
|
||||
|
||||
def _plain_html_text(content: str) -> str:
|
||||
"""把图片或图表 body 内容收敛为适合 alt description 的纯文本。"""
|
||||
if not content.strip():
|
||||
return ""
|
||||
return BeautifulSoup(content, "html.parser").get_text(" ", strip=True)
|
||||
|
||||
|
||||
def _html_inline_nodes(node: Tag) -> list[InlineNode]:
|
||||
"""把表格单元格中的安全 HTML 行内标签转换为共享 InlineNode。"""
|
||||
nodes: list[InlineNode] = []
|
||||
for child in node.children:
|
||||
if isinstance(child, NavigableString):
|
||||
if str(child):
|
||||
nodes.append(InlineText(str(child)))
|
||||
continue
|
||||
if not isinstance(child, Tag):
|
||||
continue
|
||||
name = child.name.lower()
|
||||
children = _html_inline_nodes(child)
|
||||
if name == "eq":
|
||||
latex = child.get_text().strip()
|
||||
if latex:
|
||||
nodes.append(InlineEquation(latex))
|
||||
elif name in {"strong", "b"}:
|
||||
nodes.append(InlineStyled(children, ("bold",)))
|
||||
elif name in {"em", "i"}:
|
||||
nodes.append(InlineStyled(children, ("italic",)))
|
||||
elif name == "u":
|
||||
nodes.append(InlineStyled(children, ("underline",)))
|
||||
elif name == "s":
|
||||
nodes.append(InlineStyled(children, ("strikethrough",)))
|
||||
elif name == "sup":
|
||||
nodes.append(InlineStyled(children, ("superscript",)))
|
||||
elif name == "sub":
|
||||
nodes.append(InlineStyled(children, ("subscript",)))
|
||||
elif name == "a":
|
||||
nodes.append(InlineLink(children, str(child.get("href", "")).strip()))
|
||||
elif name == "br":
|
||||
nodes.append(InlineText("\n"))
|
||||
elif name == "text":
|
||||
nodes.extend(parse_inline_content(str(child)))
|
||||
elif name not in {"img", "table"}:
|
||||
nodes.extend(children)
|
||||
return nodes
|
||||
|
||||
|
||||
def _set_cell_margins(
|
||||
cell: _Cell,
|
||||
*,
|
||||
top: int,
|
||||
start: int,
|
||||
bottom: int,
|
||||
end: int,
|
||||
) -> None:
|
||||
"""用 DXA 为 Word 单元格写入四边内边距。"""
|
||||
cell_properties = cell._tc.get_or_add_tcPr()
|
||||
margins = cell_properties.find(qn("w:tcMar"))
|
||||
if margins is None:
|
||||
margins = OxmlElement("w:tcMar")
|
||||
cell_properties.append(margins)
|
||||
for side, value in (("top", top), ("start", start), ("bottom", bottom), ("end", end)):
|
||||
element = margins.find(qn(f"w:{side}"))
|
||||
if element is None:
|
||||
element = OxmlElement(f"w:{side}")
|
||||
margins.append(element)
|
||||
element.set(qn("w:w"), str(value))
|
||||
element.set(qn("w:type"), "dxa")
|
||||
|
||||
|
||||
def _cell_content_width_emu(cell: _Cell, *, horizontal_margin_twips: int) -> int:
|
||||
"""从当前物理或合并单元格 tcW 计算扣除左右内边距后的可用宽度。"""
|
||||
cell_width = cell._tc.get_or_add_tcPr().get_or_add_tcW().w
|
||||
width_twips = int(cell_width) if cell_width is not None else 1
|
||||
return int(Twips(max(1, width_twips - horizontal_margin_twips)))
|
||||
|
||||
|
||||
def _paragraph_has_content(paragraph: Paragraph) -> bool:
|
||||
"""判断段落是否包含 pPr 之外的 run、公式、链接或 drawing。"""
|
||||
return any(child.tag != qn("w:pPr") for child in paragraph._p)
|
||||
return _render_docx(middle_json, mode=mode, asset_resolver=asset_resolver)
|
||||
|
||||
|
||||
__all__ = ["DocxRenderError", "render_docx"]
|
||||
|
||||
+7
-918
@@ -1,633 +1,10 @@
|
||||
# Copyright (c) Opendatalab. All rights reserved.
|
||||
"""严格 MiddleJson 到 MinerU 风格 HTML 的公共渲染实现。"""
|
||||
"""严格 MiddleJson 到 HTML 的轻量公共门面。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import re
|
||||
from functools import lru_cache
|
||||
from importlib import resources
|
||||
from urllib.parse import quote
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
from loguru import logger
|
||||
|
||||
from mineru.render._internal.common.index import strip_index_page_tail
|
||||
from mineru.render._internal.common.inline import inline_plain_text, parse_inline_content
|
||||
from mineru.render._internal.common.list_items import ListItem, parse_list_item_marker, reference_list_needs_bullets
|
||||
from mineru.render._internal.common.planner import PlannedBlock, build_render_plan
|
||||
from mineru.render._internal.html.inline import (
|
||||
HtmlInlineResult,
|
||||
render_inline_content_html,
|
||||
render_inline_nodes_html,
|
||||
render_joined_inline_contents_html,
|
||||
render_math_html,
|
||||
)
|
||||
from mineru.render._internal.html.sanitizer import (
|
||||
is_supported_html_markup,
|
||||
sanitize_html_fragment,
|
||||
sanitize_image_source,
|
||||
)
|
||||
from mineru.render._internal.html.table import looks_like_gfm_table, render_gfm_table_html
|
||||
from mineru.render.contracts import RenderMode
|
||||
from mineru.types import (
|
||||
PAGE_AUXILIARY_BLOCK_TYPES,
|
||||
RAW_ALGORITHM,
|
||||
BlockType,
|
||||
ChartAnnotationBlock,
|
||||
ChartBlock,
|
||||
ChartBodyBlock,
|
||||
CodeAnnotationBlock,
|
||||
CodeBlock,
|
||||
CodeBodyBlock,
|
||||
DocTitleBlock,
|
||||
EquationBlock,
|
||||
ImageAnnotationBlock,
|
||||
ImageBlock,
|
||||
ImageBodyBlock,
|
||||
IndexBlock,
|
||||
ListBlock,
|
||||
MiddleJson,
|
||||
PageAuxTextBlock,
|
||||
ParagraphTitleBlock,
|
||||
RefTextBlock,
|
||||
TableAnnotationBlock,
|
||||
TableBlock,
|
||||
TableBodyBlock,
|
||||
TextBlock,
|
||||
TitleBlockBase,
|
||||
)
|
||||
|
||||
_STYLE_RESOURCE_NAME = "mineru.min.css"
|
||||
_MATHJAX_URL = "https://cdn.jsdelivr.net/npm/mathjax@4.1.2/tex-chtml.js"
|
||||
_MATHJAX_INTEGRITY = "sha384-zAhQQhdaMeHsMProNntGGg6nOUVcfuF9F22C3d1qJ9NZAVzCplXk1X85D2O5iufn"
|
||||
_PRISM_CORE_URL = "https://cdn.jsdelivr.net/npm/prismjs@1.30.0/components/prism-core.min.js"
|
||||
_PRISM_CORE_INTEGRITY = "sha384-zLRFO4dwowZvh8kzutOb5AWhH7f39HeJp+N7PtHF1SQtTBnifRx0AtmvTYs3F4YV"
|
||||
_PRISM_AUTOLOADER_URL = "https://cdn.jsdelivr.net/npm/prismjs@1.30.0/plugins/autoloader/prism-autoloader.min.js"
|
||||
_PRISM_AUTOLOADER_INTEGRITY = "sha384-Uq05+JLko69eOiPr39ta9bh7kld5PKZoU+fF7g0EXTAriEollhZ+DrN8Q/Oi8J2Q"
|
||||
_PRISM_LANGUAGES_PATH = "https://cdn.jsdelivr.net/npm/prismjs@1.30.0/components/"
|
||||
_MERMAID_URL = "https://cdn.jsdelivr.net/npm/mermaid@11.16.1/dist/mermaid.min.js"
|
||||
_MERMAID_INTEGRITY = "sha384-aBQXj4hK6Jm05i7aQAsUV3bLdSUrHX1BGYfMB0166TtWt/RRaw+h0Eelme9OCOvy"
|
||||
_MERMAID_MAX_TEXT_SIZE = 50_000
|
||||
_MERMAID_FLOWCHART_HEADER_RE = re.compile(
|
||||
r"(?:graph|flowchart)\s+(?:TB|TD|BT|RL|LR)\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_SAFE_PRISM_LANGUAGE_RE = re.compile(r"[a-z0-9][a-z0-9-]{0,31}")
|
||||
_BLOCKED_PRISM_LANGUAGE_NAMES = {"constructor", "prototype"}
|
||||
_PRISM_LANGUAGE_ALIASES = {
|
||||
"c#": "csharp",
|
||||
"c++": "cpp",
|
||||
"html": "markup",
|
||||
"js": "javascript",
|
||||
"py": "python",
|
||||
"sh": "bash",
|
||||
"shell": "bash",
|
||||
"none": "",
|
||||
"text": "",
|
||||
"txt": "",
|
||||
"ts": "typescript",
|
||||
"xml": "markup",
|
||||
"yml": "yaml",
|
||||
}
|
||||
|
||||
|
||||
class _HtmlRenderer:
|
||||
"""维护一次 HTML 渲染所需的 anchor、公式与代码资源状态。"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
middle_json: MiddleJson,
|
||||
*,
|
||||
mode: RenderMode,
|
||||
asset_base_url: str,
|
||||
standalone: bool,
|
||||
document_title: str | None,
|
||||
) -> None:
|
||||
"""保存严格输入和调用选项,并预收集真实正文标题 anchor。"""
|
||||
self.middle_json = middle_json
|
||||
self.mode = mode
|
||||
self.asset_base_url = asset_base_url
|
||||
self.standalone = standalone
|
||||
self.document_title = document_title
|
||||
self.anchor_targets = _collect_title_anchor_ids(middle_json)
|
||||
self.emitted_anchors: set[str] = set()
|
||||
self.has_math = False
|
||||
self.has_prism = False
|
||||
self.has_mermaid = False
|
||||
|
||||
def render(self) -> str:
|
||||
"""生成 fragment article,并按需包装完整 HTML 文档。"""
|
||||
planned_pages = build_render_plan(self.middle_json, self.mode)
|
||||
if self.mode is RenderMode.FULL:
|
||||
body = self._render_full_pages(planned_pages)
|
||||
else:
|
||||
body = self._render_default_pages(planned_pages)
|
||||
article = f'<article class="mineru-document mineru-document--{self.mode.value}">\n{body}\n</article>'
|
||||
if not self.standalone:
|
||||
return article
|
||||
# 最终依赖以真实 DOM 为准,避免回退分支或普通文本中的 class 字样误触发。
|
||||
article_soup = BeautifulSoup(article, "html.parser")
|
||||
self.has_math = article_soup.select_one(".mineru-math") is not None
|
||||
self.has_prism = any(
|
||||
any(str(class_name).startswith("language-") for class_name in code.get("class", []))
|
||||
for code in article_soup.find_all("code")
|
||||
if code.get_text()
|
||||
)
|
||||
self.has_mermaid = article_soup.select_one(".mineru-flowchart") is not None
|
||||
return self._render_standalone(article)
|
||||
|
||||
def _render_default_pages(self, planned_pages: list[list[PlannedBlock]]) -> str:
|
||||
"""把 DEFAULT 计划展平为无页面边界的连续阅读内容。"""
|
||||
rendered: list[str] = []
|
||||
for page in planned_pages:
|
||||
for planned in page:
|
||||
block = self._render_planned_block(planned)
|
||||
if block:
|
||||
rendered.append(block)
|
||||
return "\n".join(rendered)
|
||||
|
||||
def _render_full_pages(self, planned_pages: list[list[PlannedBlock]]) -> str:
|
||||
"""为 FULL 计划保留所有页面 section 和相邻页面分隔。"""
|
||||
sections: list[str] = []
|
||||
for page in planned_pages:
|
||||
page_idx = page[0].page_idx if page else None
|
||||
if page_idx is None:
|
||||
page_position = len(sections)
|
||||
page_idx = self.middle_json.pages[page_position].page_idx
|
||||
blocks = [rendered for planned in page if (rendered := self._render_planned_block(planned))]
|
||||
rendered_blocks = "\n".join(blocks)
|
||||
sections.append(f'<section class="mineru-page" data-page-idx="{page_idx}">\n{rendered_blocks}\n</section>')
|
||||
return '\n<hr class="mineru-page-break" aria-hidden="true">\n'.join(sections)
|
||||
|
||||
def _render_planned_block(self, planned: PlannedBlock) -> str:
|
||||
"""过滤计划块、分派类型 visitor,并添加稳定来源元数据。"""
|
||||
if planned.removed:
|
||||
return ""
|
||||
block = planned.block
|
||||
if self.mode is RenderMode.DEFAULT and block.type in PAGE_AUXILIARY_BLOCK_TYPES:
|
||||
return ""
|
||||
content = self._render_block_content(planned)
|
||||
if not content or not content.strip():
|
||||
return ""
|
||||
attrs = [
|
||||
'class="mineru-block"',
|
||||
f'data-page-idx="{planned.page_idx}"',
|
||||
f'data-block-type="{html.escape(str(block.type), quote=True)}"',
|
||||
]
|
||||
if block.index is not None:
|
||||
attrs.append(f'data-block-index="{block.index}"')
|
||||
return f"<div {' '.join(attrs)}>\n{content}\n</div>"
|
||||
|
||||
def _render_block_content(self, planned: PlannedBlock) -> str:
|
||||
"""按严格 PageBlock 具体类型返回对应语义 HTML。"""
|
||||
block = planned.block
|
||||
if isinstance(block, TextBlock):
|
||||
rendered = render_joined_inline_contents_html(planned.text_contents or [block.content])
|
||||
self._observe_inline(rendered)
|
||||
return f'<p class="mineru-text">{rendered.html}</p>' if rendered.html else ""
|
||||
if isinstance(block, RefTextBlock):
|
||||
rendered = render_joined_inline_contents_html(planned.text_contents or [block.content])
|
||||
self._observe_inline(rendered)
|
||||
return f'<p class="mineru-ref-text">{rendered.html}</p>' if rendered.html else ""
|
||||
if isinstance(block, (DocTitleBlock, ParagraphTitleBlock)):
|
||||
return self._render_title(block)
|
||||
if isinstance(block, PageAuxTextBlock):
|
||||
rendered = render_inline_content_html(block.content)
|
||||
self._observe_inline(rendered)
|
||||
class_name = html.escape(str(block.type).replace("_", "-"), quote=True)
|
||||
return f'<div class="mineru-page-aux mineru-page-aux--{class_name}">{rendered.html}</div>' if rendered.html else ""
|
||||
if isinstance(block, EquationBlock):
|
||||
return self._render_equation(block)
|
||||
if isinstance(block, ListBlock):
|
||||
return self._render_list(block)
|
||||
if isinstance(block, IndexBlock):
|
||||
return self._render_index(block)
|
||||
if isinstance(block, ImageBlock):
|
||||
return self._render_image_block(block)
|
||||
if isinstance(block, TableBlock):
|
||||
return self._render_table_block(block)
|
||||
if isinstance(block, ChartBlock):
|
||||
return self._render_chart_block(block)
|
||||
if isinstance(block, CodeBlock):
|
||||
return self._render_code_block(block)
|
||||
raise TypeError(f"Unsupported PageBlock type: {type(block).__name__}")
|
||||
|
||||
def _render_title(self, block: DocTitleBlock | ParagraphTitleBlock) -> str:
|
||||
"""渲染标题,并保证正文 anchor 只在首次出现时生成 id。"""
|
||||
rendered = render_inline_content_html(block.content)
|
||||
self._observe_inline(rendered)
|
||||
if not rendered.html:
|
||||
return ""
|
||||
level = min(max(block.level, 1), 6)
|
||||
attrs = [f'class="mineru-heading mineru-heading--{level}"', f'data-heading-level="{block.level}"']
|
||||
anchor = _anchor_key(block.anchor)
|
||||
if anchor:
|
||||
if anchor not in self.emitted_anchors:
|
||||
anchor_id = self.anchor_targets.get(anchor)
|
||||
if anchor_id:
|
||||
attrs.append(f'id="{html.escape(anchor_id, quote=True)}"')
|
||||
self.emitted_anchors.add(anchor)
|
||||
else:
|
||||
logger.warning("Duplicate HTML title anchor omitted: {!r}", anchor)
|
||||
return f"<h{level} {' '.join(attrs)}>{rendered.html}</h{level}>"
|
||||
|
||||
def _render_equation(self, block: EquationBlock) -> str:
|
||||
"""优先输出行间公式,空 LaTeX 时回退到安全图片来源。"""
|
||||
if block.content.strip():
|
||||
rendered = render_math_html(block.content, display=True)
|
||||
self._observe_inline(rendered)
|
||||
return rendered.html
|
||||
source = self._safe_block_image_source(block)
|
||||
return self._render_image(source, alt="formula", class_name="mineru-equation-image") if source else ""
|
||||
|
||||
def _render_list(self, block: ListBlock) -> str:
|
||||
"""按直属 marker 类型渲染一层列表,并递归挂接嵌套列表。"""
|
||||
parsed_leaves = [
|
||||
parse_list_item_marker(child.content)
|
||||
for child in block.content
|
||||
if not isinstance(child, ListBlock) and child.content.strip()
|
||||
]
|
||||
add_reference_bullets = reference_list_needs_bullets(block)
|
||||
container_tag, list_type, class_name = _classify_list(parsed_leaves, add_reference_bullets)
|
||||
items: list[dict[str, object]] = []
|
||||
expected_value: int | None = None
|
||||
|
||||
for child in block.content:
|
||||
if isinstance(child, ListBlock):
|
||||
nested = self._render_list(child)
|
||||
if not nested:
|
||||
continue
|
||||
if not items:
|
||||
items.append({"content": "", "attrs": ' class="mineru-list-item--orphan"', "nested": []})
|
||||
nested_items = items[-1]["nested"]
|
||||
assert isinstance(nested_items, list)
|
||||
nested_items.append(nested)
|
||||
continue
|
||||
|
||||
parsed = parse_list_item_marker(child.content)
|
||||
item_content, marker = _list_item_content(
|
||||
parsed,
|
||||
add_reference_bullets,
|
||||
explicit_markers=class_name == "mineru-list--explicit",
|
||||
)
|
||||
rendered = render_inline_content_html(item_content)
|
||||
self._observe_inline(rendered)
|
||||
if not rendered.html and not marker:
|
||||
items.append({"content": "", "attrs": ' class="mineru-list-item--markerless"', "nested": []})
|
||||
continue
|
||||
|
||||
attrs: list[str] = []
|
||||
if class_name == "mineru-list--explicit":
|
||||
attrs.append('class="mineru-list-item--explicit"')
|
||||
if container_tag == "ol" and parsed.kind == "ordered" and parsed.value is not None:
|
||||
if expected_value is None:
|
||||
expected_value = parsed.value
|
||||
if parsed.value != expected_value:
|
||||
attrs.append(f'value="{parsed.value}"')
|
||||
expected_value = parsed.value + 1
|
||||
marker_html = (
|
||||
f'<span class="mineru-list-marker">{html.escape(_replace_html_controls(marker), quote=False)}</span>'
|
||||
if marker
|
||||
else ""
|
||||
)
|
||||
if class_name == "mineru-list--explicit" and not marker_html:
|
||||
marker_html = '<span class="mineru-list-marker"></span>'
|
||||
items.append(
|
||||
{
|
||||
"content": f'{marker_html}<span class="mineru-list-content">{rendered.html}</span>',
|
||||
"attrs": f" {' '.join(attrs)}" if attrs else "",
|
||||
"nested": [],
|
||||
}
|
||||
)
|
||||
|
||||
rendered_items: list[str] = []
|
||||
for item in items:
|
||||
nested = item["nested"]
|
||||
assert isinstance(nested, list)
|
||||
item_content = str(item["content"])
|
||||
if not item_content and not nested:
|
||||
continue
|
||||
rendered_items.append(f"<li{item['attrs']}>{item_content}{''.join(str(value) for value in nested)}</li>")
|
||||
if not rendered_items:
|
||||
return ""
|
||||
|
||||
container_attrs = [f'class="mineru-list {class_name}"']
|
||||
if list_type:
|
||||
container_attrs.append(f'type="{list_type}"')
|
||||
if container_tag == "ol" and parsed_leaves and parsed_leaves[0].value not in (None, 1):
|
||||
container_attrs.append(f'start="{parsed_leaves[0].value}"')
|
||||
return f"<{container_tag} {' '.join(container_attrs)}>{''.join(rendered_items)}</{container_tag}>"
|
||||
|
||||
def _render_index(self, block: IndexBlock) -> str:
|
||||
"""递归渲染目录,并仅链接到真实正文标题 anchor。"""
|
||||
items: list[dict[str, object]] = []
|
||||
for child in block.content:
|
||||
if isinstance(child, IndexBlock):
|
||||
nested = self._render_index_list(child)
|
||||
if not nested:
|
||||
continue
|
||||
if not items:
|
||||
items.append({"content": "", "orphan": True, "nested": []})
|
||||
nested_items = items[-1]["nested"]
|
||||
assert isinstance(nested_items, list)
|
||||
nested_items.append(nested)
|
||||
continue
|
||||
rendered = self._render_index_leaf(child)
|
||||
items.append({"content": rendered, "orphan": False, "nested": []})
|
||||
inner = _serialize_index_items(items)
|
||||
return f'<nav class="mineru-index" aria-label="Table of contents"><ul>{inner}</ul></nav>' if inner else ""
|
||||
|
||||
def _render_index_list(self, block: IndexBlock) -> str:
|
||||
"""渲染一个嵌套 IndexBlock 为可挂接到父项的 ul。"""
|
||||
items: list[dict[str, object]] = []
|
||||
for child in block.content:
|
||||
if isinstance(child, IndexBlock):
|
||||
nested = self._render_index_list(child)
|
||||
if nested:
|
||||
if not items:
|
||||
items.append({"content": "", "orphan": True, "nested": []})
|
||||
nested_items = items[-1]["nested"]
|
||||
assert isinstance(nested_items, list)
|
||||
nested_items.append(nested)
|
||||
continue
|
||||
rendered = self._render_index_leaf(child)
|
||||
items.append({"content": rendered, "orphan": False, "nested": []})
|
||||
inner = _serialize_index_items(items)
|
||||
return f"<ul>{inner}</ul>" if inner else ""
|
||||
|
||||
def _render_index_leaf(self, block: TextBlock | TitleBlockBase) -> str:
|
||||
"""渲染目录叶子,并在 anchor 命中正文标题时生成内部链接。"""
|
||||
content = strip_index_page_tail(block.content)
|
||||
rendered = render_inline_content_html(content)
|
||||
self._observe_inline(rendered)
|
||||
if not rendered.html:
|
||||
return ""
|
||||
anchor = _anchor_key(block.anchor) if isinstance(block, TitleBlockBase) else ""
|
||||
if anchor and (anchor_id := self.anchor_targets.get(anchor)):
|
||||
href = quote(anchor_id, safe="-._~")
|
||||
return f'<a href="#{href}">{rendered.html}</a>'
|
||||
return rendered.html
|
||||
|
||||
def _render_image_block(self, block: ImageBlock) -> str:
|
||||
"""按原始子块顺序渲染图片主体及重复说明。"""
|
||||
parts: list[str] = []
|
||||
for child in block.content:
|
||||
if isinstance(child, ImageBodyBlock):
|
||||
body = self._render_image_body(block, child)
|
||||
elif isinstance(child, ImageAnnotationBlock):
|
||||
body = self._render_annotation(child)
|
||||
else:
|
||||
raise TypeError(f"Unsupported image child: {type(child).__name__}")
|
||||
if body:
|
||||
parts.append(body)
|
||||
return f'<figure class="mineru-figure mineru-figure--image">{"".join(parts)}</figure>' if parts else ""
|
||||
|
||||
def _render_image_body(self, parent: ImageBlock, block: ImageBodyBlock) -> str:
|
||||
"""渲染图片,并把并存的识别内容放入原生 details。"""
|
||||
if parent.sub_type == "flowchart":
|
||||
mermaid_source = _extract_mermaid_flowchart_source(block.content)
|
||||
if mermaid_source is not None:
|
||||
return self._render_flowchart_body(block, mermaid_source)
|
||||
source = self._safe_block_image_source(block)
|
||||
content = self._render_embedded_content(block.content, linkify_text=parent.sub_type != "flowchart")
|
||||
alt = _plain_content_text(block.content) or parent.sub_type or "image"
|
||||
parts = [self._render_image(source, alt=alt, class_name="mineru-image")] if source else []
|
||||
if content.html:
|
||||
if source:
|
||||
parts.append(self._render_details(content.html, parent.sub_type or "image content"))
|
||||
else:
|
||||
parts.append(content.html)
|
||||
return "".join(parts)
|
||||
|
||||
def _render_flowchart_body(self, block: ImageBodyBlock, mermaid_source: str) -> str:
|
||||
"""输出 Mermaid canvas,并保留 raster 与源码两级失败回退。"""
|
||||
source = self._safe_block_image_source(block)
|
||||
escaped_source = html.escape(_replace_html_controls(mermaid_source), quote=False)
|
||||
fallback = self._render_image(source, alt="flowchart", class_name="mineru-flowchart-fallback") if source else ""
|
||||
fallback_class = " mineru-flowchart--has-raster" if fallback else ""
|
||||
details_open = "" if fallback else " open"
|
||||
return (
|
||||
f'<div class="mineru-flowchart{fallback_class}" data-mermaid-state="pending">'
|
||||
'<div class="mineru-flowchart-canvas" role="img" aria-label="flowchart"></div>'
|
||||
f"{fallback}</div>"
|
||||
f'<details class="mineru-details mineru-flowchart-details"{details_open}>'
|
||||
'<summary>flowchart source</summary>'
|
||||
f'<pre class="mineru-flowchart-source"><code>{escaped_source}</code></pre>'
|
||||
"</details>"
|
||||
)
|
||||
|
||||
def _render_table_block(self, block: TableBlock) -> str:
|
||||
"""按原始子块顺序渲染表格主体、标题与脚注。"""
|
||||
parts: list[str] = []
|
||||
for child in block.content:
|
||||
if isinstance(child, TableBodyBlock):
|
||||
body = self._render_table_body(child)
|
||||
elif isinstance(child, TableAnnotationBlock):
|
||||
body = self._render_annotation(child)
|
||||
else:
|
||||
raise TypeError(f"Unsupported table child: {type(child).__name__}")
|
||||
if body:
|
||||
parts.append(body)
|
||||
return f'<figure class="mineru-figure mineru-figure--table">{"".join(parts)}</figure>' if parts else ""
|
||||
|
||||
def _render_table_body(self, block: TableBodyBlock) -> str:
|
||||
"""优先保留安全 HTML table,空间文本和整体图片依次回退。"""
|
||||
normalized_content = _replace_html_controls(block.content)
|
||||
content = normalized_content.strip()
|
||||
if content:
|
||||
source_soup = BeautifulSoup(content, "html.parser")
|
||||
if source_soup.find("table") is not None:
|
||||
rendered = self._render_embedded_content(content)
|
||||
if rendered.html and _contains_usable_table(rendered.html):
|
||||
return rendered.html
|
||||
source = self._safe_block_image_source(block)
|
||||
if source:
|
||||
return self._render_image(source, alt="table", class_name="mineru-table-image")
|
||||
return _render_raw_fallback(content)
|
||||
return f'<pre class="mineru-table-text">{html.escape(normalized_content, quote=False)}</pre>'
|
||||
source = self._safe_block_image_source(block)
|
||||
return self._render_image(source, alt="table", class_name="mineru-table-image") if source else ""
|
||||
|
||||
def _render_chart_block(self, block: ChartBlock) -> str:
|
||||
"""按原始子块顺序渲染 chart 图片、结构内容及说明。"""
|
||||
parts: list[str] = []
|
||||
for child in block.content:
|
||||
if isinstance(child, ChartBodyBlock):
|
||||
body = self._render_chart_body(block, child)
|
||||
elif isinstance(child, ChartAnnotationBlock):
|
||||
body = self._render_annotation(child)
|
||||
else:
|
||||
raise TypeError(f"Unsupported chart child: {type(child).__name__}")
|
||||
if body:
|
||||
parts.append(body)
|
||||
return f'<figure class="mineru-figure mineru-figure--chart">{"".join(parts)}</figure>' if parts else ""
|
||||
|
||||
def _render_chart_body(self, parent: ChartBlock, block: ChartBodyBlock) -> str:
|
||||
"""渲染 chart 图片,并将并存结构内容放入 details。"""
|
||||
source = self._safe_block_image_source(block)
|
||||
content = self._render_chart_content(block.content)
|
||||
parts = [self._render_image(source, alt=parent.sub_type or "chart", class_name="mineru-chart-image")] if source else []
|
||||
if content.html:
|
||||
if source:
|
||||
parts.append(self._render_details(content.html, parent.sub_type or "chart content"))
|
||||
else:
|
||||
parts.append(content.html)
|
||||
return "".join(parts)
|
||||
|
||||
def _render_chart_content(self, content: str) -> HtmlInlineResult:
|
||||
"""按 HTML、严格 GFM 表格、普通行内内容的顺序渲染 chart content。"""
|
||||
normalized = content.strip()
|
||||
if not normalized:
|
||||
return HtmlInlineResult("")
|
||||
if is_supported_html_markup(normalized):
|
||||
return self._render_embedded_content(normalized)
|
||||
if looks_like_gfm_table(normalized):
|
||||
rendered = render_gfm_table_html(normalized)
|
||||
if rendered is None:
|
||||
return HtmlInlineResult(_render_raw_fallback(content))
|
||||
self._observe_inline(rendered)
|
||||
return rendered
|
||||
rendered = render_inline_content_html(content)
|
||||
self._observe_inline(rendered)
|
||||
return rendered
|
||||
|
||||
def _render_code_block(self, block: CodeBlock) -> str:
|
||||
"""按原始子块顺序渲染代码或算法及其说明。"""
|
||||
parts: list[str] = []
|
||||
for child in block.content:
|
||||
if isinstance(child, CodeBodyBlock):
|
||||
body = self._render_code_body(block, child)
|
||||
elif isinstance(child, CodeAnnotationBlock):
|
||||
body = self._render_annotation(child)
|
||||
else:
|
||||
raise TypeError(f"Unsupported code child: {type(child).__name__}")
|
||||
if body:
|
||||
parts.append(body)
|
||||
return f'<figure class="mineru-figure mineru-figure--code">{"".join(parts)}</figure>' if parts else ""
|
||||
|
||||
def _render_code_body(self, parent: CodeBlock, block: CodeBodyBlock) -> str:
|
||||
"""代码使用 Prism class,算法使用共享 inline AST 保留空白。"""
|
||||
if parent.sub_type == BlockType.CODE:
|
||||
escaped = html.escape(_replace_html_controls(block.content), quote=False)
|
||||
if not block.content:
|
||||
return '<pre class="mineru-code"><code></code></pre>'
|
||||
language = _normalize_prism_language(parent.guess_lang)
|
||||
class_attr = f' class="language-{language}"' if language else ""
|
||||
pre_class = f"mineru-code language-{language}" if language else "mineru-code"
|
||||
self.has_prism = bool(language) or self.has_prism
|
||||
return f'<pre class="{pre_class}"><code{class_attr}>{escaped}</code></pre>'
|
||||
if parent.sub_type == RAW_ALGORITHM:
|
||||
rendered = render_inline_nodes_html(
|
||||
parse_inline_content(block.content),
|
||||
linkify_text=False,
|
||||
separate_adjacent_math=True,
|
||||
preserve_newlines=True,
|
||||
)
|
||||
self._observe_inline(rendered)
|
||||
return f'<div class="mineru-algorithm">{rendered.html}</div>' if rendered.html else ""
|
||||
raise ValueError(f"Unsupported code subtype: {parent.sub_type}")
|
||||
|
||||
def _render_annotation(
|
||||
self,
|
||||
block: ImageAnnotationBlock | TableAnnotationBlock | ChartAnnotationBlock | CodeAnnotationBlock,
|
||||
) -> str:
|
||||
"""把视觉说明渲染为保持原类型的独立段落。"""
|
||||
rendered = render_inline_content_html(block.content)
|
||||
self._observe_inline(rendered)
|
||||
if not rendered.html:
|
||||
return ""
|
||||
is_caption = str(block.type).endswith("caption")
|
||||
role_class = "mineru-caption" if is_caption else "mineru-footnote"
|
||||
type_class = html.escape(str(block.type).replace("_", "-"), quote=True)
|
||||
return f'<p class="{role_class} {role_class}--{type_class}">{rendered.html}</p>'
|
||||
|
||||
def _render_embedded_content(self, content: str, *, linkify_text: bool = True) -> HtmlInlineResult:
|
||||
"""安全处理 body 富 HTML;普通内容走共享 inline AST。"""
|
||||
normalized = content.strip()
|
||||
if not normalized:
|
||||
return HtmlInlineResult("")
|
||||
if not is_supported_html_markup(normalized):
|
||||
rendered = render_inline_nodes_html(parse_inline_content(content), linkify_text=linkify_text)
|
||||
self._observe_inline(rendered)
|
||||
return rendered
|
||||
sanitized = sanitize_html_fragment(normalized, asset_base_url=self.asset_base_url)
|
||||
rendered = self._replace_safe_equations(sanitized)
|
||||
self._observe_inline(rendered)
|
||||
return rendered
|
||||
|
||||
def _replace_safe_equations(self, markup: str) -> HtmlInlineResult:
|
||||
"""把 sanitizer 保留的 eq 标签替换为可信 MathJax carrier。"""
|
||||
if not markup:
|
||||
return HtmlInlineResult("")
|
||||
soup = BeautifulSoup(markup, "html.parser")
|
||||
has_math = False
|
||||
for equation in list(soup.find_all("eq")):
|
||||
rendered = render_math_html(equation.get_text(), display=False)
|
||||
if not rendered.html:
|
||||
equation.decompose()
|
||||
continue
|
||||
replacement = BeautifulSoup(rendered.html, "html.parser")
|
||||
equation.replace_with(*list(replacement.contents))
|
||||
has_math = True
|
||||
return HtmlInlineResult(str(soup), has_math)
|
||||
|
||||
def _safe_block_image_source(self, block: EquationBlock | ImageBodyBlock | TableBodyBlock | ChartBodyBlock) -> str | None:
|
||||
"""解析 block 图片来源,并应用 HTML renderer 的 URL 安全策略。"""
|
||||
if block.image_path:
|
||||
return sanitize_image_source(block.image_path, asset_base_url=self.asset_base_url)
|
||||
if block.image_base64:
|
||||
return sanitize_image_source(block.image_base64)
|
||||
return None
|
||||
|
||||
def _render_image(self, source: str, *, alt: str, class_name: str) -> str:
|
||||
"""构造属性已转义的安全 img 元素。"""
|
||||
return (
|
||||
f'<img class="{class_name}" src="{html.escape(source, quote=True)}" '
|
||||
f'alt="{html.escape(_replace_html_controls(alt), quote=True)}">'
|
||||
)
|
||||
|
||||
def _render_details(self, content: str, summary: str) -> str:
|
||||
"""构造浏览器原生折叠详情,不引入额外脚本。"""
|
||||
return (
|
||||
'<details class="mineru-details">'
|
||||
f"<summary>{html.escape(_replace_html_controls(summary), quote=False)}</summary>{content}</details>"
|
||||
)
|
||||
|
||||
def _observe_inline(self, rendered: HtmlInlineResult) -> None:
|
||||
"""累计当前文档是否需要加载 MathJax。"""
|
||||
self.has_math = rendered.has_math or self.has_math
|
||||
|
||||
def _render_standalone(self, article: str) -> str:
|
||||
"""用固定 CSS 和按需 CDN 资源包装完整 HTML 文档。"""
|
||||
title = html.escape(
|
||||
_replace_html_controls(_resolve_document_title(self.middle_json, self.document_title)),
|
||||
quote=False,
|
||||
)
|
||||
styles = _load_html_styles()
|
||||
dependencies = []
|
||||
if self.has_math:
|
||||
dependencies.append(_mathjax_head())
|
||||
if self.has_prism:
|
||||
dependencies.append(_prism_head())
|
||||
if self.has_mermaid:
|
||||
dependencies.append(_mermaid_head())
|
||||
dependency_html = "\n".join(dependencies)
|
||||
return (
|
||||
"<!doctype html>\n"
|
||||
'<html lang="und">\n'
|
||||
"<head>\n"
|
||||
'<meta charset="utf-8">\n'
|
||||
'<meta name="viewport" content="width=device-width, initial-scale=1.0">\n'
|
||||
f"<title>{title}</title>\n"
|
||||
f"<style>{styles}</style>\n"
|
||||
f"{dependency_html}\n"
|
||||
"</head>\n"
|
||||
'<body class="mineru-html-body">\n'
|
||||
f"{article}\n"
|
||||
"</body>\n"
|
||||
"</html>"
|
||||
)
|
||||
from ..types import MiddleJson
|
||||
from .contracts import RenderMode
|
||||
|
||||
|
||||
def render_html(
|
||||
@@ -638,304 +15,16 @@ def render_html(
|
||||
standalone: bool = True,
|
||||
document_title: str | None = None,
|
||||
) -> str:
|
||||
"""把严格 MiddleJson 无副作用地渲染为 HTML 文档或单根 fragment。"""
|
||||
if not isinstance(middle_json, MiddleJson):
|
||||
raise TypeError("render_html expects a MiddleJson instance")
|
||||
if not isinstance(mode, RenderMode):
|
||||
raise TypeError("mode must be a RenderMode value")
|
||||
if not isinstance(asset_base_url, str):
|
||||
raise TypeError("asset_base_url must be a string")
|
||||
if not isinstance(standalone, bool):
|
||||
raise TypeError("standalone must be a bool")
|
||||
if document_title is not None and not isinstance(document_title, str):
|
||||
raise TypeError("document_title must be a string or None")
|
||||
return _HtmlRenderer(
|
||||
"""惰性加载 HTML 实现并渲染严格 MiddleJson。"""
|
||||
from ._internal.html.renderer import render_html as _render_html
|
||||
|
||||
return _render_html(
|
||||
middle_json,
|
||||
mode=mode,
|
||||
asset_base_url=asset_base_url,
|
||||
standalone=standalone,
|
||||
document_title=document_title,
|
||||
).render()
|
||||
|
||||
|
||||
def _classify_list(items: list[ListItem], add_reference_bullets: bool) -> tuple[str, str | None, str]:
|
||||
"""根据直属 marker 选择原生列表类型或显式 marker 模式。"""
|
||||
if add_reference_bullets:
|
||||
return "ul", None, "mineru-list--reference"
|
||||
if items and all(item.kind == "unordered" for item in items):
|
||||
return "ul", None, "mineru-list--unordered"
|
||||
if items and all(item.kind == "ordered" for item in items):
|
||||
styles = {item.ordered_style for item in items}
|
||||
if len(styles) == 1:
|
||||
style = next(iter(styles))
|
||||
list_type = {
|
||||
"lower-alpha": "a",
|
||||
"upper-alpha": "A",
|
||||
"lower-roman": "i",
|
||||
"upper-roman": "I",
|
||||
}.get(style or "")
|
||||
return "ol", list_type, "mineru-list--ordered"
|
||||
if items and all(item.kind == "none" for item in items):
|
||||
return "ul", None, "mineru-list--unmarked"
|
||||
return "ul", None, "mineru-list--explicit"
|
||||
|
||||
|
||||
def _list_item_content(
|
||||
item: ListItem,
|
||||
add_reference_bullets: bool,
|
||||
*,
|
||||
explicit_markers: bool,
|
||||
) -> tuple[str, str | None]:
|
||||
"""决定一个列表项应剥离、保留还是显式显示源 marker。"""
|
||||
if add_reference_bullets:
|
||||
if item.kind == "unordered":
|
||||
return item.body, None
|
||||
original = f"{item.leading}{item.marker or ''}{item.separator}{item.body}"
|
||||
return original, None
|
||||
if explicit_markers:
|
||||
return item.body, item.marker
|
||||
if item.kind in {"unordered", "ordered"}:
|
||||
return item.body, None
|
||||
return item.body, item.marker
|
||||
|
||||
|
||||
def _serialize_index_items(items: list[dict[str, object]]) -> str:
|
||||
"""把目录项及其已挂接的嵌套 ul 序列化为 li。"""
|
||||
rendered: list[str] = []
|
||||
for item in items:
|
||||
nested = item["nested"]
|
||||
assert isinstance(nested, list)
|
||||
content = str(item["content"])
|
||||
if not content and not nested:
|
||||
continue
|
||||
class_attr = ' class="mineru-list-item--orphan"' if item["orphan"] else ""
|
||||
rendered.append(f"<li{class_attr}>{content}{''.join(str(value) for value in nested)}</li>")
|
||||
return "".join(rendered)
|
||||
|
||||
|
||||
def _collect_title_anchor_ids(middle_json: MiddleJson) -> dict[str, str]:
|
||||
"""为正文首次出现的非空 anchor 分配无空白且 document-wide 唯一的 HTML id。"""
|
||||
anchor_ids: dict[str, str] = {}
|
||||
used_ids: set[str] = set()
|
||||
for page in middle_json.pages:
|
||||
for block in page.blocks:
|
||||
if not isinstance(block, TitleBlockBase):
|
||||
continue
|
||||
anchor = _anchor_key(block.anchor)
|
||||
visible_title = inline_plain_text(parse_inline_content(block.content)).strip()
|
||||
if not anchor or not visible_title or anchor in anchor_ids:
|
||||
continue
|
||||
base_id = _html_anchor_base(anchor)
|
||||
candidate = base_id
|
||||
suffix = 2
|
||||
while candidate in used_ids:
|
||||
candidate = f"{base_id}-{suffix}"
|
||||
suffix += 1
|
||||
anchor_ids[anchor] = candidate
|
||||
used_ids.add(candidate)
|
||||
return anchor_ids
|
||||
|
||||
|
||||
def _plain_content_text(content: str) -> str:
|
||||
"""从 body 内容提取安全 img alt 所需的可见纯文本。"""
|
||||
if not content:
|
||||
return ""
|
||||
if is_supported_html_markup(content):
|
||||
return BeautifulSoup(content, "html.parser").get_text(" ", strip=True)
|
||||
return inline_plain_text(parse_inline_content(content)).strip()
|
||||
|
||||
|
||||
def _extract_mermaid_flowchart_source(content: str) -> str | None:
|
||||
"""提取受限 Mermaid flowchart fence,拒绝其他图类型和可改写配置的语法。"""
|
||||
normalized = _replace_html_controls(content).replace("\r\n", "\n").replace("\r", "\n").strip()
|
||||
if not normalized:
|
||||
return None
|
||||
lines = normalized.split("\n")
|
||||
if len(lines) < 3:
|
||||
return None
|
||||
opening = re.fullmatch(r"(?P<fence>\x60{3,})[ \t]*mermaid[ \t]*", lines[0], re.IGNORECASE)
|
||||
if opening is None:
|
||||
return None
|
||||
fence_length = len(opening.group("fence"))
|
||||
if re.fullmatch(rf"\x60{{{fence_length},}}[ \t]*", lines[-1]) is None:
|
||||
return None
|
||||
source = "\n".join(lines[1:-1]).strip("\n")
|
||||
if not source or len(source) > _MERMAID_MAX_TEXT_SIZE:
|
||||
return None
|
||||
if source.lstrip().startswith("---") or "%%{" in source:
|
||||
return None
|
||||
first_statement = next(
|
||||
(line.strip() for line in source.splitlines() if line.strip() and not line.lstrip().startswith("%%")),
|
||||
"",
|
||||
)
|
||||
if _MERMAID_FLOWCHART_HEADER_RE.match(first_statement) is None:
|
||||
return None
|
||||
return source
|
||||
|
||||
|
||||
def _contains_usable_table(markup: str) -> bool:
|
||||
"""确认清洗后的 markup 至少保留一个含单元格的 table。"""
|
||||
soup = BeautifulSoup(markup, "html.parser")
|
||||
return any(table.find(("td", "th")) is not None for table in soup.find_all("table"))
|
||||
|
||||
|
||||
def _render_raw_fallback(content: str) -> str:
|
||||
"""把无法安全结构化的原内容转义为可见 pre,避免静默丢失。"""
|
||||
return f'<pre class="mineru-raw-fallback">{html.escape(_replace_html_controls(content), quote=False)}</pre>'
|
||||
|
||||
|
||||
def _normalize_prism_language(language: str | None) -> str | None:
|
||||
"""把模型语言名归一化为不可构造路径穿越的 Prism component 名。"""
|
||||
normalized = (language or "").strip().lower()
|
||||
normalized = _PRISM_LANGUAGE_ALIASES.get(normalized, normalized.replace("_", "-"))
|
||||
if not normalized or normalized in _BLOCKED_PRISM_LANGUAGE_NAMES:
|
||||
return None
|
||||
return normalized if _SAFE_PRISM_LANGUAGE_RE.fullmatch(normalized) else None
|
||||
|
||||
|
||||
def _resolve_document_title(middle_json: MiddleJson, explicit_title: str | None) -> str:
|
||||
"""按显式参数、首个正文文档标题、固定回退值解析 HTML title。"""
|
||||
if explicit_title and explicit_title.strip():
|
||||
return explicit_title.strip()
|
||||
for page in middle_json.pages:
|
||||
for block in page.blocks:
|
||||
if isinstance(block, DocTitleBlock):
|
||||
title = inline_plain_text(parse_inline_content(block.content)).strip()
|
||||
if title:
|
||||
return title
|
||||
return "MinerU Document"
|
||||
|
||||
|
||||
def _anchor_key(anchor: str | None) -> str:
|
||||
"""保留 producer anchor 身份,仅去除首尾空白供标题与目录精确匹配。"""
|
||||
return (anchor or "").strip()
|
||||
|
||||
|
||||
def _html_anchor_base(anchor: str) -> str:
|
||||
"""把原始 anchor 转成无空白的 HTML id 基值,碰撞由 document registry 处理。"""
|
||||
return re.sub(r"\s+", "-", _replace_html_controls(anchor))
|
||||
|
||||
|
||||
def _replace_html_controls(content: str) -> str:
|
||||
"""替换 HTML 不允许的 C0 控制字符与不可编码的孤立 surrogate。"""
|
||||
return re.sub(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f\ud800-\udfff]", "\ufffd", content)
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _load_html_styles() -> str:
|
||||
"""从包资源读取并缓存压缩后的 MinerU 独立样式。"""
|
||||
root = resources.files("mineru").joinpath("resources", "html")
|
||||
return root.joinpath(_STYLE_RESOURCE_NAME).read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def _mathjax_head() -> str:
|
||||
"""返回固定、收紧公式输入能力的 MathJax 4.1.2 head 片段。"""
|
||||
return f"""<script>
|
||||
window.MathJax = {{
|
||||
loader: {{load: ['ui/safe']}},
|
||||
tex: {{
|
||||
inlineMath: [['\\\\(', '\\\\)']],
|
||||
displayMath: [['\\\\[', '\\\\]']],
|
||||
processEscapes: true,
|
||||
processEnvironments: false,
|
||||
processRefs: false,
|
||||
packages: {{'[-]': ['require']}}
|
||||
}},
|
||||
options: {{
|
||||
ignoreHtmlClass: 'mineru-document',
|
||||
processHtmlClass: 'mineru-math',
|
||||
enableMenu: false,
|
||||
enableEnrichment: false,
|
||||
safeOptions: {{allow: {{URLs: 'none', classes: 'none', cssIDs: 'none', styles: 'none'}}}}
|
||||
}},
|
||||
startup: {{
|
||||
ready: function () {{
|
||||
window.MathJax.startup.defaultReady();
|
||||
var options = window.MathJax.startup.document.options;
|
||||
options.enableMenu = false;
|
||||
options.enableEnrichment = false;
|
||||
}}
|
||||
}}
|
||||
}};
|
||||
</script>
|
||||
<script id="MathJax-script" defer src="{_MATHJAX_URL}"
|
||||
integrity="{_MATHJAX_INTEGRITY}" crossorigin="anonymous" referrerpolicy="no-referrer"></script>"""
|
||||
|
||||
|
||||
def _prism_head() -> str:
|
||||
"""返回固定 Prism core、Autoloader 和当前文档局部高亮脚本。"""
|
||||
return f"""<script defer data-manual src="{_PRISM_CORE_URL}"
|
||||
integrity="{_PRISM_CORE_INTEGRITY}" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
|
||||
<script defer src="{_PRISM_AUTOLOADER_URL}"
|
||||
integrity="{_PRISM_AUTOLOADER_INTEGRITY}" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
|
||||
<script>
|
||||
document.addEventListener('DOMContentLoaded', function () {{
|
||||
if (!window.Prism || !Prism.plugins || !Prism.plugins.autoloader) return;
|
||||
Prism.plugins.autoloader.languages_path = '{_PRISM_LANGUAGES_PATH}';
|
||||
var root = document.querySelector('.mineru-document');
|
||||
if (root) Prism.highlightAllUnder(root);
|
||||
}});
|
||||
</script>"""
|
||||
|
||||
|
||||
def _mermaid_head() -> str:
|
||||
"""返回固定 Mermaid 入口和逐图安全渲染、失败回退脚本。"""
|
||||
return f"""<script defer src="{_MERMAID_URL}"
|
||||
integrity="{_MERMAID_INTEGRITY}" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
|
||||
<script>
|
||||
function markMineruMermaidError(host) {{
|
||||
host.dataset.mermaidState = 'error';
|
||||
if (!host.classList.contains('mineru-flowchart--has-raster')) {{
|
||||
var details = host.nextElementSibling;
|
||||
if (details && details.classList.contains('mineru-flowchart-details')) details.open = true;
|
||||
}}
|
||||
}}
|
||||
document.addEventListener('DOMContentLoaded', function () {{
|
||||
var root = document.querySelector('.mineru-document');
|
||||
var hosts = root ? Array.from(root.querySelectorAll('.mineru-flowchart')) : [];
|
||||
if (!hosts.length) return;
|
||||
if (!window.mermaid) {{
|
||||
hosts.forEach(markMineruMermaidError);
|
||||
return;
|
||||
}}
|
||||
try {{
|
||||
window.mermaid.initialize({{
|
||||
startOnLoad: false,
|
||||
securityLevel: 'strict',
|
||||
suppressErrorRendering: true,
|
||||
theme: 'neutral',
|
||||
maxTextSize: 50000,
|
||||
maxEdges: 500,
|
||||
flowchart: {{htmlLabels: false, useMaxWidth: true}}
|
||||
}});
|
||||
}} catch (_error) {{
|
||||
hosts.forEach(markMineruMermaidError);
|
||||
return;
|
||||
}}
|
||||
(async function () {{
|
||||
for (var index = 0; index < hosts.length; index += 1) {{
|
||||
var host = hosts[index];
|
||||
var details = host.nextElementSibling;
|
||||
var source = details && details.querySelector('.mineru-flowchart-source code');
|
||||
var canvas = host.querySelector('.mineru-flowchart-canvas');
|
||||
if (!source || !canvas) {{
|
||||
markMineruMermaidError(host);
|
||||
continue;
|
||||
}}
|
||||
var renderId = 'mineru-mermaid-' + index;
|
||||
while (document.getElementById(renderId)) renderId += '-x';
|
||||
host.dataset.mermaidState = 'rendering';
|
||||
try {{
|
||||
var result = await window.mermaid.render(renderId, source.textContent);
|
||||
canvas.innerHTML = result.svg;
|
||||
host.dataset.mermaidState = 'rendered';
|
||||
}} catch (_error) {{
|
||||
markMineruMermaidError(host);
|
||||
}}
|
||||
}}
|
||||
}})();
|
||||
}});
|
||||
</script>"""
|
||||
|
||||
|
||||
__all__ = ["render_html"]
|
||||
|
||||
+10
-53
@@ -1,16 +1,11 @@
|
||||
# Copyright (c) Opendatalab. All rights reserved.
|
||||
"""严格 MiddleJson 到 Markdown 的公共渲染实现。"""
|
||||
"""严格 MiddleJson 到 Markdown 的轻量公共门面。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from ..config import LatexDelimitersConfig, config
|
||||
from ..types import PAGE_AUXILIARY_BLOCK_TYPES, MiddleJson
|
||||
from ._internal.common.planner import PlannedBlock, build_render_plan
|
||||
from ._internal.markdown.blocks import render_planned_block
|
||||
from ..types import MiddleJson
|
||||
from .contracts import ImageRenderer, RenderMode
|
||||
|
||||
_PAGE_SEPARATOR = "\n\n---\n\n"
|
||||
|
||||
|
||||
def render_markdown(
|
||||
middle_json: MiddleJson,
|
||||
@@ -19,53 +14,15 @@ def render_markdown(
|
||||
asset_base_url: str = "",
|
||||
image_renderer: ImageRenderer | None = None,
|
||||
) -> str:
|
||||
"""把严格 MiddleJson 纯函数式渲染为 Markdown 字符串。"""
|
||||
if not isinstance(middle_json, MiddleJson):
|
||||
raise TypeError("render_markdown expects a MiddleJson instance")
|
||||
if not isinstance(mode, RenderMode):
|
||||
raise TypeError("mode must be a RenderMode value")
|
||||
"""惰性加载 Markdown 实现并渲染严格 MiddleJson。"""
|
||||
from ._internal.markdown.renderer import render_markdown as _render_markdown
|
||||
|
||||
delimiters = config.render.latex_delimiters
|
||||
planned_pages = build_render_plan(middle_json, mode)
|
||||
rendered_pages = [
|
||||
_render_page(
|
||||
page,
|
||||
mode=mode,
|
||||
delimiters=delimiters,
|
||||
asset_base_url=asset_base_url,
|
||||
image_renderer=image_renderer,
|
||||
)
|
||||
for page in planned_pages
|
||||
]
|
||||
if mode is RenderMode.FULL:
|
||||
return _PAGE_SEPARATOR.join(rendered_pages)
|
||||
return "\n\n".join(page for page in rendered_pages if page)
|
||||
|
||||
|
||||
def _render_page(
|
||||
planned_blocks: list[PlannedBlock],
|
||||
*,
|
||||
mode: RenderMode,
|
||||
delimiters: LatexDelimitersConfig,
|
||||
asset_base_url: str,
|
||||
image_renderer: ImageRenderer | None = None,
|
||||
) -> str:
|
||||
"""渲染单页逻辑块,并在默认模式中过滤重复页元素。"""
|
||||
rendered: list[str] = []
|
||||
for planned in planned_blocks:
|
||||
if planned.removed:
|
||||
continue
|
||||
if mode is RenderMode.DEFAULT and planned.block.type in PAGE_AUXILIARY_BLOCK_TYPES:
|
||||
continue
|
||||
text = render_planned_block(
|
||||
planned,
|
||||
delimiters=delimiters,
|
||||
asset_base_url=asset_base_url,
|
||||
image_renderer=image_renderer,
|
||||
)
|
||||
if text and text.strip():
|
||||
rendered.append(text.strip("\n"))
|
||||
return "\n\n".join(rendered)
|
||||
return _render_markdown(
|
||||
middle_json,
|
||||
mode=mode,
|
||||
asset_base_url=asset_base_url,
|
||||
image_renderer=image_renderer,
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["render_markdown"]
|
||||
|
||||
@@ -1,56 +1,11 @@
|
||||
# Copyright (c) Opendatalab. All rights reserved.
|
||||
"""严格 MiddleJson 到树形 Markdown structured_content 的公共渲染实现。"""
|
||||
"""严格 MiddleJson 到 Structured Content 的轻量公共门面。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, TypeAlias
|
||||
from typing import Any
|
||||
|
||||
from mineru.config import LatexDelimitersConfig, config
|
||||
from mineru.render._internal.markdown.blocks import (
|
||||
render_single_block,
|
||||
render_title_inline_content,
|
||||
render_visual_annotation,
|
||||
render_visual_body_content,
|
||||
)
|
||||
from mineru.render._internal.markdown.assets import normalize_image_source, resolve_image_source
|
||||
from mineru.render._internal.markdown.escaping import escape_standalone_marker_rule, escape_text_block_markdown_prefix
|
||||
from mineru.types import (
|
||||
BlockType,
|
||||
ChartAnnotationBlock,
|
||||
ChartBlock,
|
||||
ChartBodyBlock,
|
||||
CodeAnnotationBlock,
|
||||
CodeBlock,
|
||||
DocTitleBlock,
|
||||
EquationBlock,
|
||||
ImageAnnotationBlock,
|
||||
ImageBlock,
|
||||
ImageBodyBlock,
|
||||
ImagePayloadBlock,
|
||||
MiddleJson,
|
||||
PageBlock,
|
||||
ParagraphTitleBlock,
|
||||
TableAnnotationBlock,
|
||||
TableBlock,
|
||||
TableBodyBlock,
|
||||
)
|
||||
|
||||
VisualBlock: TypeAlias = ImageBlock | TableBlock | ChartBlock | CodeBlock
|
||||
VisualAnnotationBlock: TypeAlias = ImageAnnotationBlock | TableAnnotationBlock | ChartAnnotationBlock | CodeAnnotationBlock
|
||||
|
||||
_CAPTION_TYPES = {
|
||||
BlockType.IMAGE_CAPTION,
|
||||
BlockType.TABLE_CAPTION,
|
||||
BlockType.CHART_CAPTION,
|
||||
BlockType.CODE_CAPTION,
|
||||
}
|
||||
_FOOTNOTE_TYPES = {
|
||||
BlockType.IMAGE_FOOTNOTE,
|
||||
BlockType.TABLE_FOOTNOTE,
|
||||
BlockType.CHART_FOOTNOTE,
|
||||
BlockType.CODE_FOOTNOTE,
|
||||
}
|
||||
_REMOVED_BLOCK_FIELDS = {"content", "index", "guess_lang", "image_path", "image_base64"}
|
||||
from ..types import MiddleJson
|
||||
|
||||
|
||||
def render_structured_content(
|
||||
@@ -58,124 +13,10 @@ def render_structured_content(
|
||||
*,
|
||||
asset_base_url: str = "",
|
||||
) -> dict[str, Any]:
|
||||
"""把严格 MiddleJson 无副作用地渲染为树形 Markdown structured_content。"""
|
||||
if not isinstance(middle_json, MiddleJson):
|
||||
raise TypeError("render_structured_content expects a MiddleJson instance")
|
||||
"""惰性加载 Structured Content 实现并渲染严格 MiddleJson。"""
|
||||
from ._internal.structured_content.renderer import render_structured_content as _render_structured_content
|
||||
|
||||
delimiters = config.render.latex_delimiters
|
||||
document_fields = middle_json.model_dump(
|
||||
mode="json",
|
||||
exclude={"pages"},
|
||||
exclude_defaults=True,
|
||||
)
|
||||
pages = [
|
||||
{
|
||||
"page_idx": page.page_idx,
|
||||
"blocks": [
|
||||
_render_content_block(
|
||||
block,
|
||||
delimiters=delimiters,
|
||||
asset_base_url=asset_base_url,
|
||||
)
|
||||
for block in page.blocks
|
||||
],
|
||||
}
|
||||
for page in middle_json.pages
|
||||
]
|
||||
return {"pages": pages, **document_fields}
|
||||
|
||||
|
||||
def _render_content_block(
|
||||
block: PageBlock,
|
||||
*,
|
||||
delimiters: LatexDelimitersConfig,
|
||||
asset_base_url: str,
|
||||
) -> dict[str, Any]:
|
||||
"""保留父块元数据,并把 block 内容收敛为 Markdown 字符串。"""
|
||||
payload = block.model_dump(
|
||||
mode="json",
|
||||
exclude=_REMOVED_BLOCK_FIELDS,
|
||||
exclude_defaults=True,
|
||||
)
|
||||
if isinstance(block, (DocTitleBlock, ParagraphTitleBlock)):
|
||||
content = render_title_inline_content(block, delimiters)
|
||||
payload["content"] = escape_standalone_marker_rule(escape_text_block_markdown_prefix(content))
|
||||
return payload
|
||||
if isinstance(block, EquationBlock):
|
||||
payload["content"] = block.content.strip()
|
||||
image_source = _resolve_content_image_source(block, asset_base_url)
|
||||
if image_source is not None:
|
||||
payload["image_source"] = image_source
|
||||
return payload
|
||||
if not isinstance(block, (ImageBlock, TableBlock, ChartBlock, CodeBlock)):
|
||||
payload["content"] = render_single_block(
|
||||
block,
|
||||
delimiters=delimiters,
|
||||
asset_base_url=asset_base_url,
|
||||
)
|
||||
return payload
|
||||
|
||||
payload["content"] = render_visual_body_content(
|
||||
block,
|
||||
delimiters=delimiters,
|
||||
asset_base_url=asset_base_url,
|
||||
)
|
||||
payload["captions"] = _render_annotation_group(block, _CAPTION_TYPES, delimiters)
|
||||
payload["footnotes"] = _render_annotation_group(block, _FOOTNOTE_TYPES, delimiters)
|
||||
image_source = _resolve_visual_image_source(block, asset_base_url)
|
||||
if image_source is not None:
|
||||
payload["image_source"] = image_source
|
||||
return payload
|
||||
|
||||
|
||||
def _render_annotation_group(
|
||||
block: VisualBlock,
|
||||
accepted_types: set[str],
|
||||
delimiters: LatexDelimitersConfig,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""按源 index 稳定排序视觉说明,并保留可用 bbox 与 Markdown 内容。"""
|
||||
annotations: list[tuple[int, VisualAnnotationBlock]] = []
|
||||
for position, child in enumerate(block.content):
|
||||
if (
|
||||
isinstance(
|
||||
child,
|
||||
(ImageAnnotationBlock, TableAnnotationBlock, ChartAnnotationBlock, CodeAnnotationBlock),
|
||||
)
|
||||
and child.type in accepted_types
|
||||
):
|
||||
annotations.append((position, child))
|
||||
annotations.sort(key=_annotation_sort_key)
|
||||
rendered_annotations: list[dict[str, Any]] = []
|
||||
for _, child in annotations:
|
||||
annotation_payload: dict[str, Any] = {}
|
||||
if child.bbox is not None:
|
||||
annotation_payload["bbox"] = list(child.bbox)
|
||||
annotation_payload["content"] = render_visual_annotation(child, delimiters)
|
||||
rendered_annotations.append(annotation_payload)
|
||||
return rendered_annotations
|
||||
|
||||
|
||||
def _annotation_sort_key(
|
||||
item: tuple[int, VisualAnnotationBlock],
|
||||
) -> tuple[bool, int, int]:
|
||||
"""让有 index 的说明升序优先,缺失 index 的说明稳定排在末尾。"""
|
||||
position, block = item
|
||||
return block.index is None, block.index if block.index is not None else 0, position
|
||||
|
||||
|
||||
def _resolve_visual_image_source(block: VisualBlock, asset_base_url: str) -> str | None:
|
||||
"""解析视觉 body 实际选择的图片来源,并返回安全的 Markdown 地址。"""
|
||||
for child in block.content:
|
||||
if not isinstance(child, (ImageBodyBlock, TableBodyBlock, ChartBodyBlock)):
|
||||
continue
|
||||
return _resolve_content_image_source(child, asset_base_url)
|
||||
return None
|
||||
|
||||
|
||||
def _resolve_content_image_source(block: ImagePayloadBlock, asset_base_url: str) -> str | None:
|
||||
"""把图片载荷收敛为 structured_content 中唯一且安全的 image_source。"""
|
||||
source = resolve_image_source(block, asset_base_url)
|
||||
return normalize_image_source(source) if source else None
|
||||
return _render_structured_content(middle_json, asset_base_url=asset_base_url)
|
||||
|
||||
|
||||
__all__ = ["render_structured_content"]
|
||||
|
||||
@@ -1,63 +0,0 @@
|
||||
# Copyright (c) Opendatalab. All rights reserved.
|
||||
import os
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
|
||||
class DataWriter(ABC):
|
||||
@abstractmethod
|
||||
def write(self, path: str, data: bytes) -> None:
|
||||
"""Write the data to the file.
|
||||
|
||||
Args:
|
||||
path (str): the target file where to write
|
||||
data (bytes): the data want to write
|
||||
"""
|
||||
pass
|
||||
|
||||
def write_string(self, path: str, data: str) -> None:
|
||||
"""Write the data to file, the data will be encoded to bytes.
|
||||
|
||||
Args:
|
||||
path (str): the target file where to write
|
||||
data (str): the data want to write
|
||||
"""
|
||||
|
||||
def safe_encode(data: str, method: str):
|
||||
try:
|
||||
bit_data = data.encode(encoding=method, errors="replace")
|
||||
return bit_data, True
|
||||
except: # noqa
|
||||
return None, False
|
||||
|
||||
for method in ["utf-8", "ascii"]:
|
||||
bit_data, flag = safe_encode(data, method)
|
||||
if flag:
|
||||
self.write(path, bit_data)
|
||||
break
|
||||
|
||||
|
||||
class FileBasedDataWriter(DataWriter):
|
||||
def __init__(self, parent_dir: str = "") -> None:
|
||||
"""Initialized with parent_dir.
|
||||
|
||||
Args:
|
||||
parent_dir (str, optional): the parent directory that may be used within methods. Defaults to ''.
|
||||
"""
|
||||
self._parent_dir = parent_dir
|
||||
|
||||
def write(self, path: str, data: bytes) -> None:
|
||||
"""Write file with data.
|
||||
|
||||
Args:
|
||||
path (str): the path of file, if the path is relative path, it will be joined with parent_dir.
|
||||
data (bytes): the data want to write
|
||||
"""
|
||||
fn_path = path
|
||||
if not os.path.isabs(fn_path) and len(self._parent_dir) > 0:
|
||||
fn_path = os.path.join(self._parent_dir, path)
|
||||
|
||||
if not os.path.exists(os.path.dirname(fn_path)) and os.path.dirname(fn_path) != "":
|
||||
os.makedirs(os.path.dirname(fn_path), exist_ok=True)
|
||||
|
||||
with open(fn_path, "wb") as f:
|
||||
f.write(data)
|
||||
Reference in New Issue
Block a user