refactor: enforce backend and render ownership boundaries

This commit is contained in:
myhloli
2026-08-25 23:30:54 +08:00
parent e1c72fc6d3
commit 0bd6c38dcc
72 changed files with 2376 additions and 2240 deletions
+1 -1
View File
@@ -8,7 +8,7 @@ import httpx
from mineru.cli_old import api_client as _api_client
from mineru.cli_old.common import image_suffixes, office_suffixes, pdf_suffixes
from mineru.utils.guess_suffix_or_lang import guess_suffix_by_path
from mineru.parser.file_type import guess_suffix_by_path
SUPPORTED_INPUT_SUFFIXES = set(pdf_suffixes + image_suffixes + office_suffixes)
+2
View File
@@ -1 +1,3 @@
# Copyright (c) Opendatalab. All rights reserved.
__all__: list[str] = []
+2
View File
@@ -1,2 +1,4 @@
# Copyright (c) Opendatalab. All rights reserved.
"""文档 model-list 生产层。"""
__all__: list[str] = []
+1 -1
View File
@@ -6,7 +6,7 @@ from __future__ import annotations
import time
from io import BytesIO
from mineru.model.flash import DocModel, DocxModel, PptModel, PptxModel, XlsModel, XlsxModel
from ...model.flash import DocModel, DocxModel, PptModel, PptxModel, XlsModel, XlsxModel
from .contracts import AnalysisResult, OfficeSuffix
+2
View File
@@ -1,2 +1,4 @@
# Copyright (c) Opendatalab. All rights reserved.
"""PDF 分析流水线。"""
__all__: list[str] = []
+1 -1
View File
@@ -5,7 +5,7 @@ from __future__ import annotations
import re
from mineru.types import RAW_ALGORITHM, RAW_CAPTION, RAW_FOOTNOTE, RAW_FORMULA_NUMBER, RAW_PHONETIC, BlockType
from ....types import RAW_ALGORITHM, RAW_CAPTION, RAW_FOOTNOTE, RAW_FORMULA_NUMBER, RAW_PHONETIC, BlockType
LAYOUT_BASE_BATCH_SIZE = 1
MFR_BASE_BATCH_SIZE = 16
+4 -4
View File
@@ -10,10 +10,10 @@ import cv2
import numpy as np
from PIL import Image
from mineru.backend.local_model_runtime import HybridLocalModelContext
from mineru.utils.bbox_utils import normalize_to_int_bbox
from mineru.utils.ocr_utils import get_rotate_crop_image_for_text_rec
from mineru.utils.text_utils import build_tagged_formula_content
from ....model.runtime.hybrid import HybridLocalModelContext
from ....utils.geometry import normalize_to_int_bbox
from ....model.ocr.image import get_rotate_crop_image_for_text_rec
from ....utils.text import build_tagged_formula_content
from ....types import RAW_FORMULA_NUMBER, BlockType
+2 -2
View File
@@ -10,8 +10,8 @@ import cv2
import numpy as np
from loguru import logger
from mineru.types import BBox
from mineru.utils.bbox_utils import normalize_to_int_bbox
from ....types import BBox
from ....utils.geometry import normalize_to_int_bbox
def _normalize_page_size(page_image: Any) -> tuple[int, int]:
@@ -8,21 +8,49 @@ import threading
import time
from concurrent.futures import ALL_COMPLETED, Future, ProcessPoolExecutor, wait
from concurrent.futures.process import BrokenProcessPool
from io import BytesIO
import math
from typing import Any, Callable, Literal
import numpy as np
import pypdfium2 as pdfium
from loguru import logger
from PIL import Image, ImageOps
from PIL import Image
from ....model.flash.pdf.pdfium import close_pdfium_child, pdfium_guard
from ....model.flash.pdf.raster import image_to_b64str, page_to_image
from ....types import BBox, IntBBox
from ....utils.geometry import normalize_to_int_bbox
from ....utils.platform import is_windows_environment
class ImageType:
"""限定 PDF 页面渲染支持的两种图像返回形态。"""
PIL = "pil_img"
BASE64 = "base64_img"
def _positive_int_env(name: str, default: int) -> int:
"""读取正整数环境变量,缺失或非法时返回默认值。"""
raw_value = os.getenv(name)
if raw_value is None:
return default
try:
value = int(raw_value)
except ValueError:
return default
return value if value > 0 else default
def get_load_images_timeout() -> int:
"""返回 PDF 批量渲染的超时秒数。"""
return _positive_int_env("MINERU_PDF_RENDER_TIMEOUT", 300)
def get_load_images_threads() -> int:
"""返回 PDF 批量渲染允许使用的进程数。"""
return _positive_int_env("MINERU_PDF_RENDER_THREADS", 3)
from ..types import BBox
from .bbox_utils import normalize_to_int_bbox
from .check_sys_env import is_windows_environment
from .enum_class import ImageType
from .os_env_config import get_load_images_threads, get_load_images_timeout
from .pdf_reader import image_to_b64str, page_to_image
from .pdfium_guard import close_pdfium_child, pdfium_guard
DEFAULT_PDF_IMAGE_DPI = 200
# DEFAULT_PDF_IMAGE_DPI = 144
@@ -34,6 +62,7 @@ PDF_RENDER_KILL_JOIN_TIMEOUT_SECONDS = 0.1
_pdf_render_executor: ProcessPoolExecutor | None = None
_pdf_render_executor_lock = threading.Lock()
_pdf_render_atexit_registered = False
_pdf_render_spawn_submit_lock = threading.Lock()
_pdf_render_spawn_submit_executor_id: int | None = None
_pdf_render_spawn_submit_count = 0
@@ -44,16 +73,7 @@ def pdf_page_to_image(
dpi: int = DEFAULT_PDF_IMAGE_DPI,
image_type: Literal["pil_img", "base64_img"] = ImageType.PIL,
) -> dict[str, Any]:
"""Convert pdfium.PdfDocument to image, Then convert the image to base64.
Args:
page (_type_): pdfium.PdfPage
dpi (int, optional): reset the dpi of dpi. Defaults to DEFAULT_PDF_IMAGE_DPI.
image_type (ImageType, optional): The type of image to return. Defaults to ImageType.PIL.
Returns:
dict: {'img_base64': str, 'img_pil': pil_img, 'scale': float }
"""
"""将单个 PDFium 页面渲染为 Pillow 图像或 base64 载荷。"""
pil_img, scale = page_to_image(page, dpi=dpi)
image_dict: dict[str, Any] = {"scale": scale}
if image_type == ImageType.BASE64:
@@ -109,6 +129,7 @@ def _close_image_dicts(images_list: list[dict[str, Any]] | None) -> None:
def _calculate_render_process_count(total_pages: int, threads: int, cpu_count: int | None = None) -> int:
"""按页数、配置和 CPU 数量计算实际渲染进程数。"""
requested_threads = max(1, threads)
available_cpus = max(1, cpu_count if cpu_count is not None else (os.cpu_count() or 1))
page_limited_threads = max(1, total_pages // MIN_PAGES_PER_RENDER_PROCESS)
@@ -125,6 +146,7 @@ def _build_render_page_ranges(
end_page_id: int,
process_count: int,
) -> list[tuple[int, int]]:
"""把闭区间页范围均匀拆分为指定数量的子范围。"""
total_pages = end_page_id - start_page_id + 1
base_pages, remainder = divmod(total_pages, process_count)
page_ranges = []
@@ -145,12 +167,14 @@ def _get_render_process_plan(
threads: int,
cpu_count: int | None = None,
) -> tuple[int, list[tuple[int, int]]]:
"""同时返回实际进程数及其页面分片计划。"""
total_pages = end_page_id - start_page_id + 1
actual_threads = _calculate_render_process_count(total_pages, threads, cpu_count)
return actual_threads, _build_render_page_ranges(start_page_id, end_page_id, actual_threads)
def _get_pdf_render_pool_capacity(cpu_count: int | None = None) -> int:
"""返回持久 PDF 渲染进程池的最大容量。"""
available_cpus = max(1, cpu_count if cpu_count is not None else (os.cpu_count() or 1))
configured_threads = max(1, get_load_images_threads())
return min(
@@ -161,6 +185,7 @@ def _get_pdf_render_pool_capacity(cpu_count: int | None = None) -> int:
def _exit_pdf_render_worker_when_parent_exits() -> None:
"""等待父进程退出后立即终止孤立渲染 worker。"""
parent = multiprocessing.parent_process()
if parent is None:
return
@@ -169,6 +194,7 @@ def _exit_pdf_render_worker_when_parent_exits() -> None:
def _install_pdf_render_parent_exit_watcher() -> None:
"""在渲染 worker 中安装父进程退出监听线程。"""
watcher = threading.Thread(
target=_exit_pdf_render_worker_when_parent_exits,
name="mineru-pdf-render-parent-exit-watcher",
@@ -178,6 +204,7 @@ def _install_pdf_render_parent_exit_watcher() -> None:
def _create_pdf_render_executor(max_workers: int) -> ProcessPoolExecutor:
"""使用安全 multiprocessing 上下文创建 PDF 渲染进程池。"""
if is_windows_environment():
return ProcessPoolExecutor(max_workers=max_workers, initializer=_install_pdf_render_parent_exit_watcher)
@@ -234,6 +261,7 @@ def _submit_pdf_render_task(
def _get_pdf_render_future_states(
future_to_range: dict[Future[Any], tuple[int, int]],
) -> list[dict[str, Any]]:
"""汇总每个渲染 Future 的页面范围和运行状态。"""
states = []
for future, (range_start, range_end) in future_to_range.items():
try:
@@ -257,6 +285,7 @@ def _get_pdf_render_future_states(
def _get_pdf_render_worker_states(executor: ProcessPoolExecutor) -> list[dict[str, Any]]:
"""读取进程池 worker 的 pid、存活状态和退出码。"""
try:
process_map = getattr(executor, "_processes", None) or {}
processes = list(process_map.values())
@@ -288,10 +317,14 @@ def _get_pdf_render_worker_states(executor: ProcessPoolExecutor) -> list[dict[st
def _get_pdf_render_executor() -> ProcessPoolExecutor:
global _pdf_render_executor
"""惰性创建并复用持久 PDF 渲染进程池。"""
global _pdf_render_atexit_registered, _pdf_render_executor
with _pdf_render_executor_lock:
if _pdf_render_executor is None:
if not _pdf_render_atexit_registered:
atexit.register(shutdown_pdf_render_executor)
_pdf_render_atexit_registered = True
max_workers = _get_pdf_render_pool_capacity()
_pdf_render_executor = _create_pdf_render_executor(max_workers=max_workers)
logger.debug(f"Created persistent PDF render executor with max_workers={max_workers}")
@@ -303,6 +336,7 @@ def _recycle_pdf_render_executor(
*,
terminate_processes: bool,
) -> None:
"""从全局缓存移除指定进程池并按需终止 worker。"""
global _pdf_render_executor
if executor is None:
@@ -324,6 +358,7 @@ def _recycle_pdf_render_executor(
def shutdown_pdf_render_executor() -> None:
"""关闭当前持久 PDF 渲染进程池。"""
global _pdf_render_executor
with _pdf_render_executor_lock:
@@ -337,9 +372,6 @@ def shutdown_pdf_render_executor() -> None:
)
atexit.register(shutdown_pdf_render_executor)
def load_images_from_pdf_bytes_range(
pdf_bytes: bytes,
dpi: int = DEFAULT_PDF_IMAGE_DPI,
@@ -349,6 +381,7 @@ def load_images_from_pdf_bytes_range(
timeout: int | None = None,
threads: int | None = None,
) -> list[dict[str, Any]]:
"""在持久进程池中批量渲染指定闭区间页面。"""
if end_page_id < start_page_id:
return []
@@ -500,6 +533,7 @@ def load_images_from_pdf_core(
end_page_id: int | None = None,
image_type: Literal["pil_img", "base64_img"] = ImageType.PIL,
) -> list[dict[str, Any]]:
"""在当前进程中依次渲染 PDF 页面范围。"""
images_list = []
pdf_doc = None
try:
@@ -527,7 +561,50 @@ def load_images_from_pdf_core(
return images_list
def _model_input_bbox(item: dict[str, Any]) -> IntBBox:
"""把模型输入项的 bbox 向外取整为裁图整数坐标。"""
bbox = item["bbox"]
assert bbox is not None
xmin, ymin, xmax, ymax = [float(value) for value in bbox]
return math.floor(xmin), math.floor(ymin), math.ceil(xmax), math.ceil(ymax)
def crop_img(
input_res: dict[str, Any],
input_img: Image.Image | np.ndarray,
crop_paste_x: int = 0,
crop_paste_y: int = 0,
) -> tuple[Image.Image | np.ndarray, list[int]]:
"""按模型 bbox 裁图并在四周补白,返回坐标回投所需参数。"""
crop_xmin, crop_ymin, crop_xmax, crop_ymax = _model_input_bbox(input_res)
crop_new_width = crop_xmax - crop_xmin + crop_paste_x * 2
crop_new_height = crop_ymax - crop_ymin + crop_paste_y * 2
if isinstance(input_img, np.ndarray):
return_image: Image.Image | np.ndarray = np.ones((crop_new_height, crop_new_width, 3), dtype=np.uint8) * 255
cropped_img = input_img[crop_ymin:crop_ymax, crop_xmin:crop_xmax]
return_image[
crop_paste_y : crop_paste_y + (crop_ymax - crop_ymin),
crop_paste_x : crop_paste_x + (crop_xmax - crop_xmin),
] = cropped_img
else:
return_image = Image.new("RGB", (crop_new_width, crop_new_height), "white")
cropped_img = input_img.crop((crop_xmin, crop_ymin, crop_xmax, crop_ymax))
return_image.paste(cropped_img, (crop_paste_x, crop_paste_y))
useful_list = [
crop_paste_x,
crop_paste_y,
crop_xmin,
crop_ymin,
crop_xmax,
crop_ymax,
crop_new_width,
crop_new_height,
]
return return_image, useful_list
def get_crop_img(bbox: BBox, pil_img: Image.Image, scale: float = 2.0) -> Image.Image:
"""按缩放 bbox 裁剪 Pillow 图像。"""
scale_bbox = normalize_to_int_bbox([float(v) * scale for v in bbox])
if scale_bbox is None:
return pil_img.crop((0, 0, 0, 0))
@@ -535,6 +612,7 @@ def get_crop_img(bbox: BBox, pil_img: Image.Image, scale: float = 2.0) -> Image.
def get_crop_np_img(bbox: BBox, input_img: Image.Image | np.ndarray, scale: float = 2.0) -> np.ndarray:
"""按缩放 bbox 裁剪 Pillow 或 NumPy 图像并返回数组。"""
if isinstance(input_img, Image.Image):
np_img = np.asarray(input_img)
elif isinstance(input_img, np.ndarray):
@@ -553,28 +631,13 @@ def get_crop_np_img(bbox: BBox, input_img: Image.Image | np.ndarray, scale: floa
return np_img[scale_bbox[1] : scale_bbox[3], scale_bbox[0] : scale_bbox[2]]
def images_bytes_to_pdf_bytes(image_bytes: bytes) -> bytes:
# 内存缓冲区
pdf_buffer = BytesIO()
# 载入并转换所有图像为 RGB 模式
image = Image.open(BytesIO(image_bytes))
# 根据 EXIF 信息自动转正(处理手机拍摄的带 Orientation 标记的图片)
image = ImageOps.exif_transpose(image) or image
# 只在必要时转换
if image.mode != "RGB":
image = image.convert("RGB")
# 第一张图保存为 PDF,其余追加
image.save(
pdf_buffer,
format="PDF",
resolution=DEFAULT_PDF_IMAGE_DPI,
quality=95,
subsampling=0,
)
# 获取 PDF bytes 并重置指针(可选)
pdf_bytes = pdf_buffer.getvalue()
pdf_buffer.close()
return pdf_bytes
__all__ = [
"ImageType",
"crop_img",
"get_crop_img",
"get_crop_np_img",
"load_images_from_pdf_bytes_range",
"load_images_from_pdf_core",
"pdf_page_to_image",
"shutdown_pdf_render_executor",
]
+4 -4
View File
@@ -10,9 +10,9 @@ from typing import Any
import numpy as np
from PIL import Image
from mineru.types import BBox, BlockType
from mineru.utils.bbox_utils import normalize_to_int_bbox
from mineru.utils.pdf_image_tools import get_crop_np_img
from ....types import BBox, BlockType
from ....utils.geometry import normalize_to_int_bbox
from .images import get_crop_np_img
from .geometry import _normalize_layout_bbox_to_unit, _normalize_page_size
from .constants import (
@@ -24,7 +24,7 @@ from .constants import (
def _load_vlm_runtime() -> dict[str, Any]:
"""按需加载 VLM runtime 组件,确保只有 high/extra_high 路径触发 VLM 依赖。"""
from mineru.model.vlm.runtime import (
from ....model.vlm.runtime import (
ModelSingleton,
_get_model_async,
_maybe_enable_serial_execution,
+3 -3
View File
@@ -8,9 +8,9 @@ import re
from collections.abc import Callable
from typing import Any
from mineru.types import BBox, BlockType, RAW_PHONETIC
from mineru.utils.bbox_utils import calculate_overlap_area_2_minbox_area_ratio
from mineru.utils.text_utils import full_to_half_exclude_marks
from ....types import BBox, BlockType, RAW_PHONETIC
from ....utils.geometry import calculate_overlap_area_2_minbox_area_ratio
from ....utils.text import full_to_half_exclude_marks
from .constants import (
LAYOUT_TITLE_SPLIT_OVERLAP_THRESHOLD,
+7 -13
View File
@@ -10,19 +10,13 @@ import cv2
import numpy as np
from tqdm import tqdm
from mineru.backend.local_model_runtime import HybridLocalModelContext
from mineru.types import BlockType
from mineru.utils.bbox_utils import normalize_to_int_bbox
from mineru.utils.model_utils import crop_img
from mineru.utils.ocr_utils import (
OcrConfidence,
get_adjusted_mfdetrec_res,
get_ocr_result_list,
mask_formula_regions_for_ocr_det,
merge_det_boxes,
sorted_boxes,
update_det_boxes,
)
from ....model.runtime.hybrid import HybridLocalModelContext
from ....types import BlockType
from ....utils.geometry import normalize_to_int_bbox
from .images import crop_img
from ....model.ocr.geometry import merge_det_boxes, sorted_boxes, update_det_boxes
from ....model.ocr.image import mask_formula_regions_for_ocr_det
from ....model.ocr.results import OcrConfidence, get_adjusted_mfdetrec_res, get_ocr_result_list
from .constants import (
BATCH_RATIO,
+9 -8
View File
@@ -3,22 +3,19 @@
from __future__ import annotations
import os
import time
from typing import Any, cast
from mineru.backend.analysis.contracts import AnalysisResult, AnalyzeEffort, ParseMode, ResolvedParseMode
from mineru.backend.local_model_runtime import HybridLocalModelContextSingleton
from mineru.utils.engine_utils import get_vlm_engine
from mineru.utils.model_utils import clean_memory
from mineru.utils.pdf_document import PDFDocument
from ..contracts import AnalysisResult, AnalyzeEffort, ParseMode, ResolvedParseMode
from ....model.runtime.hybrid import HybridLocalModelContextSingleton
from ....model.runtime.memory import clean_memory
from ....model.vlm.selector import get_vlm_engine
from ....model.flash.pdf.document import PDFDocument
from .layout import _load_vlm_runtime
from .normalization import _normalize_pdf_model_list
from .window import process_pdf_windows
os.environ["PYTORCH_ENABLE_MPS_FALLBACK"] = "1"
_SUPPORTED_PDF_EFFORTS = {"flash", "medium", "high", "xhigh"}
@@ -29,6 +26,10 @@ def analyze_pdf(
image_analysis: bool = True,
) -> AnalysisResult:
"""生产 PDF model-list,并返回最终路由元数据和精确推理耗时。"""
# 只在真实 PDF 分析开始时配置 MPS 回退,避免 import backend 修改进程环境。
import os
os.environ.setdefault("PYTORCH_ENABLE_MPS_FALLBACK", "1")
if effort not in _SUPPORTED_PDF_EFFORTS:
raise ValueError(f"Unsupported analyze effort: {effort}")
+7 -7
View File
@@ -13,22 +13,22 @@ import cv2
import numpy as np
from loguru import logger
from ...local_model_runtime import HybridLocalModelContext, run_ocr_inference
from ....model.model_types import AtomicModelName
from ....model.runtime.hybrid import HybridLocalModelContext, run_ocr_inference
from ....model.runtime.contracts import AtomicModelName
from ....types import RAW_ALGORITHM, RAW_FORMULA_NUMBER, RAW_PHONETIC, BBox, BlockType
from ....utils.bbox_utils import (
from ....utils.geometry import (
calculate_overlap_area_in_bbox1_area_ratio,
normalize_to_int_bbox,
)
from ....utils.native_pdf_table import (
from ....model.flash.pdf.table_recovery import (
NativeTableInput,
coerce_native_table_rectangles,
coerce_native_table_rules,
recover_native_pdf_table,
)
from ....utils.ocr_utils import mask_formula_regions_for_ocr_det
from ....utils.pdf_document import PDFPage, get_lines_from_chars
from ....utils.spatial_text import project_ocr_table_text
from ....model.ocr.image import mask_formula_regions_for_ocr_det
from ....model.flash.pdf.document import PDFPage, get_lines_from_chars
from ....model.flash.pdf.spatial_text import project_ocr_table_text
from .constants import (
BATCH_RATIO,
@@ -1,2 +1,4 @@
# Copyright (c) Opendatalab. All rights reserved.
"""PDF 原生文本、OCR Span 和内容重建。"""
__all__: list[str] = []
+8 -7
View File
@@ -9,18 +9,19 @@ import numpy as np
from PIL import Image
from pdftext.schema import Char
from mineru.backend.local_model_runtime import HybridLocalModelContext, run_ocr_inference
from mineru.types import BBox, BlockType, ContentType
from mineru.utils.language import detect_lang
from mineru.utils.ocr_utils import OcrConfidence, rotate_vertical_crop_if_needed
from mineru.utils.pdf_document import PDFPage, get_lines_from_chars
from mineru.utils.pdf_text_styles import (
from .....model.runtime.hybrid import HybridLocalModelContext, run_ocr_inference
from .....types import BBox, BlockType, ContentType
from .....utils.language import detect_lang
from .....model.ocr.image import rotate_vertical_crop_if_needed
from .....model.ocr.results import OcrConfidence
from .....model.flash.pdf.document import PDFPage, get_lines_from_chars
from .....model.flash.pdf.text_styles import (
PDFTextLinkLine,
PDFTextStyleLine,
apply_pdf_text_links,
apply_pdf_text_styles,
)
from mineru.utils.text_utils import merge_text_line_contents
from .....utils.text import merge_text_line_contents
from ..constants import (
CODE_CONTENT_BLOCK_TYPES,
+1 -1
View File
@@ -4,7 +4,7 @@
from __future__ import annotations
from .....types import ContentType
from .....utils.ocr_utils import _is_overlaps_x_exceeds_threshold, _is_overlaps_y_exceeds_threshold
from .....model.ocr.geometry import _is_overlaps_x_exceeds_threshold, _is_overlaps_y_exceeds_threshold
from .models import _AnalyzeLine, _AnalyzeSpan
VERTICAL_SPAN_HEIGHT_TO_WIDTH_RATIO_THRESHOLD = 2
+4 -4
View File
@@ -17,10 +17,10 @@ from loguru import logger
from pdftext.schema import Char
from .....types import BBox, BlockType, ContentType
from .....utils.image_utils import calculate_contrast
from .....utils.pdf_document import PDFPage, get_lines_from_chars
from .....utils.pdf_image_tools import get_crop_img
from mineru.utils.bbox_utils import calculate_overlap_area_in_bbox1_area_ratio
from .....utils.image import calculate_contrast
from .....model.flash.pdf.document import PDFPage, get_lines_from_chars
from ..images import get_crop_img
from .....utils.geometry import calculate_overlap_area_in_bbox1_area_ratio
from .models import _AnalyzeSpan
MAX_NATIVE_TEXT_CHARS_PER_PAGE = 65535
+3 -3
View File
@@ -7,8 +7,8 @@ from typing import Any, Sequence
from pdftext.schema import Char
from mineru.utils.pdf_document import PDFPage, get_lines_from_chars
from mineru.utils.pdf_text_styles import (
from .....model.flash.pdf.document import PDFPage, get_lines_from_chars
from .....model.flash.pdf.text_styles import (
PDFTextLinkLine,
PDFTextStyleLine,
detect_pdf_text_link_lines,
@@ -30,7 +30,7 @@ def build_pdf_native_visual_lines_and_styles(
"""一次读取当前页字符,构造视觉 run、文本样式和超链接证据。"""
# 延迟导入避免 Hybrid 模块初始化时提前加载完整 Flash PDF 流水线。
from mineru.model.flash.native_pdf.native_text import _build_native_line_items
from .....model.flash.pdf.native_text import _build_native_line_items
chars = page_chars if page_chars is not None else pdf_page.get_chars()
line_items = _build_native_line_items(
+2 -2
View File
@@ -9,8 +9,8 @@ from typing import Any
import numpy as np
from loguru import logger
from mineru.types import BBox, BlockType
from mineru.utils.bbox_utils import (
from ....types import BBox, BlockType
from ....utils.geometry import (
calculate_overlap_area_2_minbox_area_ratio,
calculate_overlap_area_in_bbox1_area_ratio,
)
+20 -9
View File
@@ -9,10 +9,9 @@ from typing import Any, Literal
import numpy as np
from loguru import logger
from mineru.backend.local_model_runtime import HybridLocalModelContext
from mineru.utils.config_reader import get_processing_window_size
from mineru.utils.pdf_document import PDFDocument, PDFPage
from mineru.utils.pdf_image_tools import load_images_from_pdf_bytes_range
from ....model.runtime.hybrid import HybridLocalModelContext
from ....model.flash.pdf.document import PDFDocument, PDFPage
from .images import load_images_from_pdf_bytes_range
from ..contracts import AnalyzeEffort
from .constants import (
@@ -61,6 +60,20 @@ from .text.content import (
)
def _configured_window_size(default: int = 64) -> int:
"""读取 PDF 分析窗口大小,非法配置回退到正整数默认值。"""
import os
raw_value = os.getenv("MINERU_PROCESSING_WINDOW_SIZE")
if raw_value is None:
return default
try:
return max(1, int(raw_value))
except ValueError:
logger.warning(f"Invalid MINERU_PROCESSING_WINDOW_SIZE value: {raw_value}, use default {default}")
return default
@dataclass(frozen=True)
class _ProcessingWindow:
"""记录单个 Hybrid 处理窗口的页码范围。"""
@@ -278,11 +291,11 @@ def process_pdf_windows(
model_list: list[list[dict[str, Any]]] = []
if flash_txt_mode:
# Flash 先对整份 PDF 生成完整 model_list,不依赖页面渲染和处理窗口。
from mineru.model.flash import PdfModel
from ....model.flash import PdfModel
model_list = PdfModel().predict(document)
configured_window_size = get_processing_window_size(default=64)
configured_window_size = _configured_window_size(default=64)
windows = _build_processing_windows(page_count, configured_window_size)
_log_processing_window_plan(page_count, configured_window_size, len(windows))
@@ -362,9 +375,7 @@ def process_pdf_windows(
if effort == "medium":
window_model_list = vl_style_layout_blocks
elif effort == "high":
high_vlm_blocks, accepted_native_tables = _split_native_high_table_blocks(
vl_style_layout_blocks
)
high_vlm_blocks, accepted_native_tables = _split_native_high_table_blocks(vl_style_layout_blocks)
high_vlm_results = vlm_predictor.batch_extract_with_layout(
images=images_pil_list,
blocks_list=high_vlm_blocks,
+10 -7
View File
@@ -8,13 +8,10 @@ from typing import cast
from loguru import logger
from mineru.backend.analysis.contracts import AnalyzeEffort, OfficeSuffix, ParseMode
from mineru.backend.analysis.office import analyze_office
from mineru.backend.analysis.pdf.pipeline import analyze_pdf
from mineru.backend.postprocess.document import model_json_to_middle_json
from mineru.config import config
from mineru.types import FILE_SUFFIXES, FileSuffix, MiddleJson, ModelJson
from mineru.version import __version__ as mineru_version
from .analysis.contracts import AnalyzeEffort, OfficeSuffix, ParseMode
from ..config import config
from ..types import FILE_SUFFIXES, FileSuffix, MiddleJson, ModelJson
from ..version import __version__ as mineru_version
_SUPPORTED_ANALYZE_EFFORTS = {"flash", "medium", "high", "xhigh"}
@@ -43,6 +40,8 @@ def doc_analyze(
raise ValueError(f"Unsupported analyze effort: {effort}")
if file_suffix == "pdf":
from .analysis.pdf.pipeline import analyze_pdf
result = analyze_pdf(
file_bytes,
effort=effort,
@@ -50,6 +49,8 @@ def doc_analyze(
image_analysis=image_analysis,
)
else:
from .analysis.office import analyze_office
result = analyze_office(file_bytes, cast(OfficeSuffix, file_suffix))
_log_infer_performance(file_suffix, len(result.model_list), result.elapsed)
@@ -61,6 +62,8 @@ def doc_analyze(
parse_mode=result.parse_mode,
mineru_version=mineru_version,
)
from .postprocess.document import model_json_to_middle_json
middle_json = model_json_to_middle_json(
model_json,
llm_aided_config=config.llm_aided,
+2
View File
@@ -1,2 +1,4 @@
# Copyright (c) Opendatalab. All rights reserved.
"""将严格 ModelJson 转换并增强为 MiddleJson 的后处理层。"""
__all__: list[str] = []
+2 -2
View File
@@ -3,8 +3,8 @@
from __future__ import annotations
from mineru.config import LLMAidedConfig
from mineru.types import MiddleJson, ModelJson
from ...config import LLMAidedConfig
from ...types import MiddleJson, ModelJson
from .llm_aided import apply_llm_aided_postprocess
from .pages import model_json_to_pages
@@ -8,8 +8,8 @@ import re
from dataclasses import dataclass
from typing import TypeAlias, Union
from mineru.utils.language import detect_lang
from mineru.utils.text_utils import CJK_LANGS, resolve_text_line_boundary
from ...utils.language import detect_lang
from ...utils.text import CJK_LANGS, resolve_text_line_boundary
_INLINE_START_RE = re.compile(
r"<(?P<tag>eq|text|hyperlink|sup|sub|strong|b|em|i|s|u)(?P<attrs>\s[^<>]*?)?>",
+2 -2
View File
@@ -6,8 +6,8 @@ from __future__ import annotations
import re
from typing import Any
from mineru.types import BlockType
from mineru.utils.bbox_utils import calculate_overlap_area_in_bbox1_area_ratio
from ...types import BlockType
from ...utils.geometry import calculate_overlap_area_in_bbox1_area_ratio
from .visual import _bbox_for_calculation
+2 -2
View File
@@ -7,8 +7,8 @@ import asyncio
from loguru import logger
from mineru.config import LLMAidedConfig
from mineru.types import MiddleJson
from ...config import LLMAidedConfig
from ...types import MiddleJson
from .llm_client import LLMAidedClient
from .table_merge.llm_cell_merge import apply_llm_cross_page_cell_merge
+1 -1
View File
@@ -12,7 +12,7 @@ import json_repair
from loguru import logger
from openai import AsyncOpenAI
from mineru.config import LLMAidedConfig
from ...config import LLMAidedConfig
ValidatedResult = TypeVar("ValidatedResult")
+4 -4
View File
@@ -6,16 +6,16 @@ from __future__ import annotations
import re
from typing import Any
from mineru.types import (
from ...types import (
RAW_ALGORITHM,
RAW_CAPTION,
RAW_FOOTNOTE,
BlockType,
VISUAL_MAIN_TYPES,
)
from mineru.utils.guess_suffix_or_lang import guess_language_by_text
from ...utils.language import guess_code_language
from mineru.utils.text_utils import clean_isolated_formula
from ...utils.text import clean_isolated_formula
from .content import clean_content, code_content_clean
from .lists import (
@@ -167,7 +167,7 @@ def _annotate_code_languages(code_blocks: list[BlockDict]) -> None:
continue
for sub_block in code_block["content"]:
if sub_block.get("type") == BlockType.CODE_BODY:
code_block["guess_lang"] = guess_language_by_text(sub_block.get("content", ""))
code_block["guess_lang"] = guess_code_language(sub_block.get("content", ""))
break
+5 -5
View File
@@ -8,11 +8,11 @@ from typing import Any
from pydantic import TypeAdapter
from mineru.backend.postprocess.lists import fix_office_index_title_blocks, fix_office_paragraph_titles
from mineru.backend.postprocess.page_blocks import process_page_blocks
from mineru.backend.postprocess.paragraphs import merge_para_text_blocks
from mineru.backend.postprocess.table_merge import merge_table
from mineru.types import ModelJson, PageInfo
from .lists import fix_office_index_title_blocks, fix_office_paragraph_titles
from .page_blocks import process_page_blocks
from .paragraphs import merge_para_text_blocks
from .table_merge import merge_table
from ...types import ModelJson, PageInfo
PAGE_INFO_LIST_ADAPTER = TypeAdapter(list[PageInfo])
@@ -8,7 +8,7 @@ from typing import Any
from bs4 import BeautifulSoup
from mineru.types import BlockType
from ....types import BlockType
from .html import _build_front_cache, _scan_rows
from .models import (
@@ -7,7 +7,7 @@ from copy import deepcopy
from bs4 import Tag
from mineru.types import BlockType
from ....types import BlockType
from .blocks import (
_build_post_body_child_index,
@@ -5,7 +5,7 @@ from __future__ import annotations
from typing import Any
from mineru.types import BlockType
from ....types import BlockType
from .blocks import _get_or_create_table_state
from .models import BlockDict, PageInfoDict, TableMergeState
@@ -7,7 +7,7 @@ from typing import Any
from bs4 import BeautifulSoup, Tag
from mineru.utils.text_utils import full_to_half
from ....utils.text import full_to_half
from .models import (
MAX_HEADER_ROWS,
@@ -8,7 +8,7 @@ import json
from dataclasses import dataclass
from typing import Literal, cast
from mineru.types import PageInfo, TableBlock, TableBodyBlock
from ....types import PageInfo, TableBlock, TableBodyBlock
from .html import build_row_rendered_cell_segments, build_table_state_from_html
from .structure import _expand_header_count_by_rowspan, detect_table_headers
@@ -1,7 +1,7 @@
# Copyright (c) Opendatalab. All rights reserved.
"""跨页表格延续文本与 caption 的轻量判定规则。"""
from mineru.utils.text_utils import full_to_half
from ....utils.text import full_to_half
CONTINUATION_END_MARKERS = [
"(续)",
@@ -7,7 +7,7 @@ from typing import Any
from bs4 import Tag
from mineru.types import BlockType
from ....types import BlockType
from .blocks import (
_bbox_for_calculation,
+2 -2
View File
@@ -7,8 +7,8 @@ import asyncio
import json
from typing import Any
from mineru.types import DocTitleBlock, PageInfo, ParagraphTitleBlock
from mineru.render._internal.common.inline import inline_plain_text, parse_inline_content
from ...types import DocTitleBlock, PageInfo, ParagraphTitleBlock
from .inline import inline_plain_text, parse_inline_content
from .llm_client import LLMAidedClient
+2 -2
View File
@@ -13,8 +13,8 @@ from ...types import (
VISUAL_RELATION_IGNORED_TYPES,
VISUAL_TYPE_MAPPING,
)
from mineru.utils.bbox_utils import bbox_center_distance, bbox_distance
from mineru.types import RAW_CAPTION, RAW_FOOTNOTE
from ...utils.geometry import bbox_center_distance, bbox_distance
from ...types import RAW_CAPTION, RAW_FOOTNOTE
from .table_merge.rules import is_table_continuation_text
INLINE_CAPTION_FRAGMENT_TYPES = {BlockType.TEXT, RAW_FOOTNOTE}
+1 -1
View File
@@ -9,7 +9,7 @@ from dataclasses import dataclass
from pathlib import Path
from ...filetypes import OFFICE_EXTENSIONS
from ...utils.pdf_document import PDFDocument
from ...model.flash.pdf.document import PDFDocument
# Optional office doc support
try:
+4 -4
View File
@@ -39,9 +39,9 @@ from ..types import (
_iter_child_blocks,
select_highest_cached_tier,
)
from ..utils.model_registry import ModelRepo, model_repos_for_tier
from ..utils.models_download_utils import verify_model_repo
from ..utils.pdf_document import PDFDocument
from ..model.download import verify_model_repo
from ..model.registry import ModelRepo, model_repos_for_tier
from ..model.flash.pdf.document import PDFDocument
from ..version import __version__
from .background.parse_server_health import get_health, get_managed_parse_server_tier
from .base import AsyncDoclibInterface
@@ -2078,7 +2078,7 @@ def _page_markdown_blocks(
short_id: str = "",
tier: Tier = "standard",
) -> list[tuple[int, str]]:
from mineru.config import config
from ..config import config
result: list[tuple[int, str]] = []
image_renderer = _make_doclib_image_renderer(
+1 -1
View File
@@ -1394,7 +1394,7 @@ class ParseService:
if not pages:
continue
from mineru.config import config
from ...config import config
from ...render._internal.markdown.blocks import render_single_block
delimiters = config.render.latex_delimiters
+1 -1
View File
@@ -8,7 +8,7 @@ from typing import Literal, TypeAlias
from ..filetypes import PARSEABLE_EXTENSIONS
from ..parser.base import ParseResult
from ..render.writer import FileBasedDataWriter
from ..parser.writer import FileBasedDataWriter
from ..types import Tier
from ..utils.image_payload import validate_image_sidecar_path
+2 -2
View File
@@ -10,10 +10,10 @@ from typing import Any, Literal
from ..render import render_markdown, render_structured_content
from ..render.contracts import ImageRenderer, RenderMode
from ..render.writer import DataWriter
from .writer import DataWriter
from ..types import FileSuffix, MiddleJson, PageInfo
from ..utils.image_payload import ImagePayloadCache
from ..utils.pdf_document import PDFDocument
from ..model.flash.pdf.document import PDFDocument
MIDDLE_JSON_SCHEMA_VERSION: str = "2.0"
_LEGACY_SCHEMA_VERSION: str = "1.0"
+6 -6
View File
@@ -13,7 +13,7 @@ from ..backend.analyze import aio_doc_analyze, doc_analyze
from ..errors import InvalidRequestError
from ..filetypes import IMAGE_EXTENSIONS
from ..types import FILE_SUFFIXES, FileSuffix, MiddleJson, ModelJson, PageInfo, Tier
from ..utils.backend_options import effort_for_tier
from .tier import effort_for_tier
from .base import DocumentParser, ParseResult
logger = logging.getLogger(__name__)
@@ -111,8 +111,8 @@ class MinerUParser(DocumentParser):
)
def _prepare_input(self, path: Path, page_range: str = "") -> _PreparedInput:
from ..utils.guess_suffix_or_lang import guess_suffix_by_path
from ..utils.pdf_document import PDFDocument
from .file_type import guess_suffix_by_path
from ..model.flash.pdf.document import PDFDocument
file_name = path.stem
file_bytes = path.read_bytes()
@@ -163,8 +163,8 @@ class MinerUParser(DocumentParser):
if suffix != "pdf":
return file_bytes, None, None
from ..utils.pdf_document import PDFDocument
from ..utils.pdf_page_id import parse_page_range
from ..model.flash.pdf.document import PDFDocument
from .page_range import parse_page_range
doc = PDFDocument(file_bytes)
page_indices = parse_page_range(page_range, doc.page_count)
@@ -178,7 +178,7 @@ class MinerUParser(DocumentParser):
if page_indices == list(range(doc.page_count)):
return file_bytes, None, None
from ..utils.pdfium_guard import safe_rewrite_pdf_bytes_with_pdfium_result
from ..model.flash.pdf.pdfium import safe_rewrite_pdf_bytes_with_pdfium_result
rewrite_result = safe_rewrite_pdf_bytes_with_pdfium_result(file_bytes, page_indices=page_indices)
if rewrite_result.used_original:
+3
View File
@@ -1 +1,4 @@
# Copyright (c) Opendatalab. All rights reserved.
"""MinerU renderer 的非公共实现模块。"""
__all__: list[str] = []
@@ -1 +1,4 @@
# Copyright (c) Opendatalab. All rights reserved.
"""各输出格式共用的渲染语义与规划实现。"""
__all__: list[str] = []
+1 -1
View File
@@ -5,7 +5,7 @@ from __future__ import annotations
import re
from mineru.render._internal.common.inline import inline_plain_text, parse_inline_content
from ....backend.postprocess.inline import inline_plain_text, parse_inline_content
_INDEX_ROMAN_RE = re.compile(r"[ivxlcdm]+", re.IGNORECASE)
+2 -2
View File
@@ -7,8 +7,8 @@ from dataclasses import dataclass
import re
from typing import Literal, TypeAlias
from mineru.render._internal.common.inline import inline_plain_text, parse_inline_content
from mineru.types import BlockType, ListBlock
from ....backend.postprocess.inline import inline_plain_text, parse_inline_content
from ....types import BlockType, ListBlock
ListItemKind: TypeAlias = Literal["unordered", "ordered", "explicit", "none"]
OrderedListStyle: TypeAlias = Literal["decimal", "lower-alpha", "upper-alpha", "lower-roman", "upper-roman"]
+3 -3
View File
@@ -5,9 +5,9 @@ from __future__ import annotations
from dataclasses import dataclass, field
from mineru.backend.postprocess.table_merge import merge_table_content
from mineru.render.contracts import RenderMode
from mineru.types import (
from ....backend.postprocess.table_merge import merge_table_content
from ...contracts import RenderMode
from ....types import (
PAGE_AUXILIARY_BLOCK_TYPES,
BlockType,
ContinuableTextBlockBase,
+3
View File
@@ -1 +1,4 @@
# Copyright (c) Opendatalab. All rights reserved.
"""DOCX renderer 的内部实现。"""
__all__: list[str] = []
+3 -3
View File
@@ -11,9 +11,9 @@ from urllib.parse import urlsplit
from PIL import Image, UnidentifiedImageError
from mineru.render.contracts import AssetResolver
from mineru.types import ImagePayloadBlock
from mineru.utils.image_payload import parse_image_data_uri_strict, validate_image_sidecar_path
from ...contracts import AssetResolver
from ....types import ImagePayloadBlock
from ....utils.image_payload import parse_image_data_uri_strict, validate_image_sidecar_path
_PIL_FORMAT_EXTENSIONS = {
"BMP": "bmp",
+2 -2
View File
@@ -17,7 +17,7 @@ from docx.shared import Pt, RGBColor
from lxml import etree
from loguru import logger
from mineru.render._internal.common.inline import (
from ....backend.postprocess.inline import (
InlineEquation,
InlineLink,
InlineNode,
@@ -27,7 +27,7 @@ from mineru.render._internal.common.inline import (
join_inline_contents,
parse_inline_content,
)
from mineru.render._internal.docx.math import DocxFormulaError, latex_to_omml
from .math import DocxFormulaError, latex_to_omml
_BOOKMARK_SAFE_RE = re.compile(r"[^A-Za-z0-9_]+")
_BOOKMARK_MAX_LENGTH = 40
+852
View File
@@ -0,0 +1,852 @@
# Copyright (c) Opendatalab. All rights reserved.
"""严格 MiddleJson 到可编辑 DOCX bytes 的公共渲染实现。"""
from __future__ import annotations
from collections.abc import Iterable
from io import BytesIO
import re
from bs4 import BeautifulSoup, NavigableString, Tag
from docx import Document
from docx.document import Document as DocumentType
from docx.enum.table import WD_CELL_VERTICAL_ALIGNMENT
from docx.enum.text import WD_ALIGN_PARAGRAPH, WD_TAB_ALIGNMENT
from docx.oxml import OxmlElement
from docx.oxml.ns import qn
from docx.shared import Emu, Mm, Pt, Twips
from docx.table import _Cell
from docx.text.paragraph import Paragraph
from lxml import etree
from loguru import logger
from ..common.index import strip_index_page_tail
from ....backend.postprocess.inline import (
InlineEquation,
InlineLink,
InlineNode,
InlineStyled,
InlineText,
parse_inline_content,
)
from ..common.list_items import parse_list_item_marker
from ..common.planner import PlannedBlock, build_render_plan
from .assets import (
DocxAssetError,
PreparedImage,
prepare_block_image,
prepare_html_image,
)
from .inline import (
BookmarkRegistry,
InlineRenderContext,
append_inline_content,
append_inline_nodes,
append_internal_link,
append_joined_inline_contents,
sanitize_xml_text,
)
from .math import DocxFormulaError, latex_to_omml, split_formula_tag
from .styles import (
AUXILIARY_STYLE,
BODY_STYLE,
CAPTION_STYLE,
CODE_STYLE,
FOOTNOTE_STYLE,
FORMULA_FALLBACK_STYLE,
SPATIAL_TABLE_STYLE,
configure_document,
usable_width_emu,
usable_width_twips,
)
from .table import DocxTableError, NestedTableWriter, materialize_docx_tables
from ...contracts import AssetResolver, RenderMode
from ...docx import DocxRenderError
from ....types import (
PAGE_AUXILIARY_BLOCK_TYPES,
RAW_ALGORITHM,
BlockBase,
BlockType,
BBox,
ChartAnnotationBlock,
ChartBlock,
ChartBodyBlock,
CodeAnnotationBlock,
CodeBlock,
CodeBodyBlock,
DocTitleBlock,
EquationBlock,
ImageAnnotationBlock,
ImageBlock,
ImageBodyBlock,
ImagePayloadBlock,
IndexBlock,
ListBlock,
MiddleJson,
PageAuxTextBlock,
ParagraphTitleBlock,
RefTextBlock,
TableAnnotationBlock,
TableBlock,
TableBodyBlock,
TextBlock,
TitleBlockBase,
)
_HTML_TABLE_RE = re.compile(r"<table\b", re.IGNORECASE)
_ANNOTATION_CAPTION_TYPES = {
BlockType.IMAGE_CAPTION,
BlockType.TABLE_CAPTION,
BlockType.CHART_CAPTION,
BlockType.CODE_CAPTION,
}
_ANNOTATION_FOOTNOTE_TYPES = {
BlockType.IMAGE_FOOTNOTE,
BlockType.TABLE_FOOTNOTE,
BlockType.CHART_FOOTNOTE,
BlockType.CODE_FOOTNOTE,
}
class _DocxRenderer:
"""持有单次 DOCX 渲染所需的 document、素材解析器和书签状态。"""
def __init__(
self,
middle_json: MiddleJson,
*,
mode: RenderMode,
asset_resolver: AssetResolver | None,
) -> None:
"""初始化 renderer,并预注册全部标题 anchor。"""
self.middle_json = middle_json
self.mode = mode
self.asset_resolver = asset_resolver
self.document = Document()
configure_document(self.document)
self.bookmarks = BookmarkRegistry(_iter_document_anchors(middle_json))
self.usable_width_emu = usable_width_emu(self.document)
self.usable_width_twips = usable_width_twips(self.document)
def render(self) -> bytes:
"""执行逐页 visitor,并把 python-docx document 序列化为 bytes。"""
planned_pages = build_render_plan(self.middle_json, self.mode)
for page_position, planned_blocks in enumerate(planned_pages):
if self.mode is RenderMode.FULL and page_position > 0:
self.document.add_page_break()
for planned in planned_blocks:
if planned.removed:
continue
if self.mode is RenderMode.DEFAULT and planned.block.type in PAGE_AUXILIARY_BLOCK_TYPES:
continue
self._render_planned_block(planned)
output = BytesIO()
self.document.save(output)
return output.getvalue()
def _render_planned_block(self, planned: PlannedBlock) -> None:
"""按严格 PageBlock 具体类型分派 Word 写入逻辑。"""
block = planned.block
context = self._context(planned.page_idx, block)
if isinstance(block, (TextBlock, RefTextBlock)):
paragraph = self.document.add_paragraph(style=BODY_STYLE)
append_joined_inline_contents(
paragraph,
planned.text_contents or [block.content],
context=context,
)
return
if isinstance(block, (DocTitleBlock, ParagraphTitleBlock)):
self._render_title(block, context)
return
if isinstance(block, PageAuxTextBlock):
paragraph = self.document.add_paragraph(style=AUXILIARY_STYLE)
append_inline_content(paragraph, block.content, context=context)
return
if isinstance(block, EquationBlock):
self._render_equation(block, context)
return
if isinstance(block, ListBlock):
self._render_list(block, context, depth=0)
return
if isinstance(block, IndexBlock):
self._render_index(block, context, depth=0)
return
if isinstance(block, ImageBlock):
self._render_image_block(block, context)
return
if isinstance(block, TableBlock):
self._render_table_block(block, context)
return
if isinstance(block, ChartBlock):
self._render_chart_block(block, context)
return
if isinstance(block, CodeBlock):
self._render_code_block(block, context)
return
raise TypeError(f"Unsupported PageBlock type: {type(block).__name__}")
def _context(self, page_idx: int, block: BlockBase) -> InlineRenderContext:
"""为当前 block 构造行内渲染上下文。"""
return InlineRenderContext(
bookmarks=self.bookmarks,
page_idx=page_idx,
block_index=block.index,
block_type=block.type,
)
def _render_title(
self,
block: DocTitleBlock | ParagraphTitleBlock,
context: InlineRenderContext,
) -> None:
"""用 Heading 1–6 写严格标题,并在段落内容外添加 bookmark。"""
level = min(max(block.level, 1), 9)
paragraph = self.document.add_paragraph(style=f"Heading {level}")
append_inline_content(paragraph, block.content, context=context)
self.bookmarks.attach(paragraph, block.anchor)
def _render_equation(self, block: EquationBlock, context: InlineRenderContext) -> None:
"""优先写可编辑 OMML,失败时按图片、可见 LaTeX 的顺序回退。"""
content = block.content.strip()
if content:
formula, tag = split_formula_tag(content)
try:
math_element = latex_to_omml(formula, display=True)
self._append_display_formula(math_element, tag, context=context)
return
except DocxFormulaError as exc:
logger.warning("DOCX display formula fallback: {} ({})", exc, context.location())
if block.image_base64 is not None or block.image_path is not None:
try:
self._append_block_image(block, context=context, alt_text="formula")
return
except DocxRenderError:
if not content:
raise
logger.warning("DOCX formula image fallback failed; preserving LaTeX ({})", context.location())
if content:
paragraph = self.document.add_paragraph(style=FORMULA_FALLBACK_STYLE)
paragraph.add_run(sanitize_xml_text(content, context=context))
return
raise self._render_error("Equation does not contain usable LaTeX or image", context)
def _append_display_formula(
self,
math_element: etree._Element,
tag: str | None,
*,
context: InlineRenderContext,
) -> None:
"""把块公式居中写入;带 tag 时使用中心与右对齐 tab。"""
if tag is None:
properties = OxmlElement("m:oMathParaPr")
justification = OxmlElement("m:jc")
justification.set(qn("m:val"), "center")
properties.append(justification)
math_element.insert(0, properties)
paragraph = self.document.add_paragraph(style=BODY_STYLE)
paragraph.paragraph_format.space_before = Pt(4)
paragraph.paragraph_format.space_after = Pt(5)
paragraph._p.append(math_element)
return
equation = math_element.find(qn("m:oMath"))
if equation is None:
raise ValueError("Display OMML does not contain m:oMath")
math_element.remove(equation)
paragraph = self.document.add_paragraph(style=BODY_STYLE)
paragraph.paragraph_format.space_before = Pt(4)
paragraph.paragraph_format.space_after = Pt(5)
paragraph.paragraph_format.tab_stops.add_tab_stop(
Twips(self.usable_width_twips // 2),
WD_TAB_ALIGNMENT.CENTER,
)
paragraph.paragraph_format.tab_stops.add_tab_stop(
Twips(self.usable_width_twips),
WD_TAB_ALIGNMENT.RIGHT,
)
paragraph.add_run().add_tab()
paragraph._p.append(equation)
paragraph.add_run().add_tab()
rendered_tag = tag if tag.startswith("(") and tag.endswith(")") else f"({tag})"
paragraph.add_run(sanitize_xml_text(rendered_tag, context=context))
def _render_list(
self,
block: ListBlock,
context: InlineRenderContext,
*,
depth: int,
) -> None:
"""保留叶子原 marker,并仅用缩进表达递归列表层级。"""
for child in block.content:
if isinstance(child, ListBlock):
self._render_list(child, context, depth=depth + 1)
continue
paragraph = self.document.add_paragraph(style=BODY_STYLE)
item = parse_list_item_marker(child.content)
if item.marker is None:
paragraph.paragraph_format.left_indent = Mm(depth * 6)
else:
paragraph.paragraph_format.left_indent = Mm((depth + 1) * 6)
paragraph.paragraph_format.first_line_indent = Mm(-6)
append_inline_content(paragraph, child.content, context=context)
def _render_index(
self,
block: IndexBlock,
context: InlineRenderContext,
*,
depth: int,
) -> None:
"""递归输出目录项,并把标题叶子链接到预注册 bookmark。"""
for child in block.content:
if isinstance(child, IndexBlock):
self._render_index(child, context, depth=depth + 1)
continue
content = strip_index_page_tail(child.content)
nodes = parse_inline_content(content)
if not nodes:
continue
paragraph = self.document.add_paragraph(style=BODY_STYLE)
paragraph.paragraph_format.left_indent = Mm((depth + 1) * 6)
paragraph.paragraph_format.first_line_indent = Mm(-4)
paragraph.add_run("• ")
anchor = child.anchor if isinstance(child, TitleBlockBase) else None
append_internal_link(paragraph, nodes, anchor=anchor, context=context)
def _render_image_block(self, block: ImageBlock, context: InlineRenderContext) -> None:
"""按原始子块顺序写图片主体、caption 与 footnote。"""
for child in block.content:
if isinstance(child, ImageBodyBlock):
alt_text = _plain_html_text(child.content) or block.sub_type or "image"
self._append_block_image(child, context=context, alt_text=alt_text)
elif isinstance(child, ImageAnnotationBlock):
self._render_annotation(child, context)
else:
raise TypeError(f"Unsupported image child: {type(child).__name__}")
def _render_table_block(self, block: TableBlock, context: InlineRenderContext) -> None:
"""按原始子块顺序写原生 HTML table 或预格式空间投影文本。"""
for child in block.content:
if isinstance(child, TableBodyBlock):
if _HTML_TABLE_RE.search(child.content):
try:
self._append_html_tables(child.content, context=context, depth=0)
except DocxRenderError as exc:
logger.warning("DOCX HTML table fallback: {}", exc)
if child.image_base64 is None and child.image_path is None:
raise self._render_error(
"HTML table cannot be materialized and has no image fallback",
context,
) from exc
self._append_block_image(child, context=context, alt_text="table")
else:
if child.content and not child.content.isspace():
paragraph = self.document.add_paragraph(style=SPATIAL_TABLE_STYLE)
paragraph.add_run(sanitize_xml_text(child.content, context=context))
elif child.image_base64 is not None or child.image_path is not None:
self._append_block_image(child, context=context, alt_text="table")
else:
raise self._render_error(
"Spatial table does not contain text content or image",
context,
)
elif isinstance(child, TableAnnotationBlock):
self._render_annotation(child, context)
else:
raise TypeError(f"Unsupported table child: {type(child).__name__}")
def _render_chart_block(self, block: ChartBlock, context: InlineRenderContext) -> None:
"""先写图表图片,再把 HTML 结构化数据追加为原生表格。"""
for child in block.content:
if isinstance(child, ChartBodyBlock):
has_image = child.image_base64 is not None or child.image_path is not None
if has_image:
self._append_block_image(
child,
context=context,
alt_text=_plain_html_text(child.content) or block.sub_type or "chart",
)
if _HTML_TABLE_RE.search(child.content):
try:
self._append_html_tables(child.content, context=context, depth=0)
except DocxRenderError as exc:
if not has_image:
raise
logger.warning("DOCX chart data table omitted after image fallback: {}", exc)
elif child.content.strip() and not has_image:
paragraph = self.document.add_paragraph(style=BODY_STYLE)
append_inline_content(paragraph, child.content, context=context)
elif not has_image and not child.content.strip():
raise self._render_error("Chart does not contain image or structured content", context)
elif isinstance(child, ChartAnnotationBlock):
self._render_annotation(child, context)
else:
raise TypeError(f"Unsupported chart child: {type(child).__name__}")
def _render_code_block(self, block: CodeBlock, context: InlineRenderContext) -> None:
"""按原始子块顺序写代码或算法主体及说明。"""
for child in block.content:
if isinstance(child, CodeBodyBlock):
paragraph = self.document.add_paragraph(style=CODE_STYLE)
if block.sub_type == BlockType.CODE:
paragraph.add_run(sanitize_xml_text(child.content, context=context))
elif block.sub_type == RAW_ALGORITHM:
append_inline_content(paragraph, child.content, context=context)
else:
raise ValueError(f"Unsupported code subtype: {block.sub_type}")
elif isinstance(child, CodeAnnotationBlock):
self._render_annotation(child, context)
else:
raise TypeError(f"Unsupported code child: {type(child).__name__}")
def _render_annotation(
self,
block: ImageAnnotationBlock | TableAnnotationBlock | ChartAnnotationBlock | CodeAnnotationBlock,
context: InlineRenderContext,
) -> None:
"""根据说明类型选择 Caption 或 Footnote 样式并写入行内内容。"""
if block.type in _ANNOTATION_CAPTION_TYPES:
style = CAPTION_STYLE
elif block.type in _ANNOTATION_FOOTNOTE_TYPES:
style = FOOTNOTE_STYLE
else:
raise TypeError(f"Unsupported annotation type: {block.type}")
paragraph = self.document.add_paragraph(style=style)
append_inline_content(paragraph, block.content, context=context)
def _append_block_image(
self,
block: ImagePayloadBlock,
*,
context: InlineRenderContext,
alt_text: str,
) -> None:
"""安全加载 block 图片,并按 bbox/自然尺寸限制到可用页面范围。"""
try:
prepared = prepare_block_image(block, self.asset_resolver)
except DocxAssetError as exc:
raise self._render_error(str(exc), context) from exc
self._append_prepared_image(
prepared,
bbox=block.bbox,
alt_text=alt_text,
max_width_emu=self.usable_width_emu,
context=context,
)
def _append_prepared_image(
self,
prepared: PreparedImage,
*,
bbox: BBox | None,
alt_text: str,
max_width_emu: int,
context: InlineRenderContext,
paragraph: Paragraph | None = None,
) -> None:
"""把准备好的图片按比例插入居中段落并写入 alt description。"""
target_paragraph = paragraph or self.document.add_paragraph(style=BODY_STYLE)
target_paragraph.alignment = WD_ALIGN_PARAGRAPH.CENTER
natural_width = round(prepared.width_px / 96 * 914400)
desired_width = natural_width
if bbox is not None:
desired_width = round(max_width_emu * max(0.0, min(1.0, bbox[2] - bbox[0])))
minimum_width = min(int(Mm(5)), max_width_emu)
desired_width = max(minimum_width, min(desired_width, max_width_emu))
usable_height = int(Mm(257))
rendered_height = round(desired_width * prepared.height_px / max(prepared.width_px, 1))
if rendered_height > usable_height:
desired_width = round(usable_height * prepared.width_px / max(prepared.height_px, 1))
shape = target_paragraph.add_run().add_picture(
BytesIO(prepared.data),
width=Emu(max(1, desired_width)),
)
description = sanitize_xml_text(
re.sub(r"\s+", " ", alt_text).strip()[:2048],
context=context,
)
if description:
shape._inline.docPr.set("descr", description)
def _append_html_tables(
self,
markup: str,
*,
context: InlineRenderContext,
depth: int,
container: DocumentType | _Cell | None = None,
) -> None:
"""物化 HTML 表格,并通过共享行内/素材逻辑填充 origin cell。"""
if depth >= 4:
raise self._render_error("Nested HTML table depth exceeds 4", context)
target_container = container or self.document
target_xml = target_container._element.body if isinstance(target_container, DocumentType) else target_container._tc
existing_child_count = len(target_xml)
insertion_index = existing_child_count - 1 if isinstance(target_container, DocumentType) else existing_child_count
existing_relationship_ids = set(self.document.part.rels)
def fill_cell(cell: _Cell, source: Tag, write_nested: NestedTableWriter) -> None:
"""填充一个 origin cell,并在遇到 nested table 时调用递归 writer。"""
self._fill_html_cell(
cell,
source,
write_nested=write_nested,
context=context,
)
try:
materialize_docx_tables(
target_container,
markup,
width_twips=self.usable_width_twips,
fill_cell=fill_cell,
)
except DocxRenderError:
self._rollback_table_materialization(
target_xml,
existing_child_count=existing_child_count,
insertion_index=insertion_index,
existing_relationship_ids=existing_relationship_ids,
)
raise
except DocxTableError as exc:
self._rollback_table_materialization(
target_xml,
existing_child_count=existing_child_count,
insertion_index=insertion_index,
existing_relationship_ids=existing_relationship_ids,
)
raise self._render_error(str(exc), context) from exc
def _rollback_table_materialization(
self,
target_xml: etree._Element,
*,
existing_child_count: int,
insertion_index: int,
existing_relationship_ids: set[str],
) -> None:
"""回滚本轮表格新增的 XML 根节点和 relationship,避免残留半成品。"""
while len(target_xml) > existing_child_count:
target_xml.remove(target_xml[insertion_index])
for relationship_id in tuple(self.document.part.rels):
if relationship_id not in existing_relationship_ids:
self.document.part.drop_rel(relationship_id)
def _fill_html_cell(
self,
cell: _Cell,
source: Tag,
*,
write_nested: NestedTableWriter,
context: InlineRenderContext,
) -> None:
"""按源顺序填充单元格文本、图片和任意包装层中的嵌套表格。"""
cell.text = ""
cell.vertical_alignment = WD_CELL_VERTICAL_ALIGNMENT.CENTER
_set_cell_margins(cell, top=100, start=120, bottom=100, end=120)
max_image_width_emu = _cell_content_width_emu(cell, horizontal_margin_twips=240)
paragraph = cell.paragraphs[0]
paragraph.paragraph_format.space_after = Pt(0)
paragraph.paragraph_format.line_spacing = 1.0
for child in source.children:
if not isinstance(child, (NavigableString, Tag)):
continue
if isinstance(child, Tag) and child.name.lower() in {"p", "div"} and _paragraph_has_content(paragraph):
paragraph = cell.add_paragraph()
paragraph.paragraph_format.space_after = Pt(0)
paragraph.paragraph_format.line_spacing = 1.0
styles = ("bold",) if source.name == "th" else ()
paragraph, _ = self._append_html_node(
paragraph,
child,
cell=cell,
write_nested=write_nested,
context=context,
inherited_styles=styles,
max_image_width_emu=max_image_width_emu,
)
def _append_html_node(
self,
paragraph: Paragraph,
node: NavigableString | Tag,
*,
cell: _Cell,
write_nested: NestedTableWriter,
context: InlineRenderContext,
inherited_styles: tuple[str, ...],
max_image_width_emu: int,
) -> tuple[Paragraph, bool]:
"""按 HTML 源顺序写入单元格行内文本、样式、公式和图片。"""
if isinstance(node, NavigableString):
text = str(node)
if not text:
return paragraph, False
nodes: list[InlineNode] = [InlineText(text)]
if inherited_styles:
nodes = [InlineStyled(nodes, inherited_styles)]
append_inline_nodes(paragraph, nodes, context=context)
return paragraph, bool(text.strip())
name = node.name.lower()
if name == "table":
write_nested(node)
trailing_paragraph = cell.paragraphs[-1]
trailing_paragraph.paragraph_format.space_after = Pt(0)
trailing_paragraph.paragraph_format.line_spacing = 1.0
return trailing_paragraph, True
if name == "img":
self._append_html_image(
paragraph,
str(node.get("src", "")),
context=context,
alt_text=str(node.get("alt", "")),
max_width_emu=max_image_width_emu,
)
return paragraph, True
if name == "br":
paragraph.add_run().add_break()
return paragraph, True
if name == "eq":
latex = node.get_text().strip()
nodes = [InlineEquation(latex)] if latex else []
if inherited_styles and nodes:
nodes = [InlineStyled(nodes, inherited_styles)]
append_inline_nodes(paragraph, nodes, context=context)
return paragraph, bool(nodes)
if name == "a":
children = _html_inline_nodes(node)
if inherited_styles and children:
children = [InlineStyled(children, inherited_styles)]
append_inline_nodes(
paragraph,
[InlineLink(children, str(node.get("href", "")).strip())],
context=context,
)
return paragraph, bool(children)
if name in {"ul", "ol"}:
start_value = node.get("start", "1")
try:
ordered_number = int(str(start_value))
except ValueError:
ordered_number = 1
rendered = False
for item in node.find_all("li", recursive=False):
if _paragraph_has_content(paragraph) or rendered:
paragraph.add_run().add_break()
marker = f"{ordered_number}. " if name == "ol" else "- "
paragraph.add_run(marker)
if name == "ol":
ordered_number += 1
for child in item.children:
if isinstance(child, (NavigableString, Tag)):
paragraph, child_rendered = self._append_html_node(
paragraph,
child,
cell=cell,
write_nested=write_nested,
context=context,
inherited_styles=inherited_styles,
max_image_width_emu=max_image_width_emu,
)
rendered = child_rendered or rendered
return paragraph, rendered
if name == "li":
rendered = False
for child in node.children:
if isinstance(child, (NavigableString, Tag)):
paragraph, child_rendered = self._append_html_node(
paragraph,
child,
cell=cell,
write_nested=write_nested,
context=context,
inherited_styles=inherited_styles,
max_image_width_emu=max_image_width_emu,
)
rendered = child_rendered or rendered
return paragraph, rendered
if name == "text":
nodes = parse_inline_content(str(node))
if inherited_styles and nodes:
nodes = [InlineStyled(nodes, inherited_styles)]
append_inline_nodes(paragraph, nodes, context=context)
return paragraph, bool(nodes)
style = {
"strong": "bold",
"b": "bold",
"em": "italic",
"i": "italic",
"u": "underline",
"s": "strikethrough",
"sup": "superscript",
"sub": "subscript",
}.get(name)
styles = tuple(dict.fromkeys((*inherited_styles, style))) if style else inherited_styles
rendered = False
for child in node.children:
if isinstance(child, (NavigableString, Tag)):
paragraph, child_rendered = self._append_html_node(
paragraph,
child,
cell=cell,
write_nested=write_nested,
context=context,
inherited_styles=styles,
max_image_width_emu=max_image_width_emu,
)
rendered = child_rendered or rendered
return paragraph, rendered
def _append_html_image(
self,
paragraph: Paragraph,
source: str,
*,
context: InlineRenderContext,
alt_text: str,
max_width_emu: int,
) -> None:
"""安全加载表格单元格 img,并限制到紧凑的单元格宽度。"""
try:
prepared = prepare_html_image(source, self.asset_resolver)
except DocxAssetError as exc:
raise self._render_error(str(exc), context) from exc
self._append_prepared_image(
prepared,
bbox=None,
alt_text=alt_text or "table image",
max_width_emu=min(self.usable_width_emu, max_width_emu),
context=context,
paragraph=paragraph,
)
def _render_error(self, message: str, context: InlineRenderContext) -> DocxRenderError:
"""用当前 context 构造带 page/block 定位的公共异常。"""
return DocxRenderError(
message,
page_idx=context.page_idx,
block_index=context.block_index,
block_type=context.block_type,
)
def render_docx(
middle_json: MiddleJson,
*,
mode: RenderMode = RenderMode.DEFAULT,
asset_resolver: AssetResolver | None = None,
) -> bytes:
"""把严格 MiddleJson 无副作用地渲染为完整 DOCX bytes。"""
if not isinstance(middle_json, MiddleJson):
raise TypeError("render_docx expects a MiddleJson instance")
if not isinstance(mode, RenderMode):
raise TypeError("mode must be a RenderMode value")
return _DocxRenderer(
middle_json,
mode=mode,
asset_resolver=asset_resolver,
).render()
def _iter_document_anchors(middle_json: MiddleJson) -> Iterable[str]:
"""按文档顺序遍历实际会写入 bookmark 的正文标题 anchor。"""
for page in middle_json.pages:
for block in page.blocks:
if isinstance(block, TitleBlockBase) and block.anchor:
yield block.anchor
def _plain_html_text(content: str) -> str:
"""把图片或图表 body 内容收敛为适合 alt description 的纯文本。"""
if not content.strip():
return ""
return BeautifulSoup(content, "html.parser").get_text(" ", strip=True)
def _html_inline_nodes(node: Tag) -> list[InlineNode]:
"""把表格单元格中的安全 HTML 行内标签转换为共享 InlineNode。"""
nodes: list[InlineNode] = []
for child in node.children:
if isinstance(child, NavigableString):
if str(child):
nodes.append(InlineText(str(child)))
continue
if not isinstance(child, Tag):
continue
name = child.name.lower()
children = _html_inline_nodes(child)
if name == "eq":
latex = child.get_text().strip()
if latex:
nodes.append(InlineEquation(latex))
elif name in {"strong", "b"}:
nodes.append(InlineStyled(children, ("bold",)))
elif name in {"em", "i"}:
nodes.append(InlineStyled(children, ("italic",)))
elif name == "u":
nodes.append(InlineStyled(children, ("underline",)))
elif name == "s":
nodes.append(InlineStyled(children, ("strikethrough",)))
elif name == "sup":
nodes.append(InlineStyled(children, ("superscript",)))
elif name == "sub":
nodes.append(InlineStyled(children, ("subscript",)))
elif name == "a":
nodes.append(InlineLink(children, str(child.get("href", "")).strip()))
elif name == "br":
nodes.append(InlineText("\n"))
elif name == "text":
nodes.extend(parse_inline_content(str(child)))
elif name not in {"img", "table"}:
nodes.extend(children)
return nodes
def _set_cell_margins(
cell: _Cell,
*,
top: int,
start: int,
bottom: int,
end: int,
) -> None:
"""用 DXA 为 Word 单元格写入四边内边距。"""
cell_properties = cell._tc.get_or_add_tcPr()
margins = cell_properties.find(qn("w:tcMar"))
if margins is None:
margins = OxmlElement("w:tcMar")
cell_properties.append(margins)
for side, value in (("top", top), ("start", start), ("bottom", bottom), ("end", end)):
element = margins.find(qn(f"w:{side}"))
if element is None:
element = OxmlElement(f"w:{side}")
margins.append(element)
element.set(qn("w:w"), str(value))
element.set(qn("w:type"), "dxa")
def _cell_content_width_emu(cell: _Cell, *, horizontal_margin_twips: int) -> int:
"""从当前物理或合并单元格 tcW 计算扣除左右内边距后的可用宽度。"""
cell_width = cell._tc.get_or_add_tcPr().get_or_add_tcW().w
width_twips = int(cell_width) if cell_width is not None else 1
return int(Twips(max(1, width_twips - horizontal_margin_twips)))
def _paragraph_has_content(paragraph: Paragraph) -> bool:
"""判断段落是否包含 pPr 之外的 run、公式、链接或 drawing。"""
return any(child.tag != qn("w:pPr") for child in paragraph._p)
__all__ = ["render_docx"]
+3
View File
@@ -1 +1,4 @@
# Copyright (c) Opendatalab. All rights reserved.
"""HTML renderer 的内部实现。"""
__all__: list[str] = []
+5 -11
View File
@@ -9,7 +9,7 @@ import re
from dataclasses import dataclass
from urllib.parse import urlsplit
from mineru.render._internal.common.inline import (
from ....backend.postprocess.inline import (
InlineEquation,
InlineLink,
InlineNode,
@@ -19,25 +19,19 @@ from mineru.render._internal.common.inline import (
join_inline_contents,
parse_inline_content,
)
from mineru.render._internal.html.sanitizer import sanitize_link_url
from .sanitizer import sanitize_link_url
_AUTOLINK_TLD_PATTERN = r"(?:[A-Za-z]{2,63}|xn--[A-Za-z0-9-]{2,59})"
_AUTOLINK_DOMAIN_PATTERN = (
rf"(?:[A-Za-z0-9](?:[A-Za-z0-9-]{{0,61}}[A-Za-z0-9])?\.){{1,10}}{_AUTOLINK_TLD_PATTERN}"
)
_AUTOLINK_DOMAIN_PATTERN = rf"(?:[A-Za-z0-9](?:[A-Za-z0-9-]{{0,61}}[A-Za-z0-9])?\.){{1,10}}{_AUTOLINK_TLD_PATTERN}"
_AUTOLINK_EMAIL_EDGE_PATTERN = r"[A-Za-z0-9!#$%&'*+/=?^_{|}~-]"
_AUTOLINK_EMAIL_PATTERN = (
rf"{_AUTOLINK_EMAIL_EDGE_PATTERN}"
rf"(?:[A-Za-z0-9.!#$%&'*+/=?^_{{|}}~-]{{0,62}}{_AUTOLINK_EMAIL_EDGE_PATTERN})?"
rf"@{_AUTOLINK_DOMAIN_PATTERN}"
)
_AUTOLINK_EXPLICIT_HOST_PATTERN = (
rf"(?:{_AUTOLINK_DOMAIN_PATTERN}|localhost|(?:\d{{1,3}}\.){{3}}\d{{1,3}}|\[[0-9A-Fa-f:.]+\])"
)
_AUTOLINK_URL_TAIL_PATTERN = (
r"(?::\d{1,5})?(?:[/?#](?:(?!https?://)[A-Za-z0-9._~:/?#\[\]@!$&()*+,;=%-])*)?"
)
_AUTOLINK_EXPLICIT_HOST_PATTERN = rf"(?:{_AUTOLINK_DOMAIN_PATTERN}|localhost|(?:\d{{1,3}}\.){{3}}\d{{1,3}}|\[[0-9A-Fa-f:.]+\])"
_AUTOLINK_URL_TAIL_PATTERN = r"(?::\d{1,5})?(?:[/?#](?:(?!https?://)[A-Za-z0-9._~:/?#\[\]@!$&()*+,;=%-])*)?"
_AUTOLINK_RE = re.compile(
rf"(?:(?P<email>(?<![A-Za-z0-9._%+@-]){_AUTOLINK_EMAIL_PATTERN})"
rf"|(?P<explicit>https?://{_AUTOLINK_EXPLICIT_HOST_PATTERN}{_AUTOLINK_URL_TAIL_PATTERN})"
+941
View File
@@ -0,0 +1,941 @@
# Copyright (c) Opendatalab. All rights reserved.
"""严格 MiddleJson 到 MinerU 风格 HTML 的公共渲染实现。"""
from __future__ import annotations
import html
import re
from functools import lru_cache
from importlib import resources
from urllib.parse import quote
from bs4 import BeautifulSoup
from loguru import logger
from ..common.index import strip_index_page_tail
from ....backend.postprocess.inline import inline_plain_text, parse_inline_content
from ..common.list_items import ListItem, parse_list_item_marker, reference_list_needs_bullets
from ..common.planner import PlannedBlock, build_render_plan
from .inline import (
HtmlInlineResult,
render_inline_content_html,
render_inline_nodes_html,
render_joined_inline_contents_html,
render_math_html,
)
from .sanitizer import (
is_supported_html_markup,
sanitize_html_fragment,
sanitize_image_source,
)
from .table import looks_like_gfm_table, render_gfm_table_html
from ...contracts import RenderMode
from ....types import (
PAGE_AUXILIARY_BLOCK_TYPES,
RAW_ALGORITHM,
BlockType,
ChartAnnotationBlock,
ChartBlock,
ChartBodyBlock,
CodeAnnotationBlock,
CodeBlock,
CodeBodyBlock,
DocTitleBlock,
EquationBlock,
ImageAnnotationBlock,
ImageBlock,
ImageBodyBlock,
IndexBlock,
ListBlock,
MiddleJson,
PageAuxTextBlock,
ParagraphTitleBlock,
RefTextBlock,
TableAnnotationBlock,
TableBlock,
TableBodyBlock,
TextBlock,
TitleBlockBase,
)
_STYLE_RESOURCE_NAME = "mineru.min.css"
_MATHJAX_URL = "https://cdn.jsdelivr.net/npm/mathjax@4.1.2/tex-chtml.js"
_MATHJAX_INTEGRITY = "sha384-zAhQQhdaMeHsMProNntGGg6nOUVcfuF9F22C3d1qJ9NZAVzCplXk1X85D2O5iufn"
_PRISM_CORE_URL = "https://cdn.jsdelivr.net/npm/prismjs@1.30.0/components/prism-core.min.js"
_PRISM_CORE_INTEGRITY = "sha384-zLRFO4dwowZvh8kzutOb5AWhH7f39HeJp+N7PtHF1SQtTBnifRx0AtmvTYs3F4YV"
_PRISM_AUTOLOADER_URL = "https://cdn.jsdelivr.net/npm/prismjs@1.30.0/plugins/autoloader/prism-autoloader.min.js"
_PRISM_AUTOLOADER_INTEGRITY = "sha384-Uq05+JLko69eOiPr39ta9bh7kld5PKZoU+fF7g0EXTAriEollhZ+DrN8Q/Oi8J2Q"
_PRISM_LANGUAGES_PATH = "https://cdn.jsdelivr.net/npm/prismjs@1.30.0/components/"
_MERMAID_URL = "https://cdn.jsdelivr.net/npm/mermaid@11.16.1/dist/mermaid.min.js"
_MERMAID_INTEGRITY = "sha384-aBQXj4hK6Jm05i7aQAsUV3bLdSUrHX1BGYfMB0166TtWt/RRaw+h0Eelme9OCOvy"
_MERMAID_MAX_TEXT_SIZE = 50_000
_MERMAID_FLOWCHART_HEADER_RE = re.compile(
r"(?:graph|flowchart)\s+(?:TB|TD|BT|RL|LR)\b",
re.IGNORECASE,
)
_SAFE_PRISM_LANGUAGE_RE = re.compile(r"[a-z0-9][a-z0-9-]{0,31}")
_BLOCKED_PRISM_LANGUAGE_NAMES = {"constructor", "prototype"}
_PRISM_LANGUAGE_ALIASES = {
"c#": "csharp",
"c++": "cpp",
"html": "markup",
"js": "javascript",
"py": "python",
"sh": "bash",
"shell": "bash",
"none": "",
"text": "",
"txt": "",
"ts": "typescript",
"xml": "markup",
"yml": "yaml",
}
class _HtmlRenderer:
"""维护一次 HTML 渲染所需的 anchor、公式与代码资源状态。"""
def __init__(
self,
middle_json: MiddleJson,
*,
mode: RenderMode,
asset_base_url: str,
standalone: bool,
document_title: str | None,
) -> None:
"""保存严格输入和调用选项,并预收集真实正文标题 anchor。"""
self.middle_json = middle_json
self.mode = mode
self.asset_base_url = asset_base_url
self.standalone = standalone
self.document_title = document_title
self.anchor_targets = _collect_title_anchor_ids(middle_json)
self.emitted_anchors: set[str] = set()
self.has_math = False
self.has_prism = False
self.has_mermaid = False
def render(self) -> str:
"""生成 fragment article,并按需包装完整 HTML 文档。"""
planned_pages = build_render_plan(self.middle_json, self.mode)
if self.mode is RenderMode.FULL:
body = self._render_full_pages(planned_pages)
else:
body = self._render_default_pages(planned_pages)
article = f'<article class="mineru-document mineru-document--{self.mode.value}">\n{body}\n</article>'
if not self.standalone:
return article
# 最终依赖以真实 DOM 为准,避免回退分支或普通文本中的 class 字样误触发。
article_soup = BeautifulSoup(article, "html.parser")
self.has_math = article_soup.select_one(".mineru-math") is not None
self.has_prism = any(
any(str(class_name).startswith("language-") for class_name in code.get("class", []))
for code in article_soup.find_all("code")
if code.get_text()
)
self.has_mermaid = article_soup.select_one(".mineru-flowchart") is not None
return self._render_standalone(article)
def _render_default_pages(self, planned_pages: list[list[PlannedBlock]]) -> str:
"""把 DEFAULT 计划展平为无页面边界的连续阅读内容。"""
rendered: list[str] = []
for page in planned_pages:
for planned in page:
block = self._render_planned_block(planned)
if block:
rendered.append(block)
return "\n".join(rendered)
def _render_full_pages(self, planned_pages: list[list[PlannedBlock]]) -> str:
"""为 FULL 计划保留所有页面 section 和相邻页面分隔。"""
sections: list[str] = []
for page in planned_pages:
page_idx = page[0].page_idx if page else None
if page_idx is None:
page_position = len(sections)
page_idx = self.middle_json.pages[page_position].page_idx
blocks = [rendered for planned in page if (rendered := self._render_planned_block(planned))]
rendered_blocks = "\n".join(blocks)
sections.append(f'<section class="mineru-page" data-page-idx="{page_idx}">\n{rendered_blocks}\n</section>')
return '\n<hr class="mineru-page-break" aria-hidden="true">\n'.join(sections)
def _render_planned_block(self, planned: PlannedBlock) -> str:
"""过滤计划块、分派类型 visitor,并添加稳定来源元数据。"""
if planned.removed:
return ""
block = planned.block
if self.mode is RenderMode.DEFAULT and block.type in PAGE_AUXILIARY_BLOCK_TYPES:
return ""
content = self._render_block_content(planned)
if not content or not content.strip():
return ""
attrs = [
'class="mineru-block"',
f'data-page-idx="{planned.page_idx}"',
f'data-block-type="{html.escape(str(block.type), quote=True)}"',
]
if block.index is not None:
attrs.append(f'data-block-index="{block.index}"')
return f"<div {' '.join(attrs)}>\n{content}\n</div>"
def _render_block_content(self, planned: PlannedBlock) -> str:
"""按严格 PageBlock 具体类型返回对应语义 HTML。"""
block = planned.block
if isinstance(block, TextBlock):
rendered = render_joined_inline_contents_html(planned.text_contents or [block.content])
self._observe_inline(rendered)
return f'<p class="mineru-text">{rendered.html}</p>' if rendered.html else ""
if isinstance(block, RefTextBlock):
rendered = render_joined_inline_contents_html(planned.text_contents or [block.content])
self._observe_inline(rendered)
return f'<p class="mineru-ref-text">{rendered.html}</p>' if rendered.html else ""
if isinstance(block, (DocTitleBlock, ParagraphTitleBlock)):
return self._render_title(block)
if isinstance(block, PageAuxTextBlock):
rendered = render_inline_content_html(block.content)
self._observe_inline(rendered)
class_name = html.escape(str(block.type).replace("_", "-"), quote=True)
return f'<div class="mineru-page-aux mineru-page-aux--{class_name}">{rendered.html}</div>' if rendered.html else ""
if isinstance(block, EquationBlock):
return self._render_equation(block)
if isinstance(block, ListBlock):
return self._render_list(block)
if isinstance(block, IndexBlock):
return self._render_index(block)
if isinstance(block, ImageBlock):
return self._render_image_block(block)
if isinstance(block, TableBlock):
return self._render_table_block(block)
if isinstance(block, ChartBlock):
return self._render_chart_block(block)
if isinstance(block, CodeBlock):
return self._render_code_block(block)
raise TypeError(f"Unsupported PageBlock type: {type(block).__name__}")
def _render_title(self, block: DocTitleBlock | ParagraphTitleBlock) -> str:
"""渲染标题,并保证正文 anchor 只在首次出现时生成 id。"""
rendered = render_inline_content_html(block.content)
self._observe_inline(rendered)
if not rendered.html:
return ""
level = min(max(block.level, 1), 6)
attrs = [f'class="mineru-heading mineru-heading--{level}"', f'data-heading-level="{block.level}"']
anchor = _anchor_key(block.anchor)
if anchor:
if anchor not in self.emitted_anchors:
anchor_id = self.anchor_targets.get(anchor)
if anchor_id:
attrs.append(f'id="{html.escape(anchor_id, quote=True)}"')
self.emitted_anchors.add(anchor)
else:
logger.warning("Duplicate HTML title anchor omitted: {!r}", anchor)
return f"<h{level} {' '.join(attrs)}>{rendered.html}</h{level}>"
def _render_equation(self, block: EquationBlock) -> str:
"""优先输出行间公式,空 LaTeX 时回退到安全图片来源。"""
if block.content.strip():
rendered = render_math_html(block.content, display=True)
self._observe_inline(rendered)
return rendered.html
source = self._safe_block_image_source(block)
return self._render_image(source, alt="formula", class_name="mineru-equation-image") if source else ""
def _render_list(self, block: ListBlock) -> str:
"""按直属 marker 类型渲染一层列表,并递归挂接嵌套列表。"""
parsed_leaves = [
parse_list_item_marker(child.content)
for child in block.content
if not isinstance(child, ListBlock) and child.content.strip()
]
add_reference_bullets = reference_list_needs_bullets(block)
container_tag, list_type, class_name = _classify_list(parsed_leaves, add_reference_bullets)
items: list[dict[str, object]] = []
expected_value: int | None = None
for child in block.content:
if isinstance(child, ListBlock):
nested = self._render_list(child)
if not nested:
continue
if not items:
items.append({"content": "", "attrs": ' class="mineru-list-item--orphan"', "nested": []})
nested_items = items[-1]["nested"]
assert isinstance(nested_items, list)
nested_items.append(nested)
continue
parsed = parse_list_item_marker(child.content)
item_content, marker = _list_item_content(
parsed,
add_reference_bullets,
explicit_markers=class_name == "mineru-list--explicit",
)
rendered = render_inline_content_html(item_content)
self._observe_inline(rendered)
if not rendered.html and not marker:
items.append({"content": "", "attrs": ' class="mineru-list-item--markerless"', "nested": []})
continue
attrs: list[str] = []
if class_name == "mineru-list--explicit":
attrs.append('class="mineru-list-item--explicit"')
if container_tag == "ol" and parsed.kind == "ordered" and parsed.value is not None:
if expected_value is None:
expected_value = parsed.value
if parsed.value != expected_value:
attrs.append(f'value="{parsed.value}"')
expected_value = parsed.value + 1
marker_html = (
f'<span class="mineru-list-marker">{html.escape(_replace_html_controls(marker), quote=False)}</span>'
if marker
else ""
)
if class_name == "mineru-list--explicit" and not marker_html:
marker_html = '<span class="mineru-list-marker"></span>'
items.append(
{
"content": f'{marker_html}<span class="mineru-list-content">{rendered.html}</span>',
"attrs": f" {' '.join(attrs)}" if attrs else "",
"nested": [],
}
)
rendered_items: list[str] = []
for item in items:
nested = item["nested"]
assert isinstance(nested, list)
item_content = str(item["content"])
if not item_content and not nested:
continue
rendered_items.append(f"<li{item['attrs']}>{item_content}{''.join(str(value) for value in nested)}</li>")
if not rendered_items:
return ""
container_attrs = [f'class="mineru-list {class_name}"']
if list_type:
container_attrs.append(f'type="{list_type}"')
if container_tag == "ol" and parsed_leaves and parsed_leaves[0].value not in (None, 1):
container_attrs.append(f'start="{parsed_leaves[0].value}"')
return f"<{container_tag} {' '.join(container_attrs)}>{''.join(rendered_items)}</{container_tag}>"
def _render_index(self, block: IndexBlock) -> str:
"""递归渲染目录,并仅链接到真实正文标题 anchor。"""
items: list[dict[str, object]] = []
for child in block.content:
if isinstance(child, IndexBlock):
nested = self._render_index_list(child)
if not nested:
continue
if not items:
items.append({"content": "", "orphan": True, "nested": []})
nested_items = items[-1]["nested"]
assert isinstance(nested_items, list)
nested_items.append(nested)
continue
rendered = self._render_index_leaf(child)
items.append({"content": rendered, "orphan": False, "nested": []})
inner = _serialize_index_items(items)
return f'<nav class="mineru-index" aria-label="Table of contents"><ul>{inner}</ul></nav>' if inner else ""
def _render_index_list(self, block: IndexBlock) -> str:
"""渲染一个嵌套 IndexBlock 为可挂接到父项的 ul。"""
items: list[dict[str, object]] = []
for child in block.content:
if isinstance(child, IndexBlock):
nested = self._render_index_list(child)
if nested:
if not items:
items.append({"content": "", "orphan": True, "nested": []})
nested_items = items[-1]["nested"]
assert isinstance(nested_items, list)
nested_items.append(nested)
continue
rendered = self._render_index_leaf(child)
items.append({"content": rendered, "orphan": False, "nested": []})
inner = _serialize_index_items(items)
return f"<ul>{inner}</ul>" if inner else ""
def _render_index_leaf(self, block: TextBlock | TitleBlockBase) -> str:
"""渲染目录叶子,并在 anchor 命中正文标题时生成内部链接。"""
content = strip_index_page_tail(block.content)
rendered = render_inline_content_html(content)
self._observe_inline(rendered)
if not rendered.html:
return ""
anchor = _anchor_key(block.anchor) if isinstance(block, TitleBlockBase) else ""
if anchor and (anchor_id := self.anchor_targets.get(anchor)):
href = quote(anchor_id, safe="-._~")
return f'<a href="#{href}">{rendered.html}</a>'
return rendered.html
def _render_image_block(self, block: ImageBlock) -> str:
"""按原始子块顺序渲染图片主体及重复说明。"""
parts: list[str] = []
for child in block.content:
if isinstance(child, ImageBodyBlock):
body = self._render_image_body(block, child)
elif isinstance(child, ImageAnnotationBlock):
body = self._render_annotation(child)
else:
raise TypeError(f"Unsupported image child: {type(child).__name__}")
if body:
parts.append(body)
return f'<figure class="mineru-figure mineru-figure--image">{"".join(parts)}</figure>' if parts else ""
def _render_image_body(self, parent: ImageBlock, block: ImageBodyBlock) -> str:
"""渲染图片,并把并存的识别内容放入原生 details。"""
if parent.sub_type == "flowchart":
mermaid_source = _extract_mermaid_flowchart_source(block.content)
if mermaid_source is not None:
return self._render_flowchart_body(block, mermaid_source)
source = self._safe_block_image_source(block)
content = self._render_embedded_content(block.content, linkify_text=parent.sub_type != "flowchart")
alt = _plain_content_text(block.content) or parent.sub_type or "image"
parts = [self._render_image(source, alt=alt, class_name="mineru-image")] if source else []
if content.html:
if source:
parts.append(self._render_details(content.html, parent.sub_type or "image content"))
else:
parts.append(content.html)
return "".join(parts)
def _render_flowchart_body(self, block: ImageBodyBlock, mermaid_source: str) -> str:
"""输出 Mermaid canvas,并保留 raster 与源码两级失败回退。"""
source = self._safe_block_image_source(block)
escaped_source = html.escape(_replace_html_controls(mermaid_source), quote=False)
fallback = self._render_image(source, alt="flowchart", class_name="mineru-flowchart-fallback") if source else ""
fallback_class = " mineru-flowchart--has-raster" if fallback else ""
details_open = "" if fallback else " open"
return (
f'<div class="mineru-flowchart{fallback_class}" data-mermaid-state="pending">'
'<div class="mineru-flowchart-canvas" role="img" aria-label="flowchart"></div>'
f"{fallback}</div>"
f'<details class="mineru-details mineru-flowchart-details"{details_open}>'
"<summary>flowchart source</summary>"
f'<pre class="mineru-flowchart-source"><code>{escaped_source}</code></pre>'
"</details>"
)
def _render_table_block(self, block: TableBlock) -> str:
"""按原始子块顺序渲染表格主体、标题与脚注。"""
parts: list[str] = []
for child in block.content:
if isinstance(child, TableBodyBlock):
body = self._render_table_body(child)
elif isinstance(child, TableAnnotationBlock):
body = self._render_annotation(child)
else:
raise TypeError(f"Unsupported table child: {type(child).__name__}")
if body:
parts.append(body)
return f'<figure class="mineru-figure mineru-figure--table">{"".join(parts)}</figure>' if parts else ""
def _render_table_body(self, block: TableBodyBlock) -> str:
"""优先保留安全 HTML table,空间文本和整体图片依次回退。"""
normalized_content = _replace_html_controls(block.content)
content = normalized_content.strip()
if content:
source_soup = BeautifulSoup(content, "html.parser")
if source_soup.find("table") is not None:
rendered = self._render_embedded_content(content)
if rendered.html and _contains_usable_table(rendered.html):
return rendered.html
source = self._safe_block_image_source(block)
if source:
return self._render_image(source, alt="table", class_name="mineru-table-image")
return _render_raw_fallback(content)
return f'<pre class="mineru-table-text">{html.escape(normalized_content, quote=False)}</pre>'
source = self._safe_block_image_source(block)
return self._render_image(source, alt="table", class_name="mineru-table-image") if source else ""
def _render_chart_block(self, block: ChartBlock) -> str:
"""按原始子块顺序渲染 chart 图片、结构内容及说明。"""
parts: list[str] = []
for child in block.content:
if isinstance(child, ChartBodyBlock):
body = self._render_chart_body(block, child)
elif isinstance(child, ChartAnnotationBlock):
body = self._render_annotation(child)
else:
raise TypeError(f"Unsupported chart child: {type(child).__name__}")
if body:
parts.append(body)
return f'<figure class="mineru-figure mineru-figure--chart">{"".join(parts)}</figure>' if parts else ""
def _render_chart_body(self, parent: ChartBlock, block: ChartBodyBlock) -> str:
"""渲染 chart 图片,并将并存结构内容放入 details。"""
source = self._safe_block_image_source(block)
content = self._render_chart_content(block.content)
parts = [self._render_image(source, alt=parent.sub_type or "chart", class_name="mineru-chart-image")] if source else []
if content.html:
if source:
parts.append(self._render_details(content.html, parent.sub_type or "chart content"))
else:
parts.append(content.html)
return "".join(parts)
def _render_chart_content(self, content: str) -> HtmlInlineResult:
"""按 HTML、严格 GFM 表格、普通行内内容的顺序渲染 chart content。"""
normalized = content.strip()
if not normalized:
return HtmlInlineResult("")
if is_supported_html_markup(normalized):
return self._render_embedded_content(normalized)
if looks_like_gfm_table(normalized):
rendered = render_gfm_table_html(normalized)
if rendered is None:
return HtmlInlineResult(_render_raw_fallback(content))
self._observe_inline(rendered)
return rendered
rendered = render_inline_content_html(content)
self._observe_inline(rendered)
return rendered
def _render_code_block(self, block: CodeBlock) -> str:
"""按原始子块顺序渲染代码或算法及其说明。"""
parts: list[str] = []
for child in block.content:
if isinstance(child, CodeBodyBlock):
body = self._render_code_body(block, child)
elif isinstance(child, CodeAnnotationBlock):
body = self._render_annotation(child)
else:
raise TypeError(f"Unsupported code child: {type(child).__name__}")
if body:
parts.append(body)
return f'<figure class="mineru-figure mineru-figure--code">{"".join(parts)}</figure>' if parts else ""
def _render_code_body(self, parent: CodeBlock, block: CodeBodyBlock) -> str:
"""代码使用 Prism class,算法使用共享 inline AST 保留空白。"""
if parent.sub_type == BlockType.CODE:
escaped = html.escape(_replace_html_controls(block.content), quote=False)
if not block.content:
return '<pre class="mineru-code"><code></code></pre>'
language = _normalize_prism_language(parent.guess_lang)
class_attr = f' class="language-{language}"' if language else ""
pre_class = f"mineru-code language-{language}" if language else "mineru-code"
self.has_prism = bool(language) or self.has_prism
return f'<pre class="{pre_class}"><code{class_attr}>{escaped}</code></pre>'
if parent.sub_type == RAW_ALGORITHM:
rendered = render_inline_nodes_html(
parse_inline_content(block.content),
linkify_text=False,
separate_adjacent_math=True,
preserve_newlines=True,
)
self._observe_inline(rendered)
return f'<div class="mineru-algorithm">{rendered.html}</div>' if rendered.html else ""
raise ValueError(f"Unsupported code subtype: {parent.sub_type}")
def _render_annotation(
self,
block: ImageAnnotationBlock | TableAnnotationBlock | ChartAnnotationBlock | CodeAnnotationBlock,
) -> str:
"""把视觉说明渲染为保持原类型的独立段落。"""
rendered = render_inline_content_html(block.content)
self._observe_inline(rendered)
if not rendered.html:
return ""
is_caption = str(block.type).endswith("caption")
role_class = "mineru-caption" if is_caption else "mineru-footnote"
type_class = html.escape(str(block.type).replace("_", "-"), quote=True)
return f'<p class="{role_class} {role_class}--{type_class}">{rendered.html}</p>'
def _render_embedded_content(self, content: str, *, linkify_text: bool = True) -> HtmlInlineResult:
"""安全处理 body 富 HTML;普通内容走共享 inline AST。"""
normalized = content.strip()
if not normalized:
return HtmlInlineResult("")
if not is_supported_html_markup(normalized):
rendered = render_inline_nodes_html(parse_inline_content(content), linkify_text=linkify_text)
self._observe_inline(rendered)
return rendered
sanitized = sanitize_html_fragment(normalized, asset_base_url=self.asset_base_url)
rendered = self._replace_safe_equations(sanitized)
self._observe_inline(rendered)
return rendered
def _replace_safe_equations(self, markup: str) -> HtmlInlineResult:
"""把 sanitizer 保留的 eq 标签替换为可信 MathJax carrier。"""
if not markup:
return HtmlInlineResult("")
soup = BeautifulSoup(markup, "html.parser")
has_math = False
for equation in list(soup.find_all("eq")):
rendered = render_math_html(equation.get_text(), display=False)
if not rendered.html:
equation.decompose()
continue
replacement = BeautifulSoup(rendered.html, "html.parser")
equation.replace_with(*list(replacement.contents))
has_math = True
return HtmlInlineResult(str(soup), has_math)
def _safe_block_image_source(self, block: EquationBlock | ImageBodyBlock | TableBodyBlock | ChartBodyBlock) -> str | None:
"""解析 block 图片来源,并应用 HTML renderer 的 URL 安全策略。"""
if block.image_path:
return sanitize_image_source(block.image_path, asset_base_url=self.asset_base_url)
if block.image_base64:
return sanitize_image_source(block.image_base64)
return None
def _render_image(self, source: str, *, alt: str, class_name: str) -> str:
"""构造属性已转义的安全 img 元素。"""
return (
f'<img class="{class_name}" src="{html.escape(source, quote=True)}" '
f'alt="{html.escape(_replace_html_controls(alt), quote=True)}">'
)
def _render_details(self, content: str, summary: str) -> str:
"""构造浏览器原生折叠详情,不引入额外脚本。"""
return (
'<details class="mineru-details">'
f"<summary>{html.escape(_replace_html_controls(summary), quote=False)}</summary>{content}</details>"
)
def _observe_inline(self, rendered: HtmlInlineResult) -> None:
"""累计当前文档是否需要加载 MathJax。"""
self.has_math = rendered.has_math or self.has_math
def _render_standalone(self, article: str) -> str:
"""用固定 CSS 和按需 CDN 资源包装完整 HTML 文档。"""
title = html.escape(
_replace_html_controls(_resolve_document_title(self.middle_json, self.document_title)),
quote=False,
)
styles = _load_html_styles()
dependencies = []
if self.has_math:
dependencies.append(_mathjax_head())
if self.has_prism:
dependencies.append(_prism_head())
if self.has_mermaid:
dependencies.append(_mermaid_head())
dependency_html = "\n".join(dependencies)
return (
"<!doctype html>\n"
'<html lang="und">\n'
"<head>\n"
'<meta charset="utf-8">\n'
'<meta name="viewport" content="width=device-width, initial-scale=1.0">\n'
f"<title>{title}</title>\n"
f"<style>{styles}</style>\n"
f"{dependency_html}\n"
"</head>\n"
'<body class="mineru-html-body">\n'
f"{article}\n"
"</body>\n"
"</html>"
)
def render_html(
middle_json: MiddleJson,
*,
mode: RenderMode = RenderMode.DEFAULT,
asset_base_url: str = "",
standalone: bool = True,
document_title: str | None = None,
) -> str:
"""把严格 MiddleJson 无副作用地渲染为 HTML 文档或单根 fragment。"""
if not isinstance(middle_json, MiddleJson):
raise TypeError("render_html expects a MiddleJson instance")
if not isinstance(mode, RenderMode):
raise TypeError("mode must be a RenderMode value")
if not isinstance(asset_base_url, str):
raise TypeError("asset_base_url must be a string")
if not isinstance(standalone, bool):
raise TypeError("standalone must be a bool")
if document_title is not None and not isinstance(document_title, str):
raise TypeError("document_title must be a string or None")
return _HtmlRenderer(
middle_json,
mode=mode,
asset_base_url=asset_base_url,
standalone=standalone,
document_title=document_title,
).render()
def _classify_list(items: list[ListItem], add_reference_bullets: bool) -> tuple[str, str | None, str]:
"""根据直属 marker 选择原生列表类型或显式 marker 模式。"""
if add_reference_bullets:
return "ul", None, "mineru-list--reference"
if items and all(item.kind == "unordered" for item in items):
return "ul", None, "mineru-list--unordered"
if items and all(item.kind == "ordered" for item in items):
styles = {item.ordered_style for item in items}
if len(styles) == 1:
style = next(iter(styles))
list_type = {
"lower-alpha": "a",
"upper-alpha": "A",
"lower-roman": "i",
"upper-roman": "I",
}.get(style or "")
return "ol", list_type, "mineru-list--ordered"
if items and all(item.kind == "none" for item in items):
return "ul", None, "mineru-list--unmarked"
return "ul", None, "mineru-list--explicit"
def _list_item_content(
item: ListItem,
add_reference_bullets: bool,
*,
explicit_markers: bool,
) -> tuple[str, str | None]:
"""决定一个列表项应剥离、保留还是显式显示源 marker。"""
if add_reference_bullets:
if item.kind == "unordered":
return item.body, None
original = f"{item.leading}{item.marker or ''}{item.separator}{item.body}"
return original, None
if explicit_markers:
return item.body, item.marker
if item.kind in {"unordered", "ordered"}:
return item.body, None
return item.body, item.marker
def _serialize_index_items(items: list[dict[str, object]]) -> str:
"""把目录项及其已挂接的嵌套 ul 序列化为 li。"""
rendered: list[str] = []
for item in items:
nested = item["nested"]
assert isinstance(nested, list)
content = str(item["content"])
if not content and not nested:
continue
class_attr = ' class="mineru-list-item--orphan"' if item["orphan"] else ""
rendered.append(f"<li{class_attr}>{content}{''.join(str(value) for value in nested)}</li>")
return "".join(rendered)
def _collect_title_anchor_ids(middle_json: MiddleJson) -> dict[str, str]:
"""为正文首次出现的非空 anchor 分配无空白且 document-wide 唯一的 HTML id。"""
anchor_ids: dict[str, str] = {}
used_ids: set[str] = set()
for page in middle_json.pages:
for block in page.blocks:
if not isinstance(block, TitleBlockBase):
continue
anchor = _anchor_key(block.anchor)
visible_title = inline_plain_text(parse_inline_content(block.content)).strip()
if not anchor or not visible_title or anchor in anchor_ids:
continue
base_id = _html_anchor_base(anchor)
candidate = base_id
suffix = 2
while candidate in used_ids:
candidate = f"{base_id}-{suffix}"
suffix += 1
anchor_ids[anchor] = candidate
used_ids.add(candidate)
return anchor_ids
def _plain_content_text(content: str) -> str:
"""从 body 内容提取安全 img alt 所需的可见纯文本。"""
if not content:
return ""
if is_supported_html_markup(content):
return BeautifulSoup(content, "html.parser").get_text(" ", strip=True)
return inline_plain_text(parse_inline_content(content)).strip()
def _extract_mermaid_flowchart_source(content: str) -> str | None:
"""提取受限 Mermaid flowchart fence,拒绝其他图类型和可改写配置的语法。"""
normalized = _replace_html_controls(content).replace("\r\n", "\n").replace("\r", "\n").strip()
if not normalized:
return None
lines = normalized.split("\n")
if len(lines) < 3:
return None
opening = re.fullmatch(r"(?P<fence>\x60{3,})[ \t]*mermaid[ \t]*", lines[0], re.IGNORECASE)
if opening is None:
return None
fence_length = len(opening.group("fence"))
if re.fullmatch(rf"\x60{{{fence_length},}}[ \t]*", lines[-1]) is None:
return None
source = "\n".join(lines[1:-1]).strip("\n")
if not source or len(source) > _MERMAID_MAX_TEXT_SIZE:
return None
if source.lstrip().startswith("---") or "%%{" in source:
return None
first_statement = next(
(line.strip() for line in source.splitlines() if line.strip() and not line.lstrip().startswith("%%")),
"",
)
if _MERMAID_FLOWCHART_HEADER_RE.match(first_statement) is None:
return None
return source
def _contains_usable_table(markup: str) -> bool:
"""确认清洗后的 markup 至少保留一个含单元格的 table。"""
soup = BeautifulSoup(markup, "html.parser")
return any(table.find(("td", "th")) is not None for table in soup.find_all("table"))
def _render_raw_fallback(content: str) -> str:
"""把无法安全结构化的原内容转义为可见 pre,避免静默丢失。"""
return f'<pre class="mineru-raw-fallback">{html.escape(_replace_html_controls(content), quote=False)}</pre>'
def _normalize_prism_language(language: str | None) -> str | None:
"""把模型语言名归一化为不可构造路径穿越的 Prism component 名。"""
normalized = (language or "").strip().lower()
normalized = _PRISM_LANGUAGE_ALIASES.get(normalized, normalized.replace("_", "-"))
if not normalized or normalized in _BLOCKED_PRISM_LANGUAGE_NAMES:
return None
return normalized if _SAFE_PRISM_LANGUAGE_RE.fullmatch(normalized) else None
def _resolve_document_title(middle_json: MiddleJson, explicit_title: str | None) -> str:
"""按显式参数、首个正文文档标题、固定回退值解析 HTML title。"""
if explicit_title and explicit_title.strip():
return explicit_title.strip()
for page in middle_json.pages:
for block in page.blocks:
if isinstance(block, DocTitleBlock):
title = inline_plain_text(parse_inline_content(block.content)).strip()
if title:
return title
return "MinerU Document"
def _anchor_key(anchor: str | None) -> str:
"""保留 producer anchor 身份,仅去除首尾空白供标题与目录精确匹配。"""
return (anchor or "").strip()
def _html_anchor_base(anchor: str) -> str:
"""把原始 anchor 转成无空白的 HTML id 基值,碰撞由 document registry 处理。"""
return re.sub(r"\s+", "-", _replace_html_controls(anchor))
def _replace_html_controls(content: str) -> str:
"""替换 HTML 不允许的 C0 控制字符与不可编码的孤立 surrogate。"""
return re.sub(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f\ud800-\udfff]", "\ufffd", content)
@lru_cache(maxsize=1)
def _load_html_styles() -> str:
"""从包资源读取并缓存压缩后的 MinerU 独立样式。"""
root = resources.files("mineru").joinpath("resources", "html")
return root.joinpath(_STYLE_RESOURCE_NAME).read_text(encoding="utf-8")
def _mathjax_head() -> str:
"""返回固定、收紧公式输入能力的 MathJax 4.1.2 head 片段。"""
return f"""<script>
window.MathJax = {{
loader: {{load: ['ui/safe']}},
tex: {{
inlineMath: [['\\\\(', '\\\\)']],
displayMath: [['\\\\[', '\\\\]']],
processEscapes: true,
processEnvironments: false,
processRefs: false,
packages: {{'[-]': ['require']}}
}},
options: {{
ignoreHtmlClass: 'mineru-document',
processHtmlClass: 'mineru-math',
enableMenu: false,
enableEnrichment: false,
safeOptions: {{allow: {{URLs: 'none', classes: 'none', cssIDs: 'none', styles: 'none'}}}}
}},
startup: {{
ready: function () {{
window.MathJax.startup.defaultReady();
var options = window.MathJax.startup.document.options;
options.enableMenu = false;
options.enableEnrichment = false;
}}
}}
}};
</script>
<script id="MathJax-script" defer src="{_MATHJAX_URL}"
integrity="{_MATHJAX_INTEGRITY}" crossorigin="anonymous" referrerpolicy="no-referrer"></script>"""
def _prism_head() -> str:
"""返回固定 Prism core、Autoloader 和当前文档局部高亮脚本。"""
return f"""<script defer data-manual src="{_PRISM_CORE_URL}"
integrity="{_PRISM_CORE_INTEGRITY}" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
<script defer src="{_PRISM_AUTOLOADER_URL}"
integrity="{_PRISM_AUTOLOADER_INTEGRITY}" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
<script>
document.addEventListener('DOMContentLoaded', function () {{
if (!window.Prism || !Prism.plugins || !Prism.plugins.autoloader) return;
Prism.plugins.autoloader.languages_path = '{_PRISM_LANGUAGES_PATH}';
var root = document.querySelector('.mineru-document');
if (root) Prism.highlightAllUnder(root);
}});
</script>"""
def _mermaid_head() -> str:
"""返回固定 Mermaid 入口和逐图安全渲染、失败回退脚本。"""
return f"""<script defer src="{_MERMAID_URL}"
integrity="{_MERMAID_INTEGRITY}" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
<script>
function markMineruMermaidError(host) {{
host.dataset.mermaidState = 'error';
if (!host.classList.contains('mineru-flowchart--has-raster')) {{
var details = host.nextElementSibling;
if (details && details.classList.contains('mineru-flowchart-details')) details.open = true;
}}
}}
document.addEventListener('DOMContentLoaded', function () {{
var root = document.querySelector('.mineru-document');
var hosts = root ? Array.from(root.querySelectorAll('.mineru-flowchart')) : [];
if (!hosts.length) return;
if (!window.mermaid) {{
hosts.forEach(markMineruMermaidError);
return;
}}
try {{
window.mermaid.initialize({{
startOnLoad: false,
securityLevel: 'strict',
suppressErrorRendering: true,
theme: 'neutral',
maxTextSize: 50000,
maxEdges: 500,
flowchart: {{htmlLabels: false, useMaxWidth: true}}
}});
}} catch (_error) {{
hosts.forEach(markMineruMermaidError);
return;
}}
(async function () {{
for (var index = 0; index < hosts.length; index += 1) {{
var host = hosts[index];
var details = host.nextElementSibling;
var source = details && details.querySelector('.mineru-flowchart-source code');
var canvas = host.querySelector('.mineru-flowchart-canvas');
if (!source || !canvas) {{
markMineruMermaidError(host);
continue;
}}
var renderId = 'mineru-mermaid-' + index;
while (document.getElementById(renderId)) renderId += '-x';
host.dataset.mermaidState = 'rendering';
try {{
var result = await window.mermaid.render(renderId, source.textContent);
canvas.innerHTML = result.svg;
host.dataset.mermaidState = 'rendered';
}} catch (_error) {{
markMineruMermaidError(host);
}}
}}
}})();
}});
</script>"""
__all__ = ["render_html"]
+1 -1
View File
@@ -10,7 +10,7 @@ from urllib.parse import quote, unquote, urlsplit
import nh3
from bs4 import BeautifulSoup, NavigableString, Tag
from mineru.utils.image_payload import parse_image_data_uri_strict
from ....utils.image_payload import parse_image_data_uri_strict
_ALLOWED_TAGS = {
+1 -1
View File
@@ -5,7 +5,7 @@ from __future__ import annotations
import re
from mineru.render._internal.html.inline import HtmlInlineResult, render_inline_content_html
from .inline import HtmlInlineResult, render_inline_content_html
_SEPARATOR_CELL_RE = re.compile(r"^:?-{3,}:?$")
@@ -1 +1,4 @@
# Copyright (c) Opendatalab. All rights reserved.
"""Markdown 与 Content List 共用的内部实现。"""
__all__: list[str] = []
+1 -1
View File
@@ -6,7 +6,7 @@ from __future__ import annotations
import re
from urllib.parse import quote
from mineru.types import ImagePayloadBlock
from ....types import ImagePayloadBlock
_HTML_IMAGE_SRC_RE = re.compile(
r"(?P<prefix>\bsrc\s*=\s*)(?P<quote>[\"'])(?P<src>.*?)(?P=quote)",
+8 -8
View File
@@ -6,13 +6,13 @@ from __future__ import annotations
import html
import re
from mineru.config import LatexDelimitersConfig
from mineru.render._internal.common.index import strip_index_page_tail
from mineru.render._internal.common.list_items import has_markdown_unordered_marker, reference_list_needs_bullets
from mineru.render._internal.common.planner import PlannedBlock
from mineru.render._internal.markdown.assets import build_markdown_image, resolve_image_source
from mineru.render._internal.markdown.escaping import escape_standalone_marker_rule, escape_text_block_markdown_prefix
from mineru.render._internal.markdown.inline import (
from ....config import LatexDelimitersConfig
from ..common.index import strip_index_page_tail
from ..common.list_items import has_markdown_unordered_marker, reference_list_needs_bullets
from ..common.planner import PlannedBlock
from .assets import build_markdown_image, resolve_image_source
from .escaping import escape_standalone_marker_rule, escape_text_block_markdown_prefix
from .inline import (
render_inline_content,
render_internal_link,
render_joined_inline_contents,
@@ -23,7 +23,7 @@ from .table import (
render_html_table,
)
from ...contracts import ImageRenderer
from mineru.types import (
from ....types import (
RAW_ALGORITHM,
BlockType,
ChartAnnotationBlock,
+3 -3
View File
@@ -6,8 +6,8 @@ from __future__ import annotations
import html
import re
from mineru.config import LatexDelimitersConfig
from mineru.render._internal.common.inline import (
from ....config import LatexDelimitersConfig
from ....backend.postprocess.inline import (
InlineEquation,
InlineLink,
InlineNode,
@@ -17,7 +17,7 @@ from mineru.render._internal.common.inline import (
join_inline_contents,
parse_inline_content,
)
from mineru.render._internal.markdown.escaping import escape_conservative_markdown_text
from .escaping import escape_conservative_markdown_text
_ANGLE_TEXT_RE = re.compile(r"<[^<>\n]*>")
_SIMPLE_STYLE_WRAPPERS = {
@@ -0,0 +1,71 @@
# Copyright (c) Opendatalab. All rights reserved.
"""严格 MiddleJson 到 Markdown 的公共渲染实现。"""
from __future__ import annotations
from ....config import LatexDelimitersConfig, config
from ....types import PAGE_AUXILIARY_BLOCK_TYPES, MiddleJson
from ...contracts import ImageRenderer, RenderMode
from ..common.planner import PlannedBlock, build_render_plan
from .blocks import render_planned_block
_PAGE_SEPARATOR = "\n\n---\n\n"
def render_markdown(
middle_json: MiddleJson,
*,
mode: RenderMode = RenderMode.DEFAULT,
asset_base_url: str = "",
image_renderer: ImageRenderer | None = None,
) -> str:
"""把严格 MiddleJson 纯函数式渲染为 Markdown 字符串。"""
if not isinstance(middle_json, MiddleJson):
raise TypeError("render_markdown expects a MiddleJson instance")
if not isinstance(mode, RenderMode):
raise TypeError("mode must be a RenderMode value")
delimiters = config.render.latex_delimiters
planned_pages = build_render_plan(middle_json, mode)
rendered_pages = [
_render_page(
page,
mode=mode,
delimiters=delimiters,
asset_base_url=asset_base_url,
image_renderer=image_renderer,
)
for page in planned_pages
]
if mode is RenderMode.FULL:
return _PAGE_SEPARATOR.join(rendered_pages)
return "\n\n".join(page for page in rendered_pages if page)
def _render_page(
planned_blocks: list[PlannedBlock],
*,
mode: RenderMode,
delimiters: LatexDelimitersConfig,
asset_base_url: str,
image_renderer: ImageRenderer | None = None,
) -> str:
"""渲染单页逻辑块,并在默认模式中过滤重复页元素。"""
rendered: list[str] = []
for planned in planned_blocks:
if planned.removed:
continue
if mode is RenderMode.DEFAULT and planned.block.type in PAGE_AUXILIARY_BLOCK_TYPES:
continue
text = render_planned_block(
planned,
delimiters=delimiters,
asset_base_url=asset_base_url,
image_renderer=image_renderer,
)
if text and text.strip():
rendered.append(text.strip("\n"))
return "\n\n".join(rendered)
__all__ = ["render_markdown"]
@@ -0,0 +1,4 @@
# Copyright (c) Opendatalab. All rights reserved.
"""Structured Content 私有渲染实现。"""
__all__: list[str] = []
@@ -0,0 +1,181 @@
# Copyright (c) Opendatalab. All rights reserved.
"""严格 MiddleJson 到树形 Markdown structured_content 的公共渲染实现。"""
from __future__ import annotations
from typing import Any, TypeAlias
from ....config import LatexDelimitersConfig, config
from ..markdown.blocks import (
render_single_block,
render_title_inline_content,
render_visual_annotation,
render_visual_body_content,
)
from ..markdown.assets import normalize_image_source, resolve_image_source
from ..markdown.escaping import escape_standalone_marker_rule, escape_text_block_markdown_prefix
from ....types import (
BlockType,
ChartAnnotationBlock,
ChartBlock,
ChartBodyBlock,
CodeAnnotationBlock,
CodeBlock,
DocTitleBlock,
EquationBlock,
ImageAnnotationBlock,
ImageBlock,
ImageBodyBlock,
ImagePayloadBlock,
MiddleJson,
PageBlock,
ParagraphTitleBlock,
TableAnnotationBlock,
TableBlock,
TableBodyBlock,
)
VisualBlock: TypeAlias = ImageBlock | TableBlock | ChartBlock | CodeBlock
VisualAnnotationBlock: TypeAlias = ImageAnnotationBlock | TableAnnotationBlock | ChartAnnotationBlock | CodeAnnotationBlock
_CAPTION_TYPES = {
BlockType.IMAGE_CAPTION,
BlockType.TABLE_CAPTION,
BlockType.CHART_CAPTION,
BlockType.CODE_CAPTION,
}
_FOOTNOTE_TYPES = {
BlockType.IMAGE_FOOTNOTE,
BlockType.TABLE_FOOTNOTE,
BlockType.CHART_FOOTNOTE,
BlockType.CODE_FOOTNOTE,
}
_REMOVED_BLOCK_FIELDS = {"content", "index", "guess_lang", "image_path", "image_base64"}
def render_structured_content(
middle_json: MiddleJson,
*,
asset_base_url: str = "",
) -> dict[str, Any]:
"""把严格 MiddleJson 无副作用地渲染为树形 Markdown structured_content。"""
if not isinstance(middle_json, MiddleJson):
raise TypeError("render_structured_content expects a MiddleJson instance")
delimiters = config.render.latex_delimiters
document_fields = middle_json.model_dump(
mode="json",
exclude={"pages"},
exclude_defaults=True,
)
pages = [
{
"page_idx": page.page_idx,
"blocks": [
_render_content_block(
block,
delimiters=delimiters,
asset_base_url=asset_base_url,
)
for block in page.blocks
],
}
for page in middle_json.pages
]
return {"pages": pages, **document_fields}
def _render_content_block(
block: PageBlock,
*,
delimiters: LatexDelimitersConfig,
asset_base_url: str,
) -> dict[str, Any]:
"""保留父块元数据,并把 block 内容收敛为 Markdown 字符串。"""
payload = block.model_dump(
mode="json",
exclude=_REMOVED_BLOCK_FIELDS,
exclude_defaults=True,
)
if isinstance(block, (DocTitleBlock, ParagraphTitleBlock)):
content = render_title_inline_content(block, delimiters)
payload["content"] = escape_standalone_marker_rule(escape_text_block_markdown_prefix(content))
return payload
if isinstance(block, EquationBlock):
payload["content"] = block.content.strip()
image_source = _resolve_content_image_source(block, asset_base_url)
if image_source is not None:
payload["image_source"] = image_source
return payload
if not isinstance(block, (ImageBlock, TableBlock, ChartBlock, CodeBlock)):
payload["content"] = render_single_block(
block,
delimiters=delimiters,
asset_base_url=asset_base_url,
)
return payload
payload["content"] = render_visual_body_content(
block,
delimiters=delimiters,
asset_base_url=asset_base_url,
)
payload["captions"] = _render_annotation_group(block, _CAPTION_TYPES, delimiters)
payload["footnotes"] = _render_annotation_group(block, _FOOTNOTE_TYPES, delimiters)
image_source = _resolve_visual_image_source(block, asset_base_url)
if image_source is not None:
payload["image_source"] = image_source
return payload
def _render_annotation_group(
block: VisualBlock,
accepted_types: set[str],
delimiters: LatexDelimitersConfig,
) -> list[dict[str, Any]]:
"""按源 index 稳定排序视觉说明,并保留可用 bbox 与 Markdown 内容。"""
annotations: list[tuple[int, VisualAnnotationBlock]] = []
for position, child in enumerate(block.content):
if (
isinstance(
child,
(ImageAnnotationBlock, TableAnnotationBlock, ChartAnnotationBlock, CodeAnnotationBlock),
)
and child.type in accepted_types
):
annotations.append((position, child))
annotations.sort(key=_annotation_sort_key)
rendered_annotations: list[dict[str, Any]] = []
for _, child in annotations:
annotation_payload: dict[str, Any] = {}
if child.bbox is not None:
annotation_payload["bbox"] = list(child.bbox)
annotation_payload["content"] = render_visual_annotation(child, delimiters)
rendered_annotations.append(annotation_payload)
return rendered_annotations
def _annotation_sort_key(
item: tuple[int, VisualAnnotationBlock],
) -> tuple[bool, int, int]:
"""让有 index 的说明升序优先,缺失 index 的说明稳定排在末尾。"""
position, block = item
return block.index is None, block.index if block.index is not None else 0, position
def _resolve_visual_image_source(block: VisualBlock, asset_base_url: str) -> str | None:
"""解析视觉 body 实际选择的图片来源,并返回安全的 Markdown 地址。"""
for child in block.content:
if not isinstance(child, (ImageBodyBlock, TableBodyBlock, ChartBodyBlock)):
continue
return _resolve_content_image_source(child, asset_base_url)
return None
def _resolve_content_image_source(block: ImagePayloadBlock, asset_base_url: str) -> str | None:
"""把图片载荷收敛为 structured_content 中唯一且安全的 image_source。"""
source = resolve_image_source(block, asset_base_url)
return normalize_image_source(source) if source else None
__all__ = ["render_structured_content"]
+1 -1
View File
@@ -5,7 +5,7 @@ from __future__ import annotations
from typing import Any, Literal, overload
from mineru.types import MiddleJson
from ..types import MiddleJson
from .contracts import (
DocxRenderOptions,
+6 -835
View File
@@ -1,110 +1,10 @@
# Copyright (c) Opendatalab. All rights reserved.
"""严格 MiddleJson 到可编辑 DOCX bytes 的公共渲染实现。"""
"""严格 MiddleJson 到 DOCX 的轻量公共门面与稳定异常。"""
from __future__ import annotations
from collections.abc import Iterable
from io import BytesIO
import re
from bs4 import BeautifulSoup, NavigableString, Tag
from docx import Document
from docx.document import Document as DocumentType
from docx.enum.table import WD_CELL_VERTICAL_ALIGNMENT
from docx.enum.text import WD_ALIGN_PARAGRAPH, WD_TAB_ALIGNMENT
from docx.oxml import OxmlElement
from docx.oxml.ns import qn
from docx.shared import Emu, Mm, Pt, Twips
from docx.table import _Cell
from docx.text.paragraph import Paragraph
from lxml import etree
from loguru import logger
from mineru.render._internal.common.index import strip_index_page_tail
from mineru.render._internal.common.inline import (
InlineEquation,
InlineLink,
InlineNode,
InlineStyled,
InlineText,
parse_inline_content,
)
from mineru.render._internal.common.list_items import parse_list_item_marker
from mineru.render._internal.common.planner import PlannedBlock, build_render_plan
from mineru.render._internal.docx.assets import (
DocxAssetError,
PreparedImage,
prepare_block_image,
prepare_html_image,
)
from mineru.render._internal.docx.inline import (
BookmarkRegistry,
InlineRenderContext,
append_inline_content,
append_inline_nodes,
append_internal_link,
append_joined_inline_contents,
sanitize_xml_text,
)
from mineru.render._internal.docx.math import DocxFormulaError, latex_to_omml, split_formula_tag
from mineru.render._internal.docx.styles import (
AUXILIARY_STYLE,
BODY_STYLE,
CAPTION_STYLE,
CODE_STYLE,
FOOTNOTE_STYLE,
FORMULA_FALLBACK_STYLE,
SPATIAL_TABLE_STYLE,
configure_document,
usable_width_emu,
usable_width_twips,
)
from mineru.render._internal.docx.table import DocxTableError, NestedTableWriter, materialize_docx_tables
from mineru.render.contracts import AssetResolver, RenderMode
from mineru.types import (
PAGE_AUXILIARY_BLOCK_TYPES,
RAW_ALGORITHM,
BlockBase,
BlockType,
BBox,
ChartAnnotationBlock,
ChartBlock,
ChartBodyBlock,
CodeAnnotationBlock,
CodeBlock,
CodeBodyBlock,
DocTitleBlock,
EquationBlock,
ImageAnnotationBlock,
ImageBlock,
ImageBodyBlock,
ImagePayloadBlock,
IndexBlock,
ListBlock,
MiddleJson,
PageAuxTextBlock,
ParagraphTitleBlock,
RefTextBlock,
TableAnnotationBlock,
TableBlock,
TableBodyBlock,
TextBlock,
TitleBlockBase,
)
_HTML_TABLE_RE = re.compile(r"<table\b", re.IGNORECASE)
_ANNOTATION_CAPTION_TYPES = {
BlockType.IMAGE_CAPTION,
BlockType.TABLE_CAPTION,
BlockType.CHART_CAPTION,
BlockType.CODE_CAPTION,
}
_ANNOTATION_FOOTNOTE_TYPES = {
BlockType.IMAGE_FOOTNOTE,
BlockType.TABLE_FOOTNOTE,
BlockType.CHART_FOOTNOTE,
BlockType.CODE_FOOTNOTE,
}
from ..types import MiddleJson
from .contracts import AssetResolver, RenderMode
class DocxRenderError(RuntimeError):
@@ -125,745 +25,16 @@ class DocxRenderError(RuntimeError):
super().__init__(f"{message} (page_idx={page_idx}, block_index={block_index}, block_type={block_type})")
class _DocxRenderer:
"""持有单次 DOCX 渲染所需的 document、素材解析器和书签状态。"""
def __init__(
self,
middle_json: MiddleJson,
*,
mode: RenderMode,
asset_resolver: AssetResolver | None,
) -> None:
"""初始化 renderer,并预注册全部标题 anchor。"""
self.middle_json = middle_json
self.mode = mode
self.asset_resolver = asset_resolver
self.document = Document()
configure_document(self.document)
self.bookmarks = BookmarkRegistry(_iter_document_anchors(middle_json))
self.usable_width_emu = usable_width_emu(self.document)
self.usable_width_twips = usable_width_twips(self.document)
def render(self) -> bytes:
"""执行逐页 visitor,并把 python-docx document 序列化为 bytes。"""
planned_pages = build_render_plan(self.middle_json, self.mode)
for page_position, planned_blocks in enumerate(planned_pages):
if self.mode is RenderMode.FULL and page_position > 0:
self.document.add_page_break()
for planned in planned_blocks:
if planned.removed:
continue
if self.mode is RenderMode.DEFAULT and planned.block.type in PAGE_AUXILIARY_BLOCK_TYPES:
continue
self._render_planned_block(planned)
output = BytesIO()
self.document.save(output)
return output.getvalue()
def _render_planned_block(self, planned: PlannedBlock) -> None:
"""按严格 PageBlock 具体类型分派 Word 写入逻辑。"""
block = planned.block
context = self._context(planned.page_idx, block)
if isinstance(block, (TextBlock, RefTextBlock)):
paragraph = self.document.add_paragraph(style=BODY_STYLE)
append_joined_inline_contents(
paragraph,
planned.text_contents or [block.content],
context=context,
)
return
if isinstance(block, (DocTitleBlock, ParagraphTitleBlock)):
self._render_title(block, context)
return
if isinstance(block, PageAuxTextBlock):
paragraph = self.document.add_paragraph(style=AUXILIARY_STYLE)
append_inline_content(paragraph, block.content, context=context)
return
if isinstance(block, EquationBlock):
self._render_equation(block, context)
return
if isinstance(block, ListBlock):
self._render_list(block, context, depth=0)
return
if isinstance(block, IndexBlock):
self._render_index(block, context, depth=0)
return
if isinstance(block, ImageBlock):
self._render_image_block(block, context)
return
if isinstance(block, TableBlock):
self._render_table_block(block, context)
return
if isinstance(block, ChartBlock):
self._render_chart_block(block, context)
return
if isinstance(block, CodeBlock):
self._render_code_block(block, context)
return
raise TypeError(f"Unsupported PageBlock type: {type(block).__name__}")
def _context(self, page_idx: int, block: BlockBase) -> InlineRenderContext:
"""为当前 block 构造行内渲染上下文。"""
return InlineRenderContext(
bookmarks=self.bookmarks,
page_idx=page_idx,
block_index=block.index,
block_type=block.type,
)
def _render_title(
self,
block: DocTitleBlock | ParagraphTitleBlock,
context: InlineRenderContext,
) -> None:
"""用 Heading 1–6 写严格标题,并在段落内容外添加 bookmark。"""
level = min(max(block.level, 1), 9)
paragraph = self.document.add_paragraph(style=f"Heading {level}")
append_inline_content(paragraph, block.content, context=context)
self.bookmarks.attach(paragraph, block.anchor)
def _render_equation(self, block: EquationBlock, context: InlineRenderContext) -> None:
"""优先写可编辑 OMML,失败时按图片、可见 LaTeX 的顺序回退。"""
content = block.content.strip()
if content:
formula, tag = split_formula_tag(content)
try:
math_element = latex_to_omml(formula, display=True)
self._append_display_formula(math_element, tag, context=context)
return
except DocxFormulaError as exc:
logger.warning("DOCX display formula fallback: {} ({})", exc, context.location())
if block.image_base64 is not None or block.image_path is not None:
try:
self._append_block_image(block, context=context, alt_text="formula")
return
except DocxRenderError:
if not content:
raise
logger.warning("DOCX formula image fallback failed; preserving LaTeX ({})", context.location())
if content:
paragraph = self.document.add_paragraph(style=FORMULA_FALLBACK_STYLE)
paragraph.add_run(sanitize_xml_text(content, context=context))
return
raise self._render_error("Equation does not contain usable LaTeX or image", context)
def _append_display_formula(
self,
math_element: etree._Element,
tag: str | None,
*,
context: InlineRenderContext,
) -> None:
"""把块公式居中写入;带 tag 时使用中心与右对齐 tab。"""
if tag is None:
properties = OxmlElement("m:oMathParaPr")
justification = OxmlElement("m:jc")
justification.set(qn("m:val"), "center")
properties.append(justification)
math_element.insert(0, properties)
paragraph = self.document.add_paragraph(style=BODY_STYLE)
paragraph.paragraph_format.space_before = Pt(4)
paragraph.paragraph_format.space_after = Pt(5)
paragraph._p.append(math_element)
return
equation = math_element.find(qn("m:oMath"))
if equation is None:
raise ValueError("Display OMML does not contain m:oMath")
math_element.remove(equation)
paragraph = self.document.add_paragraph(style=BODY_STYLE)
paragraph.paragraph_format.space_before = Pt(4)
paragraph.paragraph_format.space_after = Pt(5)
paragraph.paragraph_format.tab_stops.add_tab_stop(
Twips(self.usable_width_twips // 2),
WD_TAB_ALIGNMENT.CENTER,
)
paragraph.paragraph_format.tab_stops.add_tab_stop(
Twips(self.usable_width_twips),
WD_TAB_ALIGNMENT.RIGHT,
)
paragraph.add_run().add_tab()
paragraph._p.append(equation)
paragraph.add_run().add_tab()
rendered_tag = tag if tag.startswith("(") and tag.endswith(")") else f"({tag})"
paragraph.add_run(sanitize_xml_text(rendered_tag, context=context))
def _render_list(
self,
block: ListBlock,
context: InlineRenderContext,
*,
depth: int,
) -> None:
"""保留叶子原 marker,并仅用缩进表达递归列表层级。"""
for child in block.content:
if isinstance(child, ListBlock):
self._render_list(child, context, depth=depth + 1)
continue
paragraph = self.document.add_paragraph(style=BODY_STYLE)
item = parse_list_item_marker(child.content)
if item.marker is None:
paragraph.paragraph_format.left_indent = Mm(depth * 6)
else:
paragraph.paragraph_format.left_indent = Mm((depth + 1) * 6)
paragraph.paragraph_format.first_line_indent = Mm(-6)
append_inline_content(paragraph, child.content, context=context)
def _render_index(
self,
block: IndexBlock,
context: InlineRenderContext,
*,
depth: int,
) -> None:
"""递归输出目录项,并把标题叶子链接到预注册 bookmark。"""
for child in block.content:
if isinstance(child, IndexBlock):
self._render_index(child, context, depth=depth + 1)
continue
content = strip_index_page_tail(child.content)
nodes = parse_inline_content(content)
if not nodes:
continue
paragraph = self.document.add_paragraph(style=BODY_STYLE)
paragraph.paragraph_format.left_indent = Mm((depth + 1) * 6)
paragraph.paragraph_format.first_line_indent = Mm(-4)
paragraph.add_run("• ")
anchor = child.anchor if isinstance(child, TitleBlockBase) else None
append_internal_link(paragraph, nodes, anchor=anchor, context=context)
def _render_image_block(self, block: ImageBlock, context: InlineRenderContext) -> None:
"""按原始子块顺序写图片主体、caption 与 footnote。"""
for child in block.content:
if isinstance(child, ImageBodyBlock):
alt_text = _plain_html_text(child.content) or block.sub_type or "image"
self._append_block_image(child, context=context, alt_text=alt_text)
elif isinstance(child, ImageAnnotationBlock):
self._render_annotation(child, context)
else:
raise TypeError(f"Unsupported image child: {type(child).__name__}")
def _render_table_block(self, block: TableBlock, context: InlineRenderContext) -> None:
"""按原始子块顺序写原生 HTML table 或预格式空间投影文本。"""
for child in block.content:
if isinstance(child, TableBodyBlock):
if _HTML_TABLE_RE.search(child.content):
try:
self._append_html_tables(child.content, context=context, depth=0)
except DocxRenderError as exc:
logger.warning("DOCX HTML table fallback: {}", exc)
if child.image_base64 is None and child.image_path is None:
raise self._render_error(
"HTML table cannot be materialized and has no image fallback",
context,
) from exc
self._append_block_image(child, context=context, alt_text="table")
else:
if child.content and not child.content.isspace():
paragraph = self.document.add_paragraph(style=SPATIAL_TABLE_STYLE)
paragraph.add_run(sanitize_xml_text(child.content, context=context))
elif child.image_base64 is not None or child.image_path is not None:
self._append_block_image(child, context=context, alt_text="table")
else:
raise self._render_error(
"Spatial table does not contain text content or image",
context,
)
elif isinstance(child, TableAnnotationBlock):
self._render_annotation(child, context)
else:
raise TypeError(f"Unsupported table child: {type(child).__name__}")
def _render_chart_block(self, block: ChartBlock, context: InlineRenderContext) -> None:
"""先写图表图片,再把 HTML 结构化数据追加为原生表格。"""
for child in block.content:
if isinstance(child, ChartBodyBlock):
has_image = child.image_base64 is not None or child.image_path is not None
if has_image:
self._append_block_image(
child,
context=context,
alt_text=_plain_html_text(child.content) or block.sub_type or "chart",
)
if _HTML_TABLE_RE.search(child.content):
try:
self._append_html_tables(child.content, context=context, depth=0)
except DocxRenderError as exc:
if not has_image:
raise
logger.warning("DOCX chart data table omitted after image fallback: {}", exc)
elif child.content.strip() and not has_image:
paragraph = self.document.add_paragraph(style=BODY_STYLE)
append_inline_content(paragraph, child.content, context=context)
elif not has_image and not child.content.strip():
raise self._render_error("Chart does not contain image or structured content", context)
elif isinstance(child, ChartAnnotationBlock):
self._render_annotation(child, context)
else:
raise TypeError(f"Unsupported chart child: {type(child).__name__}")
def _render_code_block(self, block: CodeBlock, context: InlineRenderContext) -> None:
"""按原始子块顺序写代码或算法主体及说明。"""
for child in block.content:
if isinstance(child, CodeBodyBlock):
paragraph = self.document.add_paragraph(style=CODE_STYLE)
if block.sub_type == BlockType.CODE:
paragraph.add_run(sanitize_xml_text(child.content, context=context))
elif block.sub_type == RAW_ALGORITHM:
append_inline_content(paragraph, child.content, context=context)
else:
raise ValueError(f"Unsupported code subtype: {block.sub_type}")
elif isinstance(child, CodeAnnotationBlock):
self._render_annotation(child, context)
else:
raise TypeError(f"Unsupported code child: {type(child).__name__}")
def _render_annotation(
self,
block: ImageAnnotationBlock | TableAnnotationBlock | ChartAnnotationBlock | CodeAnnotationBlock,
context: InlineRenderContext,
) -> None:
"""根据说明类型选择 Caption 或 Footnote 样式并写入行内内容。"""
if block.type in _ANNOTATION_CAPTION_TYPES:
style = CAPTION_STYLE
elif block.type in _ANNOTATION_FOOTNOTE_TYPES:
style = FOOTNOTE_STYLE
else:
raise TypeError(f"Unsupported annotation type: {block.type}")
paragraph = self.document.add_paragraph(style=style)
append_inline_content(paragraph, block.content, context=context)
def _append_block_image(
self,
block: ImagePayloadBlock,
*,
context: InlineRenderContext,
alt_text: str,
) -> None:
"""安全加载 block 图片,并按 bbox/自然尺寸限制到可用页面范围。"""
try:
prepared = prepare_block_image(block, self.asset_resolver)
except DocxAssetError as exc:
raise self._render_error(str(exc), context) from exc
self._append_prepared_image(
prepared,
bbox=block.bbox,
alt_text=alt_text,
max_width_emu=self.usable_width_emu,
context=context,
)
def _append_prepared_image(
self,
prepared: PreparedImage,
*,
bbox: BBox | None,
alt_text: str,
max_width_emu: int,
context: InlineRenderContext,
paragraph: Paragraph | None = None,
) -> None:
"""把准备好的图片按比例插入居中段落并写入 alt description。"""
target_paragraph = paragraph or self.document.add_paragraph(style=BODY_STYLE)
target_paragraph.alignment = WD_ALIGN_PARAGRAPH.CENTER
natural_width = round(prepared.width_px / 96 * 914400)
desired_width = natural_width
if bbox is not None:
desired_width = round(max_width_emu * max(0.0, min(1.0, bbox[2] - bbox[0])))
minimum_width = min(int(Mm(5)), max_width_emu)
desired_width = max(minimum_width, min(desired_width, max_width_emu))
usable_height = int(Mm(257))
rendered_height = round(desired_width * prepared.height_px / max(prepared.width_px, 1))
if rendered_height > usable_height:
desired_width = round(usable_height * prepared.width_px / max(prepared.height_px, 1))
shape = target_paragraph.add_run().add_picture(
BytesIO(prepared.data),
width=Emu(max(1, desired_width)),
)
description = sanitize_xml_text(
re.sub(r"\s+", " ", alt_text).strip()[:2048],
context=context,
)
if description:
shape._inline.docPr.set("descr", description)
def _append_html_tables(
self,
markup: str,
*,
context: InlineRenderContext,
depth: int,
container: DocumentType | _Cell | None = None,
) -> None:
"""物化 HTML 表格,并通过共享行内/素材逻辑填充 origin cell。"""
if depth >= 4:
raise self._render_error("Nested HTML table depth exceeds 4", context)
target_container = container or self.document
target_xml = target_container._element.body if isinstance(target_container, DocumentType) else target_container._tc
existing_child_count = len(target_xml)
insertion_index = existing_child_count - 1 if isinstance(target_container, DocumentType) else existing_child_count
existing_relationship_ids = set(self.document.part.rels)
def fill_cell(cell: _Cell, source: Tag, write_nested: NestedTableWriter) -> None:
"""填充一个 origin cell,并在遇到 nested table 时调用递归 writer。"""
self._fill_html_cell(
cell,
source,
write_nested=write_nested,
context=context,
)
try:
materialize_docx_tables(
target_container,
markup,
width_twips=self.usable_width_twips,
fill_cell=fill_cell,
)
except DocxRenderError:
self._rollback_table_materialization(
target_xml,
existing_child_count=existing_child_count,
insertion_index=insertion_index,
existing_relationship_ids=existing_relationship_ids,
)
raise
except DocxTableError as exc:
self._rollback_table_materialization(
target_xml,
existing_child_count=existing_child_count,
insertion_index=insertion_index,
existing_relationship_ids=existing_relationship_ids,
)
raise self._render_error(str(exc), context) from exc
def _rollback_table_materialization(
self,
target_xml: etree._Element,
*,
existing_child_count: int,
insertion_index: int,
existing_relationship_ids: set[str],
) -> None:
"""回滚本轮表格新增的 XML 根节点和 relationship,避免残留半成品。"""
while len(target_xml) > existing_child_count:
target_xml.remove(target_xml[insertion_index])
for relationship_id in tuple(self.document.part.rels):
if relationship_id not in existing_relationship_ids:
self.document.part.drop_rel(relationship_id)
def _fill_html_cell(
self,
cell: _Cell,
source: Tag,
*,
write_nested: NestedTableWriter,
context: InlineRenderContext,
) -> None:
"""按源顺序填充单元格文本、图片和任意包装层中的嵌套表格。"""
cell.text = ""
cell.vertical_alignment = WD_CELL_VERTICAL_ALIGNMENT.CENTER
_set_cell_margins(cell, top=100, start=120, bottom=100, end=120)
max_image_width_emu = _cell_content_width_emu(cell, horizontal_margin_twips=240)
paragraph = cell.paragraphs[0]
paragraph.paragraph_format.space_after = Pt(0)
paragraph.paragraph_format.line_spacing = 1.0
for child in source.children:
if not isinstance(child, (NavigableString, Tag)):
continue
if isinstance(child, Tag) and child.name.lower() in {"p", "div"} and _paragraph_has_content(paragraph):
paragraph = cell.add_paragraph()
paragraph.paragraph_format.space_after = Pt(0)
paragraph.paragraph_format.line_spacing = 1.0
styles = ("bold",) if source.name == "th" else ()
paragraph, _ = self._append_html_node(
paragraph,
child,
cell=cell,
write_nested=write_nested,
context=context,
inherited_styles=styles,
max_image_width_emu=max_image_width_emu,
)
def _append_html_node(
self,
paragraph: Paragraph,
node: NavigableString | Tag,
*,
cell: _Cell,
write_nested: NestedTableWriter,
context: InlineRenderContext,
inherited_styles: tuple[str, ...],
max_image_width_emu: int,
) -> tuple[Paragraph, bool]:
"""按 HTML 源顺序写入单元格行内文本、样式、公式和图片。"""
if isinstance(node, NavigableString):
text = str(node)
if not text:
return paragraph, False
nodes: list[InlineNode] = [InlineText(text)]
if inherited_styles:
nodes = [InlineStyled(nodes, inherited_styles)]
append_inline_nodes(paragraph, nodes, context=context)
return paragraph, bool(text.strip())
name = node.name.lower()
if name == "table":
write_nested(node)
trailing_paragraph = cell.paragraphs[-1]
trailing_paragraph.paragraph_format.space_after = Pt(0)
trailing_paragraph.paragraph_format.line_spacing = 1.0
return trailing_paragraph, True
if name == "img":
self._append_html_image(
paragraph,
str(node.get("src", "")),
context=context,
alt_text=str(node.get("alt", "")),
max_width_emu=max_image_width_emu,
)
return paragraph, True
if name == "br":
paragraph.add_run().add_break()
return paragraph, True
if name == "eq":
latex = node.get_text().strip()
nodes = [InlineEquation(latex)] if latex else []
if inherited_styles and nodes:
nodes = [InlineStyled(nodes, inherited_styles)]
append_inline_nodes(paragraph, nodes, context=context)
return paragraph, bool(nodes)
if name == "a":
children = _html_inline_nodes(node)
if inherited_styles and children:
children = [InlineStyled(children, inherited_styles)]
append_inline_nodes(
paragraph,
[InlineLink(children, str(node.get("href", "")).strip())],
context=context,
)
return paragraph, bool(children)
if name in {"ul", "ol"}:
start_value = node.get("start", "1")
try:
ordered_number = int(str(start_value))
except ValueError:
ordered_number = 1
rendered = False
for item in node.find_all("li", recursive=False):
if _paragraph_has_content(paragraph) or rendered:
paragraph.add_run().add_break()
marker = f"{ordered_number}. " if name == "ol" else "- "
paragraph.add_run(marker)
if name == "ol":
ordered_number += 1
for child in item.children:
if isinstance(child, (NavigableString, Tag)):
paragraph, child_rendered = self._append_html_node(
paragraph,
child,
cell=cell,
write_nested=write_nested,
context=context,
inherited_styles=inherited_styles,
max_image_width_emu=max_image_width_emu,
)
rendered = child_rendered or rendered
return paragraph, rendered
if name == "li":
rendered = False
for child in node.children:
if isinstance(child, (NavigableString, Tag)):
paragraph, child_rendered = self._append_html_node(
paragraph,
child,
cell=cell,
write_nested=write_nested,
context=context,
inherited_styles=inherited_styles,
max_image_width_emu=max_image_width_emu,
)
rendered = child_rendered or rendered
return paragraph, rendered
if name == "text":
nodes = parse_inline_content(str(node))
if inherited_styles and nodes:
nodes = [InlineStyled(nodes, inherited_styles)]
append_inline_nodes(paragraph, nodes, context=context)
return paragraph, bool(nodes)
style = {
"strong": "bold",
"b": "bold",
"em": "italic",
"i": "italic",
"u": "underline",
"s": "strikethrough",
"sup": "superscript",
"sub": "subscript",
}.get(name)
styles = tuple(dict.fromkeys((*inherited_styles, style))) if style else inherited_styles
rendered = False
for child in node.children:
if isinstance(child, (NavigableString, Tag)):
paragraph, child_rendered = self._append_html_node(
paragraph,
child,
cell=cell,
write_nested=write_nested,
context=context,
inherited_styles=styles,
max_image_width_emu=max_image_width_emu,
)
rendered = child_rendered or rendered
return paragraph, rendered
def _append_html_image(
self,
paragraph: Paragraph,
source: str,
*,
context: InlineRenderContext,
alt_text: str,
max_width_emu: int,
) -> None:
"""安全加载表格单元格 img,并限制到紧凑的单元格宽度。"""
try:
prepared = prepare_html_image(source, self.asset_resolver)
except DocxAssetError as exc:
raise self._render_error(str(exc), context) from exc
self._append_prepared_image(
prepared,
bbox=None,
alt_text=alt_text or "table image",
max_width_emu=min(self.usable_width_emu, max_width_emu),
context=context,
paragraph=paragraph,
)
def _render_error(self, message: str, context: InlineRenderContext) -> DocxRenderError:
"""用当前 context 构造带 page/block 定位的公共异常。"""
return DocxRenderError(
message,
page_idx=context.page_idx,
block_index=context.block_index,
block_type=context.block_type,
)
def render_docx(
middle_json: MiddleJson,
*,
mode: RenderMode = RenderMode.DEFAULT,
asset_resolver: AssetResolver | None = None,
) -> bytes:
"""把严格 MiddleJson 无副作用地渲染为完整 DOCX bytes。"""
if not isinstance(middle_json, MiddleJson):
raise TypeError("render_docx expects a MiddleJson instance")
if not isinstance(mode, RenderMode):
raise TypeError("mode must be a RenderMode value")
return _DocxRenderer(
middle_json,
mode=mode,
asset_resolver=asset_resolver,
).render()
"""惰性加载 DOCX 实现并渲染严格 MiddleJson。"""
from ._internal.docx.renderer import render_docx as _render_docx
def _iter_document_anchors(middle_json: MiddleJson) -> Iterable[str]:
"""按文档顺序遍历实际会写入 bookmark 的正文标题 anchor。"""
for page in middle_json.pages:
for block in page.blocks:
if isinstance(block, TitleBlockBase) and block.anchor:
yield block.anchor
def _plain_html_text(content: str) -> str:
"""把图片或图表 body 内容收敛为适合 alt description 的纯文本。"""
if not content.strip():
return ""
return BeautifulSoup(content, "html.parser").get_text(" ", strip=True)
def _html_inline_nodes(node: Tag) -> list[InlineNode]:
"""把表格单元格中的安全 HTML 行内标签转换为共享 InlineNode。"""
nodes: list[InlineNode] = []
for child in node.children:
if isinstance(child, NavigableString):
if str(child):
nodes.append(InlineText(str(child)))
continue
if not isinstance(child, Tag):
continue
name = child.name.lower()
children = _html_inline_nodes(child)
if name == "eq":
latex = child.get_text().strip()
if latex:
nodes.append(InlineEquation(latex))
elif name in {"strong", "b"}:
nodes.append(InlineStyled(children, ("bold",)))
elif name in {"em", "i"}:
nodes.append(InlineStyled(children, ("italic",)))
elif name == "u":
nodes.append(InlineStyled(children, ("underline",)))
elif name == "s":
nodes.append(InlineStyled(children, ("strikethrough",)))
elif name == "sup":
nodes.append(InlineStyled(children, ("superscript",)))
elif name == "sub":
nodes.append(InlineStyled(children, ("subscript",)))
elif name == "a":
nodes.append(InlineLink(children, str(child.get("href", "")).strip()))
elif name == "br":
nodes.append(InlineText("\n"))
elif name == "text":
nodes.extend(parse_inline_content(str(child)))
elif name not in {"img", "table"}:
nodes.extend(children)
return nodes
def _set_cell_margins(
cell: _Cell,
*,
top: int,
start: int,
bottom: int,
end: int,
) -> None:
"""用 DXA 为 Word 单元格写入四边内边距。"""
cell_properties = cell._tc.get_or_add_tcPr()
margins = cell_properties.find(qn("w:tcMar"))
if margins is None:
margins = OxmlElement("w:tcMar")
cell_properties.append(margins)
for side, value in (("top", top), ("start", start), ("bottom", bottom), ("end", end)):
element = margins.find(qn(f"w:{side}"))
if element is None:
element = OxmlElement(f"w:{side}")
margins.append(element)
element.set(qn("w:w"), str(value))
element.set(qn("w:type"), "dxa")
def _cell_content_width_emu(cell: _Cell, *, horizontal_margin_twips: int) -> int:
"""从当前物理或合并单元格 tcW 计算扣除左右内边距后的可用宽度。"""
cell_width = cell._tc.get_or_add_tcPr().get_or_add_tcW().w
width_twips = int(cell_width) if cell_width is not None else 1
return int(Twips(max(1, width_twips - horizontal_margin_twips)))
def _paragraph_has_content(paragraph: Paragraph) -> bool:
"""判断段落是否包含 pPr 之外的 run、公式、链接或 drawing。"""
return any(child.tag != qn("w:pPr") for child in paragraph._p)
return _render_docx(middle_json, mode=mode, asset_resolver=asset_resolver)
__all__ = ["DocxRenderError", "render_docx"]
+7 -918
View File
@@ -1,633 +1,10 @@
# Copyright (c) Opendatalab. All rights reserved.
"""严格 MiddleJson 到 MinerU 风格 HTML 的公共渲染实现。"""
"""严格 MiddleJson 到 HTML 的轻量公共门面。"""
from __future__ import annotations
import html
import re
from functools import lru_cache
from importlib import resources
from urllib.parse import quote
from bs4 import BeautifulSoup
from loguru import logger
from mineru.render._internal.common.index import strip_index_page_tail
from mineru.render._internal.common.inline import inline_plain_text, parse_inline_content
from mineru.render._internal.common.list_items import ListItem, parse_list_item_marker, reference_list_needs_bullets
from mineru.render._internal.common.planner import PlannedBlock, build_render_plan
from mineru.render._internal.html.inline import (
HtmlInlineResult,
render_inline_content_html,
render_inline_nodes_html,
render_joined_inline_contents_html,
render_math_html,
)
from mineru.render._internal.html.sanitizer import (
is_supported_html_markup,
sanitize_html_fragment,
sanitize_image_source,
)
from mineru.render._internal.html.table import looks_like_gfm_table, render_gfm_table_html
from mineru.render.contracts import RenderMode
from mineru.types import (
PAGE_AUXILIARY_BLOCK_TYPES,
RAW_ALGORITHM,
BlockType,
ChartAnnotationBlock,
ChartBlock,
ChartBodyBlock,
CodeAnnotationBlock,
CodeBlock,
CodeBodyBlock,
DocTitleBlock,
EquationBlock,
ImageAnnotationBlock,
ImageBlock,
ImageBodyBlock,
IndexBlock,
ListBlock,
MiddleJson,
PageAuxTextBlock,
ParagraphTitleBlock,
RefTextBlock,
TableAnnotationBlock,
TableBlock,
TableBodyBlock,
TextBlock,
TitleBlockBase,
)
_STYLE_RESOURCE_NAME = "mineru.min.css"
_MATHJAX_URL = "https://cdn.jsdelivr.net/npm/mathjax@4.1.2/tex-chtml.js"
_MATHJAX_INTEGRITY = "sha384-zAhQQhdaMeHsMProNntGGg6nOUVcfuF9F22C3d1qJ9NZAVzCplXk1X85D2O5iufn"
_PRISM_CORE_URL = "https://cdn.jsdelivr.net/npm/prismjs@1.30.0/components/prism-core.min.js"
_PRISM_CORE_INTEGRITY = "sha384-zLRFO4dwowZvh8kzutOb5AWhH7f39HeJp+N7PtHF1SQtTBnifRx0AtmvTYs3F4YV"
_PRISM_AUTOLOADER_URL = "https://cdn.jsdelivr.net/npm/prismjs@1.30.0/plugins/autoloader/prism-autoloader.min.js"
_PRISM_AUTOLOADER_INTEGRITY = "sha384-Uq05+JLko69eOiPr39ta9bh7kld5PKZoU+fF7g0EXTAriEollhZ+DrN8Q/Oi8J2Q"
_PRISM_LANGUAGES_PATH = "https://cdn.jsdelivr.net/npm/prismjs@1.30.0/components/"
_MERMAID_URL = "https://cdn.jsdelivr.net/npm/mermaid@11.16.1/dist/mermaid.min.js"
_MERMAID_INTEGRITY = "sha384-aBQXj4hK6Jm05i7aQAsUV3bLdSUrHX1BGYfMB0166TtWt/RRaw+h0Eelme9OCOvy"
_MERMAID_MAX_TEXT_SIZE = 50_000
_MERMAID_FLOWCHART_HEADER_RE = re.compile(
r"(?:graph|flowchart)\s+(?:TB|TD|BT|RL|LR)\b",
re.IGNORECASE,
)
_SAFE_PRISM_LANGUAGE_RE = re.compile(r"[a-z0-9][a-z0-9-]{0,31}")
_BLOCKED_PRISM_LANGUAGE_NAMES = {"constructor", "prototype"}
_PRISM_LANGUAGE_ALIASES = {
"c#": "csharp",
"c++": "cpp",
"html": "markup",
"js": "javascript",
"py": "python",
"sh": "bash",
"shell": "bash",
"none": "",
"text": "",
"txt": "",
"ts": "typescript",
"xml": "markup",
"yml": "yaml",
}
class _HtmlRenderer:
"""维护一次 HTML 渲染所需的 anchor、公式与代码资源状态。"""
def __init__(
self,
middle_json: MiddleJson,
*,
mode: RenderMode,
asset_base_url: str,
standalone: bool,
document_title: str | None,
) -> None:
"""保存严格输入和调用选项,并预收集真实正文标题 anchor。"""
self.middle_json = middle_json
self.mode = mode
self.asset_base_url = asset_base_url
self.standalone = standalone
self.document_title = document_title
self.anchor_targets = _collect_title_anchor_ids(middle_json)
self.emitted_anchors: set[str] = set()
self.has_math = False
self.has_prism = False
self.has_mermaid = False
def render(self) -> str:
"""生成 fragment article,并按需包装完整 HTML 文档。"""
planned_pages = build_render_plan(self.middle_json, self.mode)
if self.mode is RenderMode.FULL:
body = self._render_full_pages(planned_pages)
else:
body = self._render_default_pages(planned_pages)
article = f'<article class="mineru-document mineru-document--{self.mode.value}">\n{body}\n</article>'
if not self.standalone:
return article
# 最终依赖以真实 DOM 为准,避免回退分支或普通文本中的 class 字样误触发。
article_soup = BeautifulSoup(article, "html.parser")
self.has_math = article_soup.select_one(".mineru-math") is not None
self.has_prism = any(
any(str(class_name).startswith("language-") for class_name in code.get("class", []))
for code in article_soup.find_all("code")
if code.get_text()
)
self.has_mermaid = article_soup.select_one(".mineru-flowchart") is not None
return self._render_standalone(article)
def _render_default_pages(self, planned_pages: list[list[PlannedBlock]]) -> str:
"""把 DEFAULT 计划展平为无页面边界的连续阅读内容。"""
rendered: list[str] = []
for page in planned_pages:
for planned in page:
block = self._render_planned_block(planned)
if block:
rendered.append(block)
return "\n".join(rendered)
def _render_full_pages(self, planned_pages: list[list[PlannedBlock]]) -> str:
"""为 FULL 计划保留所有页面 section 和相邻页面分隔。"""
sections: list[str] = []
for page in planned_pages:
page_idx = page[0].page_idx if page else None
if page_idx is None:
page_position = len(sections)
page_idx = self.middle_json.pages[page_position].page_idx
blocks = [rendered for planned in page if (rendered := self._render_planned_block(planned))]
rendered_blocks = "\n".join(blocks)
sections.append(f'<section class="mineru-page" data-page-idx="{page_idx}">\n{rendered_blocks}\n</section>')
return '\n<hr class="mineru-page-break" aria-hidden="true">\n'.join(sections)
def _render_planned_block(self, planned: PlannedBlock) -> str:
"""过滤计划块、分派类型 visitor,并添加稳定来源元数据。"""
if planned.removed:
return ""
block = planned.block
if self.mode is RenderMode.DEFAULT and block.type in PAGE_AUXILIARY_BLOCK_TYPES:
return ""
content = self._render_block_content(planned)
if not content or not content.strip():
return ""
attrs = [
'class="mineru-block"',
f'data-page-idx="{planned.page_idx}"',
f'data-block-type="{html.escape(str(block.type), quote=True)}"',
]
if block.index is not None:
attrs.append(f'data-block-index="{block.index}"')
return f"<div {' '.join(attrs)}>\n{content}\n</div>"
def _render_block_content(self, planned: PlannedBlock) -> str:
"""按严格 PageBlock 具体类型返回对应语义 HTML。"""
block = planned.block
if isinstance(block, TextBlock):
rendered = render_joined_inline_contents_html(planned.text_contents or [block.content])
self._observe_inline(rendered)
return f'<p class="mineru-text">{rendered.html}</p>' if rendered.html else ""
if isinstance(block, RefTextBlock):
rendered = render_joined_inline_contents_html(planned.text_contents or [block.content])
self._observe_inline(rendered)
return f'<p class="mineru-ref-text">{rendered.html}</p>' if rendered.html else ""
if isinstance(block, (DocTitleBlock, ParagraphTitleBlock)):
return self._render_title(block)
if isinstance(block, PageAuxTextBlock):
rendered = render_inline_content_html(block.content)
self._observe_inline(rendered)
class_name = html.escape(str(block.type).replace("_", "-"), quote=True)
return f'<div class="mineru-page-aux mineru-page-aux--{class_name}">{rendered.html}</div>' if rendered.html else ""
if isinstance(block, EquationBlock):
return self._render_equation(block)
if isinstance(block, ListBlock):
return self._render_list(block)
if isinstance(block, IndexBlock):
return self._render_index(block)
if isinstance(block, ImageBlock):
return self._render_image_block(block)
if isinstance(block, TableBlock):
return self._render_table_block(block)
if isinstance(block, ChartBlock):
return self._render_chart_block(block)
if isinstance(block, CodeBlock):
return self._render_code_block(block)
raise TypeError(f"Unsupported PageBlock type: {type(block).__name__}")
def _render_title(self, block: DocTitleBlock | ParagraphTitleBlock) -> str:
"""渲染标题,并保证正文 anchor 只在首次出现时生成 id。"""
rendered = render_inline_content_html(block.content)
self._observe_inline(rendered)
if not rendered.html:
return ""
level = min(max(block.level, 1), 6)
attrs = [f'class="mineru-heading mineru-heading--{level}"', f'data-heading-level="{block.level}"']
anchor = _anchor_key(block.anchor)
if anchor:
if anchor not in self.emitted_anchors:
anchor_id = self.anchor_targets.get(anchor)
if anchor_id:
attrs.append(f'id="{html.escape(anchor_id, quote=True)}"')
self.emitted_anchors.add(anchor)
else:
logger.warning("Duplicate HTML title anchor omitted: {!r}", anchor)
return f"<h{level} {' '.join(attrs)}>{rendered.html}</h{level}>"
def _render_equation(self, block: EquationBlock) -> str:
"""优先输出行间公式,空 LaTeX 时回退到安全图片来源。"""
if block.content.strip():
rendered = render_math_html(block.content, display=True)
self._observe_inline(rendered)
return rendered.html
source = self._safe_block_image_source(block)
return self._render_image(source, alt="formula", class_name="mineru-equation-image") if source else ""
def _render_list(self, block: ListBlock) -> str:
"""按直属 marker 类型渲染一层列表,并递归挂接嵌套列表。"""
parsed_leaves = [
parse_list_item_marker(child.content)
for child in block.content
if not isinstance(child, ListBlock) and child.content.strip()
]
add_reference_bullets = reference_list_needs_bullets(block)
container_tag, list_type, class_name = _classify_list(parsed_leaves, add_reference_bullets)
items: list[dict[str, object]] = []
expected_value: int | None = None
for child in block.content:
if isinstance(child, ListBlock):
nested = self._render_list(child)
if not nested:
continue
if not items:
items.append({"content": "", "attrs": ' class="mineru-list-item--orphan"', "nested": []})
nested_items = items[-1]["nested"]
assert isinstance(nested_items, list)
nested_items.append(nested)
continue
parsed = parse_list_item_marker(child.content)
item_content, marker = _list_item_content(
parsed,
add_reference_bullets,
explicit_markers=class_name == "mineru-list--explicit",
)
rendered = render_inline_content_html(item_content)
self._observe_inline(rendered)
if not rendered.html and not marker:
items.append({"content": "", "attrs": ' class="mineru-list-item--markerless"', "nested": []})
continue
attrs: list[str] = []
if class_name == "mineru-list--explicit":
attrs.append('class="mineru-list-item--explicit"')
if container_tag == "ol" and parsed.kind == "ordered" and parsed.value is not None:
if expected_value is None:
expected_value = parsed.value
if parsed.value != expected_value:
attrs.append(f'value="{parsed.value}"')
expected_value = parsed.value + 1
marker_html = (
f'<span class="mineru-list-marker">{html.escape(_replace_html_controls(marker), quote=False)}</span>'
if marker
else ""
)
if class_name == "mineru-list--explicit" and not marker_html:
marker_html = '<span class="mineru-list-marker"></span>'
items.append(
{
"content": f'{marker_html}<span class="mineru-list-content">{rendered.html}</span>',
"attrs": f" {' '.join(attrs)}" if attrs else "",
"nested": [],
}
)
rendered_items: list[str] = []
for item in items:
nested = item["nested"]
assert isinstance(nested, list)
item_content = str(item["content"])
if not item_content and not nested:
continue
rendered_items.append(f"<li{item['attrs']}>{item_content}{''.join(str(value) for value in nested)}</li>")
if not rendered_items:
return ""
container_attrs = [f'class="mineru-list {class_name}"']
if list_type:
container_attrs.append(f'type="{list_type}"')
if container_tag == "ol" and parsed_leaves and parsed_leaves[0].value not in (None, 1):
container_attrs.append(f'start="{parsed_leaves[0].value}"')
return f"<{container_tag} {' '.join(container_attrs)}>{''.join(rendered_items)}</{container_tag}>"
def _render_index(self, block: IndexBlock) -> str:
"""递归渲染目录,并仅链接到真实正文标题 anchor。"""
items: list[dict[str, object]] = []
for child in block.content:
if isinstance(child, IndexBlock):
nested = self._render_index_list(child)
if not nested:
continue
if not items:
items.append({"content": "", "orphan": True, "nested": []})
nested_items = items[-1]["nested"]
assert isinstance(nested_items, list)
nested_items.append(nested)
continue
rendered = self._render_index_leaf(child)
items.append({"content": rendered, "orphan": False, "nested": []})
inner = _serialize_index_items(items)
return f'<nav class="mineru-index" aria-label="Table of contents"><ul>{inner}</ul></nav>' if inner else ""
def _render_index_list(self, block: IndexBlock) -> str:
"""渲染一个嵌套 IndexBlock 为可挂接到父项的 ul。"""
items: list[dict[str, object]] = []
for child in block.content:
if isinstance(child, IndexBlock):
nested = self._render_index_list(child)
if nested:
if not items:
items.append({"content": "", "orphan": True, "nested": []})
nested_items = items[-1]["nested"]
assert isinstance(nested_items, list)
nested_items.append(nested)
continue
rendered = self._render_index_leaf(child)
items.append({"content": rendered, "orphan": False, "nested": []})
inner = _serialize_index_items(items)
return f"<ul>{inner}</ul>" if inner else ""
def _render_index_leaf(self, block: TextBlock | TitleBlockBase) -> str:
"""渲染目录叶子,并在 anchor 命中正文标题时生成内部链接。"""
content = strip_index_page_tail(block.content)
rendered = render_inline_content_html(content)
self._observe_inline(rendered)
if not rendered.html:
return ""
anchor = _anchor_key(block.anchor) if isinstance(block, TitleBlockBase) else ""
if anchor and (anchor_id := self.anchor_targets.get(anchor)):
href = quote(anchor_id, safe="-._~")
return f'<a href="#{href}">{rendered.html}</a>'
return rendered.html
def _render_image_block(self, block: ImageBlock) -> str:
"""按原始子块顺序渲染图片主体及重复说明。"""
parts: list[str] = []
for child in block.content:
if isinstance(child, ImageBodyBlock):
body = self._render_image_body(block, child)
elif isinstance(child, ImageAnnotationBlock):
body = self._render_annotation(child)
else:
raise TypeError(f"Unsupported image child: {type(child).__name__}")
if body:
parts.append(body)
return f'<figure class="mineru-figure mineru-figure--image">{"".join(parts)}</figure>' if parts else ""
def _render_image_body(self, parent: ImageBlock, block: ImageBodyBlock) -> str:
"""渲染图片,并把并存的识别内容放入原生 details。"""
if parent.sub_type == "flowchart":
mermaid_source = _extract_mermaid_flowchart_source(block.content)
if mermaid_source is not None:
return self._render_flowchart_body(block, mermaid_source)
source = self._safe_block_image_source(block)
content = self._render_embedded_content(block.content, linkify_text=parent.sub_type != "flowchart")
alt = _plain_content_text(block.content) or parent.sub_type or "image"
parts = [self._render_image(source, alt=alt, class_name="mineru-image")] if source else []
if content.html:
if source:
parts.append(self._render_details(content.html, parent.sub_type or "image content"))
else:
parts.append(content.html)
return "".join(parts)
def _render_flowchart_body(self, block: ImageBodyBlock, mermaid_source: str) -> str:
"""输出 Mermaid canvas,并保留 raster 与源码两级失败回退。"""
source = self._safe_block_image_source(block)
escaped_source = html.escape(_replace_html_controls(mermaid_source), quote=False)
fallback = self._render_image(source, alt="flowchart", class_name="mineru-flowchart-fallback") if source else ""
fallback_class = " mineru-flowchart--has-raster" if fallback else ""
details_open = "" if fallback else " open"
return (
f'<div class="mineru-flowchart{fallback_class}" data-mermaid-state="pending">'
'<div class="mineru-flowchart-canvas" role="img" aria-label="flowchart"></div>'
f"{fallback}</div>"
f'<details class="mineru-details mineru-flowchart-details"{details_open}>'
'<summary>flowchart source</summary>'
f'<pre class="mineru-flowchart-source"><code>{escaped_source}</code></pre>'
"</details>"
)
def _render_table_block(self, block: TableBlock) -> str:
"""按原始子块顺序渲染表格主体、标题与脚注。"""
parts: list[str] = []
for child in block.content:
if isinstance(child, TableBodyBlock):
body = self._render_table_body(child)
elif isinstance(child, TableAnnotationBlock):
body = self._render_annotation(child)
else:
raise TypeError(f"Unsupported table child: {type(child).__name__}")
if body:
parts.append(body)
return f'<figure class="mineru-figure mineru-figure--table">{"".join(parts)}</figure>' if parts else ""
def _render_table_body(self, block: TableBodyBlock) -> str:
"""优先保留安全 HTML table,空间文本和整体图片依次回退。"""
normalized_content = _replace_html_controls(block.content)
content = normalized_content.strip()
if content:
source_soup = BeautifulSoup(content, "html.parser")
if source_soup.find("table") is not None:
rendered = self._render_embedded_content(content)
if rendered.html and _contains_usable_table(rendered.html):
return rendered.html
source = self._safe_block_image_source(block)
if source:
return self._render_image(source, alt="table", class_name="mineru-table-image")
return _render_raw_fallback(content)
return f'<pre class="mineru-table-text">{html.escape(normalized_content, quote=False)}</pre>'
source = self._safe_block_image_source(block)
return self._render_image(source, alt="table", class_name="mineru-table-image") if source else ""
def _render_chart_block(self, block: ChartBlock) -> str:
"""按原始子块顺序渲染 chart 图片、结构内容及说明。"""
parts: list[str] = []
for child in block.content:
if isinstance(child, ChartBodyBlock):
body = self._render_chart_body(block, child)
elif isinstance(child, ChartAnnotationBlock):
body = self._render_annotation(child)
else:
raise TypeError(f"Unsupported chart child: {type(child).__name__}")
if body:
parts.append(body)
return f'<figure class="mineru-figure mineru-figure--chart">{"".join(parts)}</figure>' if parts else ""
def _render_chart_body(self, parent: ChartBlock, block: ChartBodyBlock) -> str:
"""渲染 chart 图片,并将并存结构内容放入 details。"""
source = self._safe_block_image_source(block)
content = self._render_chart_content(block.content)
parts = [self._render_image(source, alt=parent.sub_type or "chart", class_name="mineru-chart-image")] if source else []
if content.html:
if source:
parts.append(self._render_details(content.html, parent.sub_type or "chart content"))
else:
parts.append(content.html)
return "".join(parts)
def _render_chart_content(self, content: str) -> HtmlInlineResult:
"""按 HTML、严格 GFM 表格、普通行内内容的顺序渲染 chart content。"""
normalized = content.strip()
if not normalized:
return HtmlInlineResult("")
if is_supported_html_markup(normalized):
return self._render_embedded_content(normalized)
if looks_like_gfm_table(normalized):
rendered = render_gfm_table_html(normalized)
if rendered is None:
return HtmlInlineResult(_render_raw_fallback(content))
self._observe_inline(rendered)
return rendered
rendered = render_inline_content_html(content)
self._observe_inline(rendered)
return rendered
def _render_code_block(self, block: CodeBlock) -> str:
"""按原始子块顺序渲染代码或算法及其说明。"""
parts: list[str] = []
for child in block.content:
if isinstance(child, CodeBodyBlock):
body = self._render_code_body(block, child)
elif isinstance(child, CodeAnnotationBlock):
body = self._render_annotation(child)
else:
raise TypeError(f"Unsupported code child: {type(child).__name__}")
if body:
parts.append(body)
return f'<figure class="mineru-figure mineru-figure--code">{"".join(parts)}</figure>' if parts else ""
def _render_code_body(self, parent: CodeBlock, block: CodeBodyBlock) -> str:
"""代码使用 Prism class,算法使用共享 inline AST 保留空白。"""
if parent.sub_type == BlockType.CODE:
escaped = html.escape(_replace_html_controls(block.content), quote=False)
if not block.content:
return '<pre class="mineru-code"><code></code></pre>'
language = _normalize_prism_language(parent.guess_lang)
class_attr = f' class="language-{language}"' if language else ""
pre_class = f"mineru-code language-{language}" if language else "mineru-code"
self.has_prism = bool(language) or self.has_prism
return f'<pre class="{pre_class}"><code{class_attr}>{escaped}</code></pre>'
if parent.sub_type == RAW_ALGORITHM:
rendered = render_inline_nodes_html(
parse_inline_content(block.content),
linkify_text=False,
separate_adjacent_math=True,
preserve_newlines=True,
)
self._observe_inline(rendered)
return f'<div class="mineru-algorithm">{rendered.html}</div>' if rendered.html else ""
raise ValueError(f"Unsupported code subtype: {parent.sub_type}")
def _render_annotation(
self,
block: ImageAnnotationBlock | TableAnnotationBlock | ChartAnnotationBlock | CodeAnnotationBlock,
) -> str:
"""把视觉说明渲染为保持原类型的独立段落。"""
rendered = render_inline_content_html(block.content)
self._observe_inline(rendered)
if not rendered.html:
return ""
is_caption = str(block.type).endswith("caption")
role_class = "mineru-caption" if is_caption else "mineru-footnote"
type_class = html.escape(str(block.type).replace("_", "-"), quote=True)
return f'<p class="{role_class} {role_class}--{type_class}">{rendered.html}</p>'
def _render_embedded_content(self, content: str, *, linkify_text: bool = True) -> HtmlInlineResult:
"""安全处理 body 富 HTML;普通内容走共享 inline AST。"""
normalized = content.strip()
if not normalized:
return HtmlInlineResult("")
if not is_supported_html_markup(normalized):
rendered = render_inline_nodes_html(parse_inline_content(content), linkify_text=linkify_text)
self._observe_inline(rendered)
return rendered
sanitized = sanitize_html_fragment(normalized, asset_base_url=self.asset_base_url)
rendered = self._replace_safe_equations(sanitized)
self._observe_inline(rendered)
return rendered
def _replace_safe_equations(self, markup: str) -> HtmlInlineResult:
"""把 sanitizer 保留的 eq 标签替换为可信 MathJax carrier。"""
if not markup:
return HtmlInlineResult("")
soup = BeautifulSoup(markup, "html.parser")
has_math = False
for equation in list(soup.find_all("eq")):
rendered = render_math_html(equation.get_text(), display=False)
if not rendered.html:
equation.decompose()
continue
replacement = BeautifulSoup(rendered.html, "html.parser")
equation.replace_with(*list(replacement.contents))
has_math = True
return HtmlInlineResult(str(soup), has_math)
def _safe_block_image_source(self, block: EquationBlock | ImageBodyBlock | TableBodyBlock | ChartBodyBlock) -> str | None:
"""解析 block 图片来源,并应用 HTML renderer 的 URL 安全策略。"""
if block.image_path:
return sanitize_image_source(block.image_path, asset_base_url=self.asset_base_url)
if block.image_base64:
return sanitize_image_source(block.image_base64)
return None
def _render_image(self, source: str, *, alt: str, class_name: str) -> str:
"""构造属性已转义的安全 img 元素。"""
return (
f'<img class="{class_name}" src="{html.escape(source, quote=True)}" '
f'alt="{html.escape(_replace_html_controls(alt), quote=True)}">'
)
def _render_details(self, content: str, summary: str) -> str:
"""构造浏览器原生折叠详情,不引入额外脚本。"""
return (
'<details class="mineru-details">'
f"<summary>{html.escape(_replace_html_controls(summary), quote=False)}</summary>{content}</details>"
)
def _observe_inline(self, rendered: HtmlInlineResult) -> None:
"""累计当前文档是否需要加载 MathJax。"""
self.has_math = rendered.has_math or self.has_math
def _render_standalone(self, article: str) -> str:
"""用固定 CSS 和按需 CDN 资源包装完整 HTML 文档。"""
title = html.escape(
_replace_html_controls(_resolve_document_title(self.middle_json, self.document_title)),
quote=False,
)
styles = _load_html_styles()
dependencies = []
if self.has_math:
dependencies.append(_mathjax_head())
if self.has_prism:
dependencies.append(_prism_head())
if self.has_mermaid:
dependencies.append(_mermaid_head())
dependency_html = "\n".join(dependencies)
return (
"<!doctype html>\n"
'<html lang="und">\n'
"<head>\n"
'<meta charset="utf-8">\n'
'<meta name="viewport" content="width=device-width, initial-scale=1.0">\n'
f"<title>{title}</title>\n"
f"<style>{styles}</style>\n"
f"{dependency_html}\n"
"</head>\n"
'<body class="mineru-html-body">\n'
f"{article}\n"
"</body>\n"
"</html>"
)
from ..types import MiddleJson
from .contracts import RenderMode
def render_html(
@@ -638,304 +15,16 @@ def render_html(
standalone: bool = True,
document_title: str | None = None,
) -> str:
"""把严格 MiddleJson 无副作用地渲染为 HTML 文档或单根 fragment。"""
if not isinstance(middle_json, MiddleJson):
raise TypeError("render_html expects a MiddleJson instance")
if not isinstance(mode, RenderMode):
raise TypeError("mode must be a RenderMode value")
if not isinstance(asset_base_url, str):
raise TypeError("asset_base_url must be a string")
if not isinstance(standalone, bool):
raise TypeError("standalone must be a bool")
if document_title is not None and not isinstance(document_title, str):
raise TypeError("document_title must be a string or None")
return _HtmlRenderer(
"""惰性加载 HTML 实现并渲染严格 MiddleJson。"""
from ._internal.html.renderer import render_html as _render_html
return _render_html(
middle_json,
mode=mode,
asset_base_url=asset_base_url,
standalone=standalone,
document_title=document_title,
).render()
def _classify_list(items: list[ListItem], add_reference_bullets: bool) -> tuple[str, str | None, str]:
"""根据直属 marker 选择原生列表类型或显式 marker 模式。"""
if add_reference_bullets:
return "ul", None, "mineru-list--reference"
if items and all(item.kind == "unordered" for item in items):
return "ul", None, "mineru-list--unordered"
if items and all(item.kind == "ordered" for item in items):
styles = {item.ordered_style for item in items}
if len(styles) == 1:
style = next(iter(styles))
list_type = {
"lower-alpha": "a",
"upper-alpha": "A",
"lower-roman": "i",
"upper-roman": "I",
}.get(style or "")
return "ol", list_type, "mineru-list--ordered"
if items and all(item.kind == "none" for item in items):
return "ul", None, "mineru-list--unmarked"
return "ul", None, "mineru-list--explicit"
def _list_item_content(
item: ListItem,
add_reference_bullets: bool,
*,
explicit_markers: bool,
) -> tuple[str, str | None]:
"""决定一个列表项应剥离、保留还是显式显示源 marker。"""
if add_reference_bullets:
if item.kind == "unordered":
return item.body, None
original = f"{item.leading}{item.marker or ''}{item.separator}{item.body}"
return original, None
if explicit_markers:
return item.body, item.marker
if item.kind in {"unordered", "ordered"}:
return item.body, None
return item.body, item.marker
def _serialize_index_items(items: list[dict[str, object]]) -> str:
"""把目录项及其已挂接的嵌套 ul 序列化为 li。"""
rendered: list[str] = []
for item in items:
nested = item["nested"]
assert isinstance(nested, list)
content = str(item["content"])
if not content and not nested:
continue
class_attr = ' class="mineru-list-item--orphan"' if item["orphan"] else ""
rendered.append(f"<li{class_attr}>{content}{''.join(str(value) for value in nested)}</li>")
return "".join(rendered)
def _collect_title_anchor_ids(middle_json: MiddleJson) -> dict[str, str]:
"""为正文首次出现的非空 anchor 分配无空白且 document-wide 唯一的 HTML id。"""
anchor_ids: dict[str, str] = {}
used_ids: set[str] = set()
for page in middle_json.pages:
for block in page.blocks:
if not isinstance(block, TitleBlockBase):
continue
anchor = _anchor_key(block.anchor)
visible_title = inline_plain_text(parse_inline_content(block.content)).strip()
if not anchor or not visible_title or anchor in anchor_ids:
continue
base_id = _html_anchor_base(anchor)
candidate = base_id
suffix = 2
while candidate in used_ids:
candidate = f"{base_id}-{suffix}"
suffix += 1
anchor_ids[anchor] = candidate
used_ids.add(candidate)
return anchor_ids
def _plain_content_text(content: str) -> str:
"""从 body 内容提取安全 img alt 所需的可见纯文本。"""
if not content:
return ""
if is_supported_html_markup(content):
return BeautifulSoup(content, "html.parser").get_text(" ", strip=True)
return inline_plain_text(parse_inline_content(content)).strip()
def _extract_mermaid_flowchart_source(content: str) -> str | None:
"""提取受限 Mermaid flowchart fence,拒绝其他图类型和可改写配置的语法。"""
normalized = _replace_html_controls(content).replace("\r\n", "\n").replace("\r", "\n").strip()
if not normalized:
return None
lines = normalized.split("\n")
if len(lines) < 3:
return None
opening = re.fullmatch(r"(?P<fence>\x60{3,})[ \t]*mermaid[ \t]*", lines[0], re.IGNORECASE)
if opening is None:
return None
fence_length = len(opening.group("fence"))
if re.fullmatch(rf"\x60{{{fence_length},}}[ \t]*", lines[-1]) is None:
return None
source = "\n".join(lines[1:-1]).strip("\n")
if not source or len(source) > _MERMAID_MAX_TEXT_SIZE:
return None
if source.lstrip().startswith("---") or "%%{" in source:
return None
first_statement = next(
(line.strip() for line in source.splitlines() if line.strip() and not line.lstrip().startswith("%%")),
"",
)
if _MERMAID_FLOWCHART_HEADER_RE.match(first_statement) is None:
return None
return source
def _contains_usable_table(markup: str) -> bool:
"""确认清洗后的 markup 至少保留一个含单元格的 table。"""
soup = BeautifulSoup(markup, "html.parser")
return any(table.find(("td", "th")) is not None for table in soup.find_all("table"))
def _render_raw_fallback(content: str) -> str:
"""把无法安全结构化的原内容转义为可见 pre,避免静默丢失。"""
return f'<pre class="mineru-raw-fallback">{html.escape(_replace_html_controls(content), quote=False)}</pre>'
def _normalize_prism_language(language: str | None) -> str | None:
"""把模型语言名归一化为不可构造路径穿越的 Prism component 名。"""
normalized = (language or "").strip().lower()
normalized = _PRISM_LANGUAGE_ALIASES.get(normalized, normalized.replace("_", "-"))
if not normalized or normalized in _BLOCKED_PRISM_LANGUAGE_NAMES:
return None
return normalized if _SAFE_PRISM_LANGUAGE_RE.fullmatch(normalized) else None
def _resolve_document_title(middle_json: MiddleJson, explicit_title: str | None) -> str:
"""按显式参数、首个正文文档标题、固定回退值解析 HTML title。"""
if explicit_title and explicit_title.strip():
return explicit_title.strip()
for page in middle_json.pages:
for block in page.blocks:
if isinstance(block, DocTitleBlock):
title = inline_plain_text(parse_inline_content(block.content)).strip()
if title:
return title
return "MinerU Document"
def _anchor_key(anchor: str | None) -> str:
"""保留 producer anchor 身份,仅去除首尾空白供标题与目录精确匹配。"""
return (anchor or "").strip()
def _html_anchor_base(anchor: str) -> str:
"""把原始 anchor 转成无空白的 HTML id 基值,碰撞由 document registry 处理。"""
return re.sub(r"\s+", "-", _replace_html_controls(anchor))
def _replace_html_controls(content: str) -> str:
"""替换 HTML 不允许的 C0 控制字符与不可编码的孤立 surrogate。"""
return re.sub(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f\ud800-\udfff]", "\ufffd", content)
@lru_cache(maxsize=1)
def _load_html_styles() -> str:
"""从包资源读取并缓存压缩后的 MinerU 独立样式。"""
root = resources.files("mineru").joinpath("resources", "html")
return root.joinpath(_STYLE_RESOURCE_NAME).read_text(encoding="utf-8")
def _mathjax_head() -> str:
"""返回固定、收紧公式输入能力的 MathJax 4.1.2 head 片段。"""
return f"""<script>
window.MathJax = {{
loader: {{load: ['ui/safe']}},
tex: {{
inlineMath: [['\\\\(', '\\\\)']],
displayMath: [['\\\\[', '\\\\]']],
processEscapes: true,
processEnvironments: false,
processRefs: false,
packages: {{'[-]': ['require']}}
}},
options: {{
ignoreHtmlClass: 'mineru-document',
processHtmlClass: 'mineru-math',
enableMenu: false,
enableEnrichment: false,
safeOptions: {{allow: {{URLs: 'none', classes: 'none', cssIDs: 'none', styles: 'none'}}}}
}},
startup: {{
ready: function () {{
window.MathJax.startup.defaultReady();
var options = window.MathJax.startup.document.options;
options.enableMenu = false;
options.enableEnrichment = false;
}}
}}
}};
</script>
<script id="MathJax-script" defer src="{_MATHJAX_URL}"
integrity="{_MATHJAX_INTEGRITY}" crossorigin="anonymous" referrerpolicy="no-referrer"></script>"""
def _prism_head() -> str:
"""返回固定 Prism core、Autoloader 和当前文档局部高亮脚本。"""
return f"""<script defer data-manual src="{_PRISM_CORE_URL}"
integrity="{_PRISM_CORE_INTEGRITY}" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
<script defer src="{_PRISM_AUTOLOADER_URL}"
integrity="{_PRISM_AUTOLOADER_INTEGRITY}" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
<script>
document.addEventListener('DOMContentLoaded', function () {{
if (!window.Prism || !Prism.plugins || !Prism.plugins.autoloader) return;
Prism.plugins.autoloader.languages_path = '{_PRISM_LANGUAGES_PATH}';
var root = document.querySelector('.mineru-document');
if (root) Prism.highlightAllUnder(root);
}});
</script>"""
def _mermaid_head() -> str:
"""返回固定 Mermaid 入口和逐图安全渲染、失败回退脚本。"""
return f"""<script defer src="{_MERMAID_URL}"
integrity="{_MERMAID_INTEGRITY}" crossorigin="anonymous" referrerpolicy="no-referrer"></script>
<script>
function markMineruMermaidError(host) {{
host.dataset.mermaidState = 'error';
if (!host.classList.contains('mineru-flowchart--has-raster')) {{
var details = host.nextElementSibling;
if (details && details.classList.contains('mineru-flowchart-details')) details.open = true;
}}
}}
document.addEventListener('DOMContentLoaded', function () {{
var root = document.querySelector('.mineru-document');
var hosts = root ? Array.from(root.querySelectorAll('.mineru-flowchart')) : [];
if (!hosts.length) return;
if (!window.mermaid) {{
hosts.forEach(markMineruMermaidError);
return;
}}
try {{
window.mermaid.initialize({{
startOnLoad: false,
securityLevel: 'strict',
suppressErrorRendering: true,
theme: 'neutral',
maxTextSize: 50000,
maxEdges: 500,
flowchart: {{htmlLabels: false, useMaxWidth: true}}
}});
}} catch (_error) {{
hosts.forEach(markMineruMermaidError);
return;
}}
(async function () {{
for (var index = 0; index < hosts.length; index += 1) {{
var host = hosts[index];
var details = host.nextElementSibling;
var source = details && details.querySelector('.mineru-flowchart-source code');
var canvas = host.querySelector('.mineru-flowchart-canvas');
if (!source || !canvas) {{
markMineruMermaidError(host);
continue;
}}
var renderId = 'mineru-mermaid-' + index;
while (document.getElementById(renderId)) renderId += '-x';
host.dataset.mermaidState = 'rendering';
try {{
var result = await window.mermaid.render(renderId, source.textContent);
canvas.innerHTML = result.svg;
host.dataset.mermaidState = 'rendered';
}} catch (_error) {{
markMineruMermaidError(host);
}}
}}
}})();
}});
</script>"""
__all__ = ["render_html"]
+10 -53
View File
@@ -1,16 +1,11 @@
# Copyright (c) Opendatalab. All rights reserved.
"""严格 MiddleJson 到 Markdown 的公共渲染实现。"""
"""严格 MiddleJson 到 Markdown 的轻量公共门面。"""
from __future__ import annotations
from ..config import LatexDelimitersConfig, config
from ..types import PAGE_AUXILIARY_BLOCK_TYPES, MiddleJson
from ._internal.common.planner import PlannedBlock, build_render_plan
from ._internal.markdown.blocks import render_planned_block
from ..types import MiddleJson
from .contracts import ImageRenderer, RenderMode
_PAGE_SEPARATOR = "\n\n---\n\n"
def render_markdown(
middle_json: MiddleJson,
@@ -19,53 +14,15 @@ def render_markdown(
asset_base_url: str = "",
image_renderer: ImageRenderer | None = None,
) -> str:
"""把严格 MiddleJson 纯函数式渲染为 Markdown 字符串。"""
if not isinstance(middle_json, MiddleJson):
raise TypeError("render_markdown expects a MiddleJson instance")
if not isinstance(mode, RenderMode):
raise TypeError("mode must be a RenderMode value")
"""惰性加载 Markdown 实现并渲染严格 MiddleJson。"""
from ._internal.markdown.renderer import render_markdown as _render_markdown
delimiters = config.render.latex_delimiters
planned_pages = build_render_plan(middle_json, mode)
rendered_pages = [
_render_page(
page,
mode=mode,
delimiters=delimiters,
asset_base_url=asset_base_url,
image_renderer=image_renderer,
)
for page in planned_pages
]
if mode is RenderMode.FULL:
return _PAGE_SEPARATOR.join(rendered_pages)
return "\n\n".join(page for page in rendered_pages if page)
def _render_page(
planned_blocks: list[PlannedBlock],
*,
mode: RenderMode,
delimiters: LatexDelimitersConfig,
asset_base_url: str,
image_renderer: ImageRenderer | None = None,
) -> str:
"""渲染单页逻辑块,并在默认模式中过滤重复页元素。"""
rendered: list[str] = []
for planned in planned_blocks:
if planned.removed:
continue
if mode is RenderMode.DEFAULT and planned.block.type in PAGE_AUXILIARY_BLOCK_TYPES:
continue
text = render_planned_block(
planned,
delimiters=delimiters,
asset_base_url=asset_base_url,
image_renderer=image_renderer,
)
if text and text.strip():
rendered.append(text.strip("\n"))
return "\n\n".join(rendered)
return _render_markdown(
middle_json,
mode=mode,
asset_base_url=asset_base_url,
image_renderer=image_renderer,
)
__all__ = ["render_markdown"]
+6 -165
View File
@@ -1,56 +1,11 @@
# Copyright (c) Opendatalab. All rights reserved.
"""严格 MiddleJson 到树形 Markdown structured_content 的公共渲染实现。"""
"""严格 MiddleJson 到 Structured Content 的轻量公共门面。"""
from __future__ import annotations
from typing import Any, TypeAlias
from typing import Any
from mineru.config import LatexDelimitersConfig, config
from mineru.render._internal.markdown.blocks import (
render_single_block,
render_title_inline_content,
render_visual_annotation,
render_visual_body_content,
)
from mineru.render._internal.markdown.assets import normalize_image_source, resolve_image_source
from mineru.render._internal.markdown.escaping import escape_standalone_marker_rule, escape_text_block_markdown_prefix
from mineru.types import (
BlockType,
ChartAnnotationBlock,
ChartBlock,
ChartBodyBlock,
CodeAnnotationBlock,
CodeBlock,
DocTitleBlock,
EquationBlock,
ImageAnnotationBlock,
ImageBlock,
ImageBodyBlock,
ImagePayloadBlock,
MiddleJson,
PageBlock,
ParagraphTitleBlock,
TableAnnotationBlock,
TableBlock,
TableBodyBlock,
)
VisualBlock: TypeAlias = ImageBlock | TableBlock | ChartBlock | CodeBlock
VisualAnnotationBlock: TypeAlias = ImageAnnotationBlock | TableAnnotationBlock | ChartAnnotationBlock | CodeAnnotationBlock
_CAPTION_TYPES = {
BlockType.IMAGE_CAPTION,
BlockType.TABLE_CAPTION,
BlockType.CHART_CAPTION,
BlockType.CODE_CAPTION,
}
_FOOTNOTE_TYPES = {
BlockType.IMAGE_FOOTNOTE,
BlockType.TABLE_FOOTNOTE,
BlockType.CHART_FOOTNOTE,
BlockType.CODE_FOOTNOTE,
}
_REMOVED_BLOCK_FIELDS = {"content", "index", "guess_lang", "image_path", "image_base64"}
from ..types import MiddleJson
def render_structured_content(
@@ -58,124 +13,10 @@ def render_structured_content(
*,
asset_base_url: str = "",
) -> dict[str, Any]:
"""把严格 MiddleJson 无副作用地渲染为树形 Markdown structured_content。"""
if not isinstance(middle_json, MiddleJson):
raise TypeError("render_structured_content expects a MiddleJson instance")
"""惰性加载 Structured Content 实现并渲染严格 MiddleJson。"""
from ._internal.structured_content.renderer import render_structured_content as _render_structured_content
delimiters = config.render.latex_delimiters
document_fields = middle_json.model_dump(
mode="json",
exclude={"pages"},
exclude_defaults=True,
)
pages = [
{
"page_idx": page.page_idx,
"blocks": [
_render_content_block(
block,
delimiters=delimiters,
asset_base_url=asset_base_url,
)
for block in page.blocks
],
}
for page in middle_json.pages
]
return {"pages": pages, **document_fields}
def _render_content_block(
block: PageBlock,
*,
delimiters: LatexDelimitersConfig,
asset_base_url: str,
) -> dict[str, Any]:
"""保留父块元数据,并把 block 内容收敛为 Markdown 字符串。"""
payload = block.model_dump(
mode="json",
exclude=_REMOVED_BLOCK_FIELDS,
exclude_defaults=True,
)
if isinstance(block, (DocTitleBlock, ParagraphTitleBlock)):
content = render_title_inline_content(block, delimiters)
payload["content"] = escape_standalone_marker_rule(escape_text_block_markdown_prefix(content))
return payload
if isinstance(block, EquationBlock):
payload["content"] = block.content.strip()
image_source = _resolve_content_image_source(block, asset_base_url)
if image_source is not None:
payload["image_source"] = image_source
return payload
if not isinstance(block, (ImageBlock, TableBlock, ChartBlock, CodeBlock)):
payload["content"] = render_single_block(
block,
delimiters=delimiters,
asset_base_url=asset_base_url,
)
return payload
payload["content"] = render_visual_body_content(
block,
delimiters=delimiters,
asset_base_url=asset_base_url,
)
payload["captions"] = _render_annotation_group(block, _CAPTION_TYPES, delimiters)
payload["footnotes"] = _render_annotation_group(block, _FOOTNOTE_TYPES, delimiters)
image_source = _resolve_visual_image_source(block, asset_base_url)
if image_source is not None:
payload["image_source"] = image_source
return payload
def _render_annotation_group(
block: VisualBlock,
accepted_types: set[str],
delimiters: LatexDelimitersConfig,
) -> list[dict[str, Any]]:
"""按源 index 稳定排序视觉说明,并保留可用 bbox 与 Markdown 内容。"""
annotations: list[tuple[int, VisualAnnotationBlock]] = []
for position, child in enumerate(block.content):
if (
isinstance(
child,
(ImageAnnotationBlock, TableAnnotationBlock, ChartAnnotationBlock, CodeAnnotationBlock),
)
and child.type in accepted_types
):
annotations.append((position, child))
annotations.sort(key=_annotation_sort_key)
rendered_annotations: list[dict[str, Any]] = []
for _, child in annotations:
annotation_payload: dict[str, Any] = {}
if child.bbox is not None:
annotation_payload["bbox"] = list(child.bbox)
annotation_payload["content"] = render_visual_annotation(child, delimiters)
rendered_annotations.append(annotation_payload)
return rendered_annotations
def _annotation_sort_key(
item: tuple[int, VisualAnnotationBlock],
) -> tuple[bool, int, int]:
"""让有 index 的说明升序优先,缺失 index 的说明稳定排在末尾。"""
position, block = item
return block.index is None, block.index if block.index is not None else 0, position
def _resolve_visual_image_source(block: VisualBlock, asset_base_url: str) -> str | None:
"""解析视觉 body 实际选择的图片来源,并返回安全的 Markdown 地址。"""
for child in block.content:
if not isinstance(child, (ImageBodyBlock, TableBodyBlock, ChartBodyBlock)):
continue
return _resolve_content_image_source(child, asset_base_url)
return None
def _resolve_content_image_source(block: ImagePayloadBlock, asset_base_url: str) -> str | None:
"""把图片载荷收敛为 structured_content 中唯一且安全的 image_source。"""
source = resolve_image_source(block, asset_base_url)
return normalize_image_source(source) if source else None
return _render_structured_content(middle_json, asset_base_url=asset_base_url)
__all__ = ["render_structured_content"]
-63
View File
@@ -1,63 +0,0 @@
# Copyright (c) Opendatalab. All rights reserved.
import os
from abc import ABC, abstractmethod
class DataWriter(ABC):
@abstractmethod
def write(self, path: str, data: bytes) -> None:
"""Write the data to the file.
Args:
path (str): the target file where to write
data (bytes): the data want to write
"""
pass
def write_string(self, path: str, data: str) -> None:
"""Write the data to file, the data will be encoded to bytes.
Args:
path (str): the target file where to write
data (str): the data want to write
"""
def safe_encode(data: str, method: str):
try:
bit_data = data.encode(encoding=method, errors="replace")
return bit_data, True
except: # noqa
return None, False
for method in ["utf-8", "ascii"]:
bit_data, flag = safe_encode(data, method)
if flag:
self.write(path, bit_data)
break
class FileBasedDataWriter(DataWriter):
def __init__(self, parent_dir: str = "") -> None:
"""Initialized with parent_dir.
Args:
parent_dir (str, optional): the parent directory that may be used within methods. Defaults to ''.
"""
self._parent_dir = parent_dir
def write(self, path: str, data: bytes) -> None:
"""Write file with data.
Args:
path (str): the path of file, if the path is relative path, it will be joined with parent_dir.
data (bytes): the data want to write
"""
fn_path = path
if not os.path.isabs(fn_path) and len(self._parent_dir) > 0:
fn_path = os.path.join(self._parent_dir, path)
if not os.path.exists(os.path.dirname(fn_path)) and os.path.dirname(fn_path) != "":
os.makedirs(os.path.dirname(fn_path), exist_ok=True)
with open(fn_path, "wb") as f:
f.write(data)