diff --git a/mineru/utils/pdf_text_tool.py b/mineru/utils/pdf_text_tool.py index fb659e14..ca5de3d7 100644 --- a/mineru/utils/pdf_text_tool.py +++ b/mineru/utils/pdf_text_tool.py @@ -2,12 +2,19 @@ import math from typing import Any, List +import numpy as np import pypdfium2 as pdfium from pdftext.pdf.chars import deduplicate_chars, get_chars from pdftext.pdf.pages import assign_scripts, get_blocks, get_lines, get_spans +from pdftext.schema import Bbox from mineru.utils.pdfium_guard import close_pdfium_child, pdfium_guard +try: + from pdftext.pdf.chars import PageChars +except ImportError: + PageChars = None + NEAR_IDENTICAL_CHAR_BBOX_TOLERANCE = 1.0 @@ -87,6 +94,115 @@ def _iter_neighbor_bbox_bucket_keys( yield bucket_x + offset_x, bucket_y + offset_y +def _is_pdftext_page_chars(chars: Any) -> bool: + """判断对象是否为 pdftext 0.7 引入的 PageChars 列式字符容器。""" + return PageChars is not None and isinstance(chars, PageChars) + + +def _materialize_page_chars(chars) -> list[dict[str, Any]]: + """将 pdftext 0.7 的 PageChars 物化为 MinerU 既有 char dict 列表。""" + boxes = chars.boxes.tolist() + rotations = chars.rotations.tolist() + font_ids = chars.font_ids.tolist() + char_indices = chars.char_indices.tolist() + + return [ + { + "bbox": Bbox([float(coord) for coord in boxes[index]]), + "char": chars.text[index], + "rotation": float(rotations[index]), + "font": chars.fonts[int(font_ids[index])], + "char_idx": int(char_indices[index]), + } + for index in range(len(chars)) + ] + + +def _ensure_legacy_chars(chars) -> list[dict[str, Any]]: + """统一输出旧版 char dict 列表,隔离 pdftext 0.7 的返回结构变化。""" + if _is_pdftext_page_chars(chars): + return _materialize_page_chars(chars) + return chars + + +def _get_single_char_text(char: dict[str, Any]) -> str: + """提取单个 PDF 字符文本,异常空值用替换符保证 PageChars 长度一致。""" + text = char.get("char", "") + if len(text) == 1: + return text + return text[:1] or "\uFFFD" + + +def _get_char_font_id( + char: dict[str, Any], + fonts: list[dict[str, Any]], + font_cache: dict[tuple[Any, Any, Any, Any], int], +) -> int: + """为旧版字符 font 生成 PageChars 需要的页内 font id。""" + font = char.get("font") or {} + font_key = ( + font.get("name"), + font.get("flags"), + font.get("size"), + font.get("weight"), + ) + font_id = font_cache.get(font_key) + if font_id is None: + font_id = len(fonts) + font_cache[font_key] = font_id + fonts.append( + { + "name": font.get("name"), + "flags": font.get("flags"), + "size": font.get("size"), + "weight": font.get("weight"), + } + ) + return font_id + + +def _get_char_index(char: dict[str, Any], fallback_idx: int) -> int: + """提取旧版字符索引,缺失或为空时回退到当前列表位置。""" + char_idx = char.get("char_idx") + if char_idx is None: + char_idx = fallback_idx + return int(char_idx) + + +def _legacy_chars_to_page_chars(chars): + """将旧版 char dict 列表打包回 pdftext 0.7 get_spans 所需的 PageChars。""" + if PageChars is None or _is_pdftext_page_chars(chars): + return chars + + fonts = [] + font_cache = {} + text_parts = [] + codes = [] + rotations = [] + boxes = [] + font_ids = [] + char_indices = [] + + for fallback_idx, char in enumerate(chars): + char_text = _get_single_char_text(char) + text_parts.append(char_text) + codes.append(ord(char_text)) + rotations.append(float(char.get("rotation") or 0.0)) + boxes.append(_get_char_bbox_coords(char)) + font_ids.append(_get_char_font_id(char, fonts, font_cache)) + char_indices.append(_get_char_index(char, fallback_idx)) + + return PageChars( + "".join(text_parts), + np.array(codes, dtype=np.uint32), + np.array(rotations, dtype=np.float64), + np.array(boxes, dtype=np.float64).reshape((len(boxes), 4)), + np.array(font_ids, dtype=np.int32), + fonts, + np.array(char_indices, dtype=np.int64), + ) + + def _deduplicate_near_identical_chars( chars: list[dict[str, Any]], ) -> list[dict[str, Any]]: @@ -149,6 +265,7 @@ def get_page_chars( chars = deduplicate_chars( get_chars(textpage, page_bbox, page_rotation, quote_loosebox) ) + chars = _ensure_legacy_chars(chars) chars = _deduplicate_near_identical_chars(chars) finally: if owns_textpage: @@ -170,6 +287,7 @@ def get_lines_from_chars( line_distance_threshold: float = 0.1, ): """从已提取的字符构建 pdftext lines,避免重复读取 PDFium textpage。""" + chars = _legacy_chars_to_page_chars(chars) spans = get_spans( chars, superscript_height_threshold=superscript_height_threshold, diff --git a/pyproject.toml b/pyproject.toml index 54a8d108..27a9e565 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,7 +41,7 @@ dependencies = [ "pypdfium2>=4.30.0", "pypdf>=5.6.0", "reportlab", - "pdftext>=0.6.3", + "pdftext>=0.6.3,<0.8.0", "modelscope>=1.26.0", "huggingface-hub>=0.32.4", "json-repair>=0.46.2",