Merge pull request #5218 from myhloli/dev

feat: integrate support for pdftext 0.7 PageChars and legacy char handling

mineru/utils/pdf_text_tool.py

@@ -2,12 +2,19 @@
 import math
 from typing import Any, List
 
+import numpy as np
 import pypdfium2 as pdfium
 from pdftext.pdf.chars import deduplicate_chars, get_chars
 from pdftext.pdf.pages import assign_scripts, get_blocks, get_lines, get_spans
+from pdftext.schema import Bbox
 
 from mineru.utils.pdfium_guard import close_pdfium_child, pdfium_guard
 
+try:
+    from pdftext.pdf.chars import PageChars
+except ImportError:
+    PageChars = None
+
 NEAR_IDENTICAL_CHAR_BBOX_TOLERANCE = 1.0
 
 
@@ -87,6 +94,115 @@ def _iter_neighbor_bbox_bucket_keys(
             yield bucket_x + offset_x, bucket_y + offset_y
 
 
+def _is_pdftext_page_chars(chars: Any) -> bool:
+    """判断对象是否为 pdftext 0.7 引入的 PageChars 列式字符容器。"""
+    return PageChars is not None and isinstance(chars, PageChars)
+
+
+def _materialize_page_chars(chars) -> list[dict[str, Any]]:
+    """将 pdftext 0.7 的 PageChars 物化为 MinerU 既有 char dict 列表。"""
+    boxes = chars.boxes.tolist()
+    rotations = chars.rotations.tolist()
+    font_ids = chars.font_ids.tolist()
+    char_indices = chars.char_indices.tolist()
+
+    return [
+        {
+            "bbox": Bbox([float(coord) for coord in boxes[index]]),
+            "char": chars.text[index],
+            "rotation": float(rotations[index]),
+            "font": chars.fonts[int(font_ids[index])],
+            "char_idx": int(char_indices[index]),
+        }
+        for index in range(len(chars))
+    ]
+
+
+def _ensure_legacy_chars(chars) -> list[dict[str, Any]]:
+    """统一输出旧版 char dict 列表，隔离 pdftext 0.7 的返回结构变化。"""
+    if _is_pdftext_page_chars(chars):
+        return _materialize_page_chars(chars)
+    return chars
+
+
+def _get_single_char_text(char: dict[str, Any]) -> str:
+    """提取单个 PDF 字符文本，异常空值用替换符保证 PageChars 长度一致。"""
+    text = char.get("char", "")
+    if len(text) == 1:
+        return text
+    return text[:1] or "\uFFFD"
+
+
+def _get_char_font_id(
+    char: dict[str, Any],
+    fonts: list[dict[str, Any]],
+    font_cache: dict[tuple[Any, Any, Any, Any], int],
+) -> int:
+    """为旧版字符 font 生成 PageChars 需要的页内 font id。"""
+    font = char.get("font") or {}
+    font_key = (
+        font.get("name"),
+        font.get("flags"),
+        font.get("size"),
+        font.get("weight"),
+    )
+    font_id = font_cache.get(font_key)
+    if font_id is None:
+        font_id = len(fonts)
+        font_cache[font_key] = font_id
+        fonts.append(
+            {
+                "name": font.get("name"),
+                "flags": font.get("flags"),
+                "size": font.get("size"),
+                "weight": font.get("weight"),
+            }
+        )
+    return font_id
+
+
+def _get_char_index(char: dict[str, Any], fallback_idx: int) -> int:
+    """提取旧版字符索引，缺失或为空时回退到当前列表位置。"""
+    char_idx = char.get("char_idx")
+    if char_idx is None:
+        char_idx = fallback_idx
+    return int(char_idx)
+
+
+def _legacy_chars_to_page_chars(chars):
+    """将旧版 char dict 列表打包回 pdftext 0.7 get_spans 所需的 PageChars。"""
+    if PageChars is None or _is_pdftext_page_chars(chars):
+        return chars
+
+    fonts = []
+    font_cache = {}
+    text_parts = []
+    codes = []
+    rotations = []
+    boxes = []
+    font_ids = []
+    char_indices = []
+
+    for fallback_idx, char in enumerate(chars):
+        char_text = _get_single_char_text(char)
+        text_parts.append(char_text)
+        codes.append(ord(char_text))
+        rotations.append(float(char.get("rotation") or 0.0))
+        boxes.append(_get_char_bbox_coords(char))
+        font_ids.append(_get_char_font_id(char, fonts, font_cache))
+        char_indices.append(_get_char_index(char, fallback_idx))
+
+    return PageChars(
+        "".join(text_parts),
+        np.array(codes, dtype=np.uint32),
+        np.array(rotations, dtype=np.float64),
+        np.array(boxes, dtype=np.float64).reshape((len(boxes), 4)),
+        np.array(font_ids, dtype=np.int32),
+        fonts,
+        np.array(char_indices, dtype=np.int64),
+    )
+
+
 def _deduplicate_near_identical_chars(
     chars: list[dict[str, Any]],
 ) -> list[dict[str, Any]]:
@@ -149,6 +265,7 @@ def get_page_chars(
             chars = deduplicate_chars(
                 get_chars(textpage, page_bbox, page_rotation, quote_loosebox)
             )
+            chars = _ensure_legacy_chars(chars)
             chars = _deduplicate_near_identical_chars(chars)
     finally:
         if owns_textpage:
@@ -170,6 +287,7 @@ def get_lines_from_chars(
     line_distance_threshold: float = 0.1,
 ):
     """从已提取的字符构建 pdftext lines，避免重复读取 PDFium textpage。"""
+    chars = _legacy_chars_to_page_chars(chars)
     spans = get_spans(
         chars,
         superscript_height_threshold=superscript_height_threshold,

pyproject.toml

@@ -41,7 +41,7 @@ dependencies = [
     "pypdfium2>=4.30.0",
     "pypdf>=5.6.0",
     "reportlab",
-    "pdftext>=0.6.3",
+    "pdftext>=0.6.3,<0.8.0",
     "modelscope>=1.26.0",
     "huggingface-hub>=0.32.4",
     "json-repair>=0.46.2",