opensource/MinerU
3
0
Fork 0
mirror of https://github.com/opendatalab/MinerU.git synced 2026-08-30 17:12:39 +08:00
Code Issues Packages Projects Releases Wiki Activity

test: guard refactored package boundaries

Browse Source
This commit is contained in:
myhloli
2026-08-25 23:31:14 +08:00
parent 0bd6c38dcc
commit d1a43ec611
56 changed files with 391 additions and 245 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CLAUDE.md
+10 -3
View File
@@ -42,11 +42,11 @@ mineru 模块内部子模块之间的引用统一使用 **relative import**，
```python
# 正确 — relative import
from .base import DocumentParser
from ..utils.enum_class import MakeMode
from ..render import RenderMode
# 错误 — 项目内不允许 absolute import 引用自身模块
from mineru.api.base import DocumentParser
from mineru.utils.enum_class import MakeMode
from mineru.render import RenderMode
```
只引用外部第三方库时使用 absolute import（如 `from loguru import logger`）。
@@ -199,7 +199,14 @@ pr-5415 重构后，Middle JSON 已收敛为 schema 2.0 的统一结构，不再
- `RenderFormat.DOCX` → `render_docx`
- `RenderFormat.STRUCTURED_CONTENT` → `render_structured_content`
`render/_internal/` 下按目标格式分目录组织共享逻辑（`common/`/`markdown/`/`html/`/`docx/`）。不再有 `pipeline_union_make`/`vlm_union_make`/`office_union_make` 三套逻辑，`content_list` 格式和 `render_content_list` 函数已删除。
`render/_internal/` 下按目标格式分目录组织共享逻辑（`common/`/`markdown/`/`html/`/`docx/`/`structured_content/`），顶层同名模块只是惰性公共门面。行内语义解析归 `backend/postprocess/inline.py`，renderer 只能单向依赖该模块和 `backend/postprocess/table_merge`。不再有 `pipeline_union_make`/`vlm_union_make`/`office_union_make` 三套逻辑，`content_list` 格式和 `render_content_list` 函数已删除。
### 5.1 目录职责
- `model/runtime/` 负责设备、显存、ONNX 与 Hybrid 本地模型生命周期；模型仓库和下载分别位于 `model/registry.py`、`model/download.py`。
- `model/flash/pdf/` 负责 PDFDocument、PDFium、原生文本、样式和表格恢复；`model/flash/office/` 负责六类 Office 格式。
- `utils/` 只保留 geometry、image、image payload、language/text、platform 和 stdio 等叶子能力；活动代码不得把业务实现重新放入 utils。
- 稳定依赖方向为 `utils/types → model → backend → render → parser/kit/doclib`，禁止反向引用。
### 6. ParseResult 与 MiddleJson 的关系
tests/fixtures/evaluate_native_pdf_table_manifest.py
Vendored
+3 -3
View File
@@ -12,14 +12,14 @@ from collections import defaultdict
from pathlib import Path, PureWindowsPath
from typing import Any
from mineru.utils.native_pdf_table import (
from mineru.model.flash.pdf.table_recovery import (
NativeTableInput,
coerce_native_table_rectangles,
coerce_native_table_rules,
recover_native_pdf_table,
)
from mineru.utils.native_pdf_table.engine import diagnose_native_pdf_table
from mineru.utils.pdf_document import PDFDocument
from mineru.model.flash.pdf.table_recovery.engine import diagnose_native_pdf_table
from mineru.model.flash.pdf.document import PDFDocument
_DEFAULT_SOURCE_ROOT = Path(__file__).resolve().parents[1] / "unittest" / "pdfs" / "native_pdf_tables"
tests/unittest/_flash_pdf_test_utils.py
+1 -1
View File
@@ -2,7 +2,7 @@ from __future__ import annotations
from mineru.model.flash.native_pdf import (
from mineru.model.flash.pdf import (
models,
)
tests/unittest/test_backend_architecture.py
+219 -70
View File
@@ -1,38 +1,107 @@
from __future__ import annotations
import ast
import importlib.util
from pathlib import Path
import subprocess
import sys
from mineru.backend.postprocess import table_merge
_PROJECT_ROOT = Path(__file__).resolve().parents[2]
_COPYRIGHT_HEADER = "# Copyright (c) Opendatalab. All rights reserved."
_CHINESE_DOCSTRING_EXTRA_PATHS = (
"mineru/model/flash/office",
"mineru/model/flash/xycut.py",
"mineru/model/model_types.py",
"mineru/utils/native_pdf_table",
"mineru/utils/spatial_text.py",
"mineru/utils/text_utils.py",
_SOURCE_ROOTS = (
_PROJECT_ROOT / "mineru/backend",
_PROJECT_ROOT / "mineru/model",
_PROJECT_ROOT / "mineru/render",
_PROJECT_ROOT / "mineru/utils",
_PROJECT_ROOT / "mineru/parser",
)
_HEADER_PATHS = (
_PROJECT_ROOT / "mineru/backend",
_PROJECT_ROOT / "mineru/model/flash",
_PROJECT_ROOT / "mineru/model/runtime",
_PROJECT_ROOT / "mineru/model/registry.py",
_PROJECT_ROOT / "mineru/model/download.py",
_PROJECT_ROOT / "mineru/model/ocr/geometry.py",
_PROJECT_ROOT / "mineru/model/ocr/image.py",
_PROJECT_ROOT / "mineru/model/ocr/language.py",
_PROJECT_ROOT / "mineru/model/ocr/results.py",
_PROJECT_ROOT / "mineru/render",
_PROJECT_ROOT / "mineru/utils",
_PROJECT_ROOT / "mineru/parser/file_type.py",
_PROJECT_ROOT / "mineru/parser/page_range.py",
_PROJECT_ROOT / "mineru/parser/process_control.py",
_PROJECT_ROOT / "mineru/parser/writer.py",
)
_CHINESE_DOCSTRING_PATHS = (
_PROJECT_ROOT / "mineru/backend",
_PROJECT_ROOT / "mineru/model/runtime/device.py",
_PROJECT_ROOT / "mineru/model/runtime/memory.py",
_PROJECT_ROOT / "mineru/model/ocr/geometry.py",
_PROJECT_ROOT / "mineru/model/ocr/image.py",
_PROJECT_ROOT / "mineru/model/ocr/results.py",
_PROJECT_ROOT / "mineru/render/markdown.py",
_PROJECT_ROOT / "mineru/render/html.py",
_PROJECT_ROOT / "mineru/render/docx.py",
_PROJECT_ROOT / "mineru/render/structured_content.py",
_PROJECT_ROOT / "mineru/utils/image.py",
)
_REMOVED_INTERNAL_MODULES = (
"mineru.backend.local_model_runtime",
"mineru.model.model_types",
"mineru.model.flash.model",
"mineru.model.flash.native_pdf",
"mineru.model.utils",
"mineru.render.writer",
"mineru.render._internal.common.inline",
"mineru.utils.backend_options",
"mineru.utils.config_reader",
"mineru.utils.model_registry",
"mineru.utils.native_pdf_table",
"mineru.utils.ocr_utils",
"mineru.utils.pdf_document",
)
def _absolute_imports(path: Path) -> set[str]:
"""读取 Python 文件中的绝对 import 模块名。"""
def _module_name(path: Path) -> str:
"""把项目内 Python 路径转换为完整模块名。"""
relative = path.relative_to(_PROJECT_ROOT).with_suffix("")
parts = list(relative.parts)
if parts[-1] == "__init__":
parts.pop()
return ".".join(parts)
def _resolved_imports(path: Path) -> set[str]:
"""把绝对和相对 import 都解析为完整模块名。"""
tree = ast.parse(path.read_text(encoding="utf-8"))
module_name = _module_name(path)
package_parts = module_name.split(".") if path.name == "__init__.py" else module_name.split(".")[:-1]
imports: set[str] = set()
for node in ast.walk(tree):
if isinstance(node, ast.Import):
imports.update(alias.name for alias in node.names)
elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module:
imports.add(node.module)
continue
if not isinstance(node, ast.ImportFrom):
continue
if node.level == 0:
if node.module:
imports.add(node.module)
continue
resolved_parts = package_parts[:]
if node.level > 1:
resolved_parts = resolved_parts[: -(node.level - 1)]
if node.module:
resolved_parts.extend(node.module.split("."))
imports.add(".".join(resolved_parts))
return imports
def _relative_imports(path: Path) -> set[str]:
"""读取 Python 文件对同包模块的相对 import 名称。"""
tree = ast.parse(path.read_text(encoding="utf-8"))
return {node.module for node in ast.walk(tree) if isinstance(node, ast.ImportFrom) and node.level > 0 and node.module}
return {node.module for node in ast.walk(tree) if isinstance(node, ast.ImportFrom) and node.level == 1 and node.module}
def _contains_chinese(text: str) -> bool:
@@ -40,83 +109,163 @@ def _contains_chinese(text: str) -> bool:
return any("\u4e00" <= char <= "\u9fff" for char in text)
def _iter_refactored_python_paths() -> list[Path]:
"""收集 backend 及本次从 utils 迁出的公共 Python 模块。"""
paths = list((_PROJECT_ROOT / "mineru/backend").rglob("*.py"))
for relative_path in _CHINESE_DOCSTRING_EXTRA_PATHS:
path = _PROJECT_ROOT / relative_path
paths.extend(path.rglob("*.py") if path.is_dir() else [path])
return sorted(paths)
def _iter_python_paths(path: Path) -> list[Path]:
"""返回文件自身或目录下全部 Python 文件。"""
return sorted(path.rglob("*.py")) if path.is_dir() else [path]
def test_backend_python_files_have_copyright_header() -> None:
"""守卫 backend 下每个 Python 文件都以统一版权声明开头。"""
def test_target_python_files_have_copyright_header() -> None:
"""守卫本次目标目录中的一方 Python 文件使用统一版权头。"""
paths = [
path
for configured_path in _HEADER_PATHS
for path in _iter_python_paths(configured_path)
if "_internal/pytorchocr" not in path.as_posix() and path.name != "cli_parser.py"
]
offenders = [
str(path.relative_to(_PROJECT_ROOT))
for path in (_PROJECT_ROOT / "mineru/backend").rglob("*.py")
if path.read_text(encoding="utf-8").splitlines()[0] != _COPYRIGHT_HEADER
str(path.relative_to(_PROJECT_ROOT)) for path in paths if path.read_text().splitlines()[0] != _COPYRIGHT_HEADER
]
assert not offenders
def test_refactored_definitions_have_chinese_docstrings() -> None:
"""守卫 backend 及迁出模块中的函数、方法和类均有中文职责说明。"""
def test_new_first_party_definitions_have_chinese_docstrings() -> None:
"""守卫本次新增的一方函数、方法和类都有中文职责说明。"""
offenders: list[str] = []
for path in _iter_refactored_python_paths():
for configured_path in _CHINESE_DOCSTRING_PATHS:
for path in _iter_python_paths(configured_path):
tree = ast.parse(path.read_text(encoding="utf-8"))
for node in ast.walk(tree):
if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
continue
docstring = ast.get_docstring(node)
if not docstring or not _contains_chinese(docstring):
offenders.append(f"{path.relative_to(_PROJECT_ROOT)}:{node.lineno}:{node.name}")
assert not offenders
def test_active_mineru_imports_use_relative_form() -> None:
"""守卫活动生产代码的 MinerU 内部引用统一使用相对 import。"""
offenders: list[str] = []
for path in (_PROJECT_ROOT / "mineru").rglob("*.py"):
if "cli_old" in path.parts:
continue
tree = ast.parse(path.read_text(encoding="utf-8"))
for node in ast.walk(tree):
if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
continue
docstring = ast.get_docstring(node)
if not docstring or not _contains_chinese(docstring):
relative_path = path.relative_to(_PROJECT_ROOT)
offenders.append(f"{relative_path}:{node.lineno}:{node.name}")
if (
isinstance(node, ast.ImportFrom)
and node.level == 0
and (node.module == "mineru" or (node.module or "").startswith("mineru."))
):
offenders.append(f"{path.relative_to(_PROJECT_ROOT)}:{node.lineno}:{node.module}")
elif isinstance(node, ast.Import):
for alias in node.names:
if alias.name == "mineru" or alias.name.startswith("mineru."):
offenders.append(f"{path.relative_to(_PROJECT_ROOT)}:{node.lineno}:{alias.name}")
assert not offenders
def test_model_and_types_do_not_import_backend() -> None:
"""守卫模型层和公开类型层不再反向依赖 backend。"""
paths = [*_PROJECT_ROOT.glob("mineru/model/**/*.py"), _PROJECT_ROOT / "mineru/types.py"]
offenders = {
str(path.relative_to(_PROJECT_ROOT)): sorted(
module for module in _absolute_imports(path) if module.startswith("mineru.backend")
def test_layer_dependencies_are_one_way() -> None:
"""守卫 utils、model、backend 与 render 的单向依赖边界。"""
offenders: dict[str, list[str]] = {}
model_paths = [*_PROJECT_ROOT.glob("mineru/model/**/*.py"), _PROJECT_ROOT / "mineru/types.py"]
for path in model_paths:
invalid = sorted(module for module in _resolved_imports(path) if module.startswith(("mineru.backend", "mineru.render")))
if invalid:
offenders[str(path.relative_to(_PROJECT_ROOT))] = invalid
for path in _PROJECT_ROOT.glob("mineru/backend/**/*.py"):
invalid = sorted(module for module in _resolved_imports(path) if module.startswith("mineru.render"))
if invalid:
offenders[str(path.relative_to(_PROJECT_ROOT))] = invalid
for path in _PROJECT_ROOT.glob("mineru/utils/**/*.py"):
invalid = sorted(
module
for module in _resolved_imports(path)
if module.startswith(("mineru.backend", "mineru.model", "mineru.render", "mineru.parser"))
)
for path in paths
}
assert not {path: modules for path, modules in offenders.items() if modules}
def test_analysis_does_not_import_postprocess() -> None:
"""守卫 model-list 生产层不反向调用 Middle JSON 后处理层。"""
offenders = {
str(path.relative_to(_PROJECT_ROOT)): sorted(
module for module in _absolute_imports(path) if module.startswith("mineru.backend.postprocess")
if invalid:
offenders[str(path.relative_to(_PROJECT_ROOT))] = invalid
for path in _PROJECT_ROOT.glob("mineru/backend/analysis/**/*.py"):
invalid = sorted(module for module in _resolved_imports(path) if module.startswith("mineru.backend.postprocess"))
if invalid:
offenders[str(path.relative_to(_PROJECT_ROOT))] = invalid
allowed_render_backend = (
"mineru.backend.postprocess.inline",
"mineru.backend.postprocess.table_merge",
)
for path in _PROJECT_ROOT.glob("mineru/render/**/*.py"):
invalid = sorted(
module
for module in _resolved_imports(path)
if module.startswith("mineru.backend")
and not any(module == allowed or module.startswith(f"{allowed}.") for allowed in allowed_render_backend)
)
for path in _PROJECT_ROOT.glob("mineru/backend/analysis/**/*.py")
}
assert not {path: modules for path, modules in offenders.items() if modules}
if invalid:
offenders[str(path.relative_to(_PROJECT_ROOT))] = invalid
assert not offenders
def test_backend_utils_package_is_removed() -> None:
"""守卫 backend/utils 不再承载任何 Python 源码。"""
utils_path = _PROJECT_ROOT / "mineru/backend/utils"
assert not utils_path.exists()
def test_llm_postprocess_business_logic_is_not_kept_in_utils() -> None:
"""守卫 LLM 客户端和标题业务统一归属 backend/postprocess。"""
removed_utils_paths = [
_PROJECT_ROOT / "mineru/utils/llm_aided.py",
_PROJECT_ROOT / "mineru/utils/title_level_postprocess.py",
]
expected_backend_paths = [
_PROJECT_ROOT / "mineru/backend/postprocess/llm_client.py",
_PROJECT_ROOT / "mineru/backend/postprocess/title_leveling.py",
_PROJECT_ROOT / "mineru/backend/postprocess/table_merge/llm_cell_merge.py",
def test_package_initializers_define_explicit_all() -> None:
"""守卫目标目录下每个包入口都显式声明 __all__。"""
offenders = [
str(path.relative_to(_PROJECT_ROOT))
for root in _SOURCE_ROOTS
for path in root.rglob("__init__.py")
if "__all__" not in path.read_text(encoding="utf-8")
]
assert not offenders
assert not [path for path in removed_utils_paths if path.exists()]
assert all(path.is_file() for path in expected_backend_paths)
def test_removed_private_module_paths_have_no_active_references() -> None:
"""守卫严格迁移后的活动生产代码不再引用旧私有路径。"""
offenders: list[str] = []
for path in (_PROJECT_ROOT / "mineru").rglob("*.py"):
if "cli_old" in path.parts:
continue
text = path.read_text(encoding="utf-8")
for module_name in _REMOVED_INTERNAL_MODULES:
if module_name in text:
offenders.append(f"{path.relative_to(_PROJECT_ROOT)}:{module_name}")
assert not offenders
def test_removed_private_module_paths_are_not_importable() -> None:
"""验证严格切换后不存在可被误用的旧私有模块壳。"""
offenders: list[str] = []
for module_name in _REMOVED_INTERNAL_MODULES:
try:
spec = importlib.util.find_spec(module_name)
except ModuleNotFoundError:
spec = None
if spec is not None:
offenders.append(module_name)
assert not offenders
def test_public_facade_imports_do_not_load_heavy_dependencies_or_mutate_env() -> None:
"""验证三个稳定门面导入时不加载重依赖或修改环境变量。"""
script = """
import os
import sys
before_env = dict(os.environ)
import mineru.backend.analyze
import mineru.render
from mineru.model.flash import PdfModel
assert PdfModel.__name__ == "PdfModel"
for prefix in ("torch", "cv2", "pypdfium2", "pdftext", "docx", "bs4", "lxml", "nh3"):
assert prefix not in sys.modules, prefix
assert not any(name.startswith(prefix + ".") for name in sys.modules), prefix
assert before_env == dict(os.environ)
"""
result = subprocess.run(
[sys.executable, "-c", script],
cwd=_PROJECT_ROOT,
capture_output=True,
text=True,
check=False,
)
assert result.returncode == 0, result.stderr
def test_table_merge_package_keeps_one_way_internal_dependencies() -> None:
tests/unittest/test_doclib_app_startup.py
+4 -4
View File
@@ -16,8 +16,8 @@ from mineru.doclib import app as doclib_app
from mineru.doclib.app import _assert_required_schema
from mineru.doclib.core.db import DatabaseManager
from mineru.doclib.server import _tail_log, _write_temp_asset
from mineru.model import download as model_download
from mineru.parser import tier as parser_tier
from mineru.utils import model_registry
from mineru.version import __version__
@@ -138,7 +138,7 @@ def test_config_set_managed_tier_rejects_missing_models(monkeypatch: pytest.Monk
monkeypatch.setattr(doclib_app, "_create_background_task", _skip_background_task)
monkeypatch.setattr("mineru.doclib.server.ensure_tier_runtime_dependencies", lambda tier: None)
monkeypatch.setattr(model_registry.config.model, "base_dir", str(tmp_path / "models"))
monkeypatch.setattr(model_download.config.model, "base_dir", str(tmp_path / "models"))
cfg = PatchedConfig(doclib={"data_dir": str(tmp_path), "sqlite": {"path": str(tmp_path / "doclib.db")}})
with TestClient(doclib_app.create_app(cfg)) as client:
@@ -161,7 +161,7 @@ def test_config_set_managed_advanced_is_rejected(monkeypatch: pytest.MonkeyPatch
monkeypatch.setattr(doclib_app, "_create_background_task", _skip_background_task)
monkeypatch.setattr("mineru.doclib.server.ensure_tier_runtime_dependencies", lambda tier: None)
monkeypatch.setattr(model_registry.config.model, "base_dir", str(tmp_path / "models"))
monkeypatch.setattr(model_download.config.model, "base_dir", str(tmp_path / "models"))
cfg = PatchedConfig(doclib={"data_dir": str(tmp_path), "sqlite": {"path": str(tmp_path / "doclib.db")}})
with TestClient(doclib_app.create_app(cfg)) as client:
@@ -181,7 +181,7 @@ def test_config_set_managed_mode_rejects_missing_models_for_current_tier(
monkeypatch.setattr(doclib_app, "_create_background_task", _skip_background_task)
monkeypatch.setattr("mineru.doclib.server.ensure_tier_runtime_dependencies", lambda tier: None)
monkeypatch.setattr(model_registry.config.model, "base_dir", str(tmp_path / "models"))
monkeypatch.setattr(model_download.config.model, "base_dir", str(tmp_path / "models"))
cfg = PatchedConfig(doclib={"data_dir": str(tmp_path), "sqlite": {"path": str(tmp_path / "doclib.db")}})
with TestClient(doclib_app.create_app(cfg)) as client:
tests/unittest/test_docx_special_char_tables.py
+1 -1
View File
@@ -5,7 +5,7 @@ from docx import Document
from docx.oxml.ns import qn
from lxml import etree
from mineru.model.flash.docx.docx_converter import DocxConverter
from mineru.model.flash.office.docx.docx_converter import DocxConverter
NO_BREAK_HYPHEN = "‑"
tests/unittest/test_flash_docx_equationxml.py
+4 -4
View File
@@ -9,15 +9,15 @@ import pytest
from mineru.backend.analyze import aio_doc_analyze, doc_analyze
from mineru.model.flash import DocxModel
from mineru.model.flash.docx.docx_converter import DocxConverter
from mineru.model.flash.docx.equationxml import DocxEquationXmlDecoder
from mineru.model.flash.legacy_office.errors import (
from mineru.model.flash.office.docx.docx_converter import DocxConverter
from mineru.model.flash.office.docx.equationxml import DocxEquationXmlDecoder
from mineru.model.flash.office.legacy.errors import (
LegacyOfficeResourceLimitError,
)
from mineru.render._internal.docx.math import latex_to_omml
from mineru.types import BlockType, MiddleJson, ModelJson
import mineru.model.flash.docx.equationxml as equationxml_module
import mineru.model.flash.office.docx.equationxml as equationxml_module
from _docx_equationxml_test_utils import (
M_NS,
WORD_2003_NS,
tests/unittest/test_flash_image_mtef.py
+1 -1
View File
@@ -2,7 +2,7 @@ from __future__ import annotations
import pytest
from mineru.model.flash.legacy_office.errors import LegacyOfficeResourceLimitError
from mineru.model.flash.office.legacy.errors import LegacyOfficeResourceLimitError
from mineru.model.flash.office import image_equation as image_equation_module
from mineru.model.flash.office.image_equation import (
OfficeImageEquationDecoder,
tests/unittest/test_flash_legacy_chart_data.py
+3 -3
View File
@@ -6,9 +6,9 @@ import zlib
from bs4 import BeautifulSoup
from mineru.backend.analyze import doc_analyze
from mineru.model.flash.ppt import parser as ppt_parser
from mineru.model.flash.ppt.records import PptRecord
from mineru.model.flash.xls.embedded_chart import extract_embedded_chart_html
from mineru.model.flash.office.ppt import parser as ppt_parser
from mineru.model.flash.office.ppt.records import PptRecord
from mineru.model.flash.office.xls.embedded_chart import extract_embedded_chart_html
from mineru.types import BlockType, ChartBlock
from _legacy_ppt_test_utils import _build_cfb
tests/unittest/test_flash_legacy_doc.py
+8 -8
View File
@@ -11,12 +11,12 @@ import pytest
from mineru.backend.analyze import aio_doc_analyze, doc_analyze
from mineru.backend.postprocess.lists import fix_office_list_blocks
from mineru.model.flash import DocModel
from mineru.model.flash.doc.fields import sanitize_hyperlink_target
from mineru.model.flash.doc.models import DocCharStyle, DocTableCell
from mineru.model.flash.doc.parser import _RawTableRow, _materialize_table_rows
from mineru.model.flash.doc.records import DocBudget
from mineru.model.flash.doc.sprm import apply_character_sprms
from mineru.model.flash.legacy_office import (
from mineru.model.flash.office.doc.fields import sanitize_hyperlink_target
from mineru.model.flash.office.doc.models import DocCharStyle, DocTableCell
from mineru.model.flash.office.doc.parser import _RawTableRow, _materialize_table_rows
from mineru.model.flash.office.doc.records import DocBudget
from mineru.model.flash.office.doc.sprm import apply_character_sprms
from mineru.model.flash.office.legacy import (
LegacyOfficeEncryptedError,
LegacyOfficeMalformedError,
LegacyOfficeMissingPartError,
@@ -170,7 +170,7 @@ def test_doc_exact_list_label_is_consumed_before_strict_projection() -> None:
def test_doc_table_grid_materializes_colspan_and_rowspan() -> None:
"""验证 Word table edge 网格能同时恢复横向和纵向合并。"""
from mineru.model.flash.doc.models import DocTableCellFormat, DocTableFormat
from mineru.model.flash.office.doc.models import DocTableCellFormat, DocTableFormat
first = DocTableCell(blocks=[])
raw_rows = [
@@ -218,7 +218,7 @@ def test_doc_rejects_word95_encryption_rtf_and_missing_word_stream() -> None:
def test_doc_budget_uses_stable_resource_limit(monkeypatch: pytest.MonkeyPatch) -> None:
"""验证 DOC 记录预算超过固定上限时使用共享错误类型。"""
import mineru.model.flash.doc.records as records
import mineru.model.flash.office.doc.records as records
monkeypatch.setattr(records, "MAX_RECORDS", 1)
budget = records.DocBudget()
tests/unittest/test_flash_legacy_doc_mtef.py
+1 -1
View File
@@ -7,7 +7,7 @@ import pytest
from mineru.backend.analyze import aio_doc_analyze, doc_analyze
from mineru.model.flash import DocModel
from mineru.model.flash.legacy_office.mtef import (
from mineru.model.flash.office.legacy.mtef import (
decode_equation_native,
decode_equation_object,
decode_mtef_v3,
tests/unittest/test_flash_legacy_ppt.py
+6 -6
View File
@@ -11,12 +11,12 @@ from pydantic import ValidationError
from mineru.backend.analyze import aio_doc_analyze, doc_analyze
from mineru.model.flash import PptModel
from mineru.model.flash.legacy_office import LegacyOfficeEncryptedError, LegacyOfficeResourceLimitError
from mineru.model.flash.ppt import parser as ppt_parser
from mineru.model.flash.ppt.models import PptPresentation, PptSlide
from mineru.model.flash.ppt.ppt_converter import PptConverter
from mineru.model.flash.ppt.records import PptRecord, RecordBudget
from mineru.model.flash.ppt.style_text import CharacterRun, StyleRuns
from mineru.model.flash.office.legacy import LegacyOfficeEncryptedError, LegacyOfficeResourceLimitError
from mineru.model.flash.office.ppt import parser as ppt_parser
from mineru.model.flash.office.ppt.models import PptPresentation, PptSlide
from mineru.model.flash.office.ppt.ppt_converter import PptConverter
from mineru.model.flash.office.ppt.records import PptRecord, RecordBudget
from mineru.model.flash.office.ppt.style_text import CharacterRun, StyleRuns
from mineru.parser import parse
from mineru.types import BlockType, ChartBlock, ImageBlock, MiddleJson, ModelJson, TableBlock
tests/unittest/test_flash_legacy_xls.py
+6 -6
View File
@@ -10,16 +10,16 @@ import pytest
from mineru.backend.analyze import aio_doc_analyze, doc_analyze
from mineru.model.flash import XlsModel
from mineru.model.flash.legacy_office import (
from mineru.model.flash.office.legacy import (
LegacyOfficeEncryptedError,
LegacyOfficeMissingPartError,
LegacyOfficeResourceLimitError,
)
from mineru.model.flash.legacy_office.limits import MAX_RECORDS
from mineru.model.flash.xls import xls_converter as xls_converter_module
from mineru.model.flash.xls import parser as xls_parser
from mineru.model.flash.xls.number_format import format_number, format_text
from mineru.model.flash.xls.records import RecordBudget
from mineru.model.flash.office.legacy.limits import MAX_RECORDS
from mineru.model.flash.office.xls import xls_converter as xls_converter_module
from mineru.model.flash.office.xls import parser as xls_parser
from mineru.model.flash.office.xls.number_format import format_number, format_text
from mineru.model.flash.office.xls.records import RecordBudget
from mineru.parser import parse
from mineru.types import BlockType, ImageBlock, MiddleJson, ModelJson, TableBlock
tests/unittest/test_flash_legacy_xls_ppt_mtef.py
+2 -2
View File
@@ -8,8 +8,8 @@ import pytest
from mineru.backend.analyze import aio_doc_analyze, doc_analyze
from mineru.model.flash import PptModel, XlsModel
from mineru.model.flash.legacy_office import LegacyOfficeResourceLimitError
from mineru.model.flash.legacy_office.limits import MAX_ENTRY_BYTES
from mineru.model.flash.office.legacy import LegacyOfficeResourceLimitError
from mineru.model.flash.office.legacy.limits import MAX_ENTRY_BYTES
from mineru.types import BlockType, MiddleJson, ModelJson
from _legacy_ppt_test_utils import build_equation_ppt
tests/unittest/test_flash_model.py
+2 -2
View File
@@ -6,8 +6,8 @@ from unittest.mock import MagicMock
import pytest
from mineru.model.flash import PdfModel
from mineru.model.flash.native_pdf import pipeline
from mineru.utils.pdf_document import PDFDocument
from mineru.model.flash.pdf import pipeline
from mineru.model.flash.pdf.document import PDFDocument
def test_pdf_model_predict_returns_native_model_list_without_owning_document(
tests/unittest/test_flash_mtef_v5.py
+4 -4
View File
@@ -4,10 +4,10 @@ import struct
import pytest
from mineru.model.flash.legacy_office import mtef_v5 as mtef_v5_module
from mineru.model.flash.legacy_office import mtef as mtef_module
from mineru.model.flash.legacy_office.errors import LegacyOfficeResourceLimitError
from mineru.model.flash.legacy_office.mtef import (
from mineru.model.flash.office.legacy import mtef_v5 as mtef_v5_module
from mineru.model.flash.office.legacy import mtef as mtef_module
from mineru.model.flash.office.legacy.errors import LegacyOfficeResourceLimitError
from mineru.model.flash.office.legacy.mtef import (
decode_equation_native,
decode_equation_object,
decode_mtef,
tests/unittest/test_flash_office_image_mtef.py
+5 -5
View File
@@ -17,9 +17,9 @@ from mineru.model.flash import (
XlsModel,
XlsxModel,
)
from mineru.model.flash.doc.doc_converter import DocConverter
from mineru.model.flash.docx.docx_converter import DocxConverter
from mineru.model.flash.doc.models import (
from mineru.model.flash.office.doc.doc_converter import DocConverter
from mineru.model.flash.office.docx.docx_converter import DocxConverter
from mineru.model.flash.office.doc.models import (
DocImage,
DocImagePayload,
DocParagraph,
@@ -27,8 +27,8 @@ from mineru.model.flash.doc.models import (
DocTableCell,
DocTableRow,
)
from mineru.model.flash.xlsx.xlsx_converter import XlsxConverter
from mineru.model.flash.pptx.pptx_converter import PptxConverter
from mineru.model.flash.office.xlsx.xlsx_converter import XlsxConverter
from mineru.model.flash.office.pptx.pptx_converter import PptxConverter
from mineru.types import BlockType, MiddleJson, ModelJson
from _docx_equationxml_test_utils import (
tests/unittest/test_flash_office_models.py
+22 -22
View File
@@ -11,17 +11,17 @@ from unittest.mock import MagicMock
import pytest
import mineru.model.flash as flash_models
import mineru.model.flash.model as flat_model_module
import mineru.model.flash.models as flat_model_module
from mineru.model.flash import DocModel, DocxModel, PdfModel, PptModel, PptxModel, XlsModel, XlsxModel
from mineru.model.flash.doc import doc_converter as doc_converter_module
from mineru.model.flash.docx import docx_converter as docx_converter_module
from mineru.model.flash.docx import main as docx_main
from mineru.model.flash.pptx import main as pptx_main
from mineru.model.flash.pptx import pptx_converter as pptx_converter_module
from mineru.model.flash.ppt import ppt_converter as ppt_converter_module
from mineru.model.flash.xls import xls_converter as xls_converter_module
from mineru.model.flash.xlsx import main as xlsx_main
from mineru.model.flash.xlsx import xlsx_converter as xlsx_converter_module
from mineru.model.flash.office.doc import doc_converter as doc_converter_module
from mineru.model.flash.office.docx import docx_converter as docx_converter_module
from mineru.model.flash.office.docx import main as docx_main
from mineru.model.flash.office.pptx import main as pptx_main
from mineru.model.flash.office.pptx import pptx_converter as pptx_converter_module
from mineru.model.flash.office.ppt import ppt_converter as ppt_converter_module
from mineru.model.flash.office.xls import xls_converter as xls_converter_module
from mineru.model.flash.office.xlsx import main as xlsx_main
from mineru.model.flash.office.xlsx import xlsx_converter as xlsx_converter_module
_PROJECT_ROOT = Path(__file__).resolve().parents[2]
@@ -174,12 +174,12 @@ def test_models_are_exported_from_flash_root() -> None:
@pytest.mark.parametrize(
("package_name", "model_name"),
[
("mineru.model.flash.doc", "DocModel"),
("mineru.model.flash.docx", "DocxModel"),
("mineru.model.flash.pptx", "PptxModel"),
("mineru.model.flash.ppt", "PptModel"),
("mineru.model.flash.xls", "XlsModel"),
("mineru.model.flash.xlsx", "XlsxModel"),
("mineru.model.flash.office.doc", "DocModel"),
("mineru.model.flash.office.docx", "DocxModel"),
("mineru.model.flash.office.pptx", "PptxModel"),
("mineru.model.flash.office.ppt", "PptModel"),
("mineru.model.flash.office.xls", "XlsModel"),
("mineru.model.flash.office.xlsx", "XlsxModel"),
],
)
def test_office_subpackages_do_not_export_models(package_name: str, model_name: str) -> None:
@@ -198,13 +198,13 @@ def test_importing_pdf_model_does_not_load_office_converters() -> None:
"import sys",
"from mineru.model.flash import PdfModel",
"assert PdfModel.__name__ == 'PdfModel'",
"assert 'mineru.model.flash.docx.docx_converter' not in sys.modules",
"assert 'mineru.model.flash.doc.doc_converter' not in sys.modules",
"assert 'mineru.model.flash.pptx.pptx_converter' not in sys.modules",
"assert 'mineru.model.flash.ppt.ppt_converter' not in sys.modules",
"assert 'mineru.model.flash.xls.xls_converter' not in sys.modules",
"assert 'mineru.model.flash.office.docx.docx_converter' not in sys.modules",
"assert 'mineru.model.flash.office.doc.doc_converter' not in sys.modules",
"assert 'mineru.model.flash.office.pptx.pptx_converter' not in sys.modules",
"assert 'mineru.model.flash.office.ppt.ppt_converter' not in sys.modules",
"assert 'mineru.model.flash.office.xls.xls_converter' not in sys.modules",
"assert 'olefile' not in sys.modules",
"assert 'mineru.model.flash.xlsx.xlsx_converter' not in sys.modules",
"assert 'mineru.model.flash.office.xlsx.xlsx_converter' not in sys.modules",
]
)
result = subprocess.run(
tests/unittest/test_flash_office_mtef_v5.py
+3 -3
View File
@@ -15,9 +15,9 @@ from mineru.model.flash import (
XlsModel,
XlsxModel,
)
from mineru.model.flash.docx.docx_converter import DocxConverter
from mineru.model.flash.pptx.pptx_converter import PptxConverter
from mineru.model.flash.xlsx.xlsx_converter import XlsxConverter
from mineru.model.flash.office.docx.docx_converter import DocxConverter
from mineru.model.flash.office.pptx.pptx_converter import PptxConverter
from mineru.model.flash.office.xlsx.xlsx_converter import XlsxConverter
from mineru.types import BlockType, MiddleJson, ModelJson
from _docx_equationxml_test_utils import (
tests/unittest/test_flash_office_title_types.py
+1 -1
View File
@@ -9,7 +9,7 @@ from pptx import Presentation
from pptx.enum.shapes import PP_PLACEHOLDER
from mineru.model.flash import DocxModel, PptxModel, XlsxModel
from mineru.model.flash.pptx.pptx_converter import (
from mineru.model.flash.office.pptx.pptx_converter import (
PptxConverter,
_EFFECTIVE_ALL_BOLD_KEY,
_EFFECTIVE_FONT_SIZE_KEY,
tests/unittest/test_flash_ooxml_mtef.py
+5 -5
View File
@@ -10,15 +10,15 @@ import pytest
from mineru.backend.analyze import aio_doc_analyze, doc_analyze
from mineru.model.flash import DocxModel, PptxModel, XlsxModel
from mineru.model.flash.docx.docx_converter import DocxConverter
from mineru.model.flash.legacy_office import LegacyOfficeResourceLimitError
from mineru.model.flash.legacy_office.limits import MAX_ASSET_TOTAL_BYTES
from mineru.model.flash.office.docx.docx_converter import DocxConverter
from mineru.model.flash.office.legacy import LegacyOfficeResourceLimitError
from mineru.model.flash.office.legacy.limits import MAX_ASSET_TOTAL_BYTES
from mineru.model.flash.office.ooxml_equation import (
OoxmlEquationDecoder,
is_mathtype_equation_prog_id,
)
from mineru.model.flash.pptx.pptx_converter import PptxConverter
from mineru.model.flash.xlsx.xlsx_converter import XlsxConverter
from mineru.model.flash.office.pptx.pptx_converter import PptxConverter
from mineru.model.flash.office.xlsx.xlsx_converter import XlsxConverter
from mineru.types import BlockType, MiddleJson, ModelJson
from _mtef_test_utils import build_equation_object, formula_corpus
tests/unittest/test_flash_pdf_auxiliary_text.py
+1 -1
View File
@@ -4,7 +4,7 @@ import inspect
import pytest
from mineru.model.flash.native_pdf import (
from mineru.model.flash.pdf import (
auxiliary_text,
models,
pipeline,
tests/unittest/test_flash_pdf_code_blocks.py
+4 -4
View File
@@ -6,13 +6,13 @@ from typing import Any
import pytest
from mineru.model.flash import PdfModel
from mineru.model.flash.native_pdf import code_blocks, models, pipeline
from mineru.render._internal.common.inline import (
from mineru.model.flash.pdf import code_blocks, models, pipeline
from mineru.backend.postprocess.inline import (
inline_plain_text,
parse_inline_content,
)
from mineru.utils.pdf_document import PDFDocument, PDFPathInfo
from mineru.utils.spatial_text import project_pdf_spatial_text
from mineru.model.flash.pdf.document import PDFDocument, PDFPathInfo
from mineru.model.flash.pdf.spatial_text import project_pdf_spatial_text
from _flash_pdf_test_utils import _text_line
tests/unittest/test_flash_pdf_document_profile.py
+1 -1
View File
@@ -1,6 +1,6 @@
from __future__ import annotations
from mineru.model.flash.native_pdf import titles
from mineru.model.flash.pdf import titles
from _flash_pdf_test_utils import _prepared_text_page, _text_line
tests/unittest/test_flash_pdf_extractor.py
+1 -1
View File
@@ -1,4 +1,4 @@
from mineru.model.flash.native_pdf import pipeline
from mineru.model.flash.pdf import pipeline
def test_marginal_header_row_does_not_break_two_column_reading_order() -> None:
tests/unittest/test_flash_pdf_formulas.py
+2 -2
View File
@@ -5,13 +5,13 @@ from dataclasses import replace
import pytest
from mineru.model.flash.native_pdf import (
from mineru.model.flash.pdf import (
formulas,
geometry,
line_merging,
models,
)
from mineru.utils.pdf_document import PDFPathInfo
from mineru.model.flash.pdf.document import PDFPathInfo
from _flash_pdf_test_utils import (
tests/unittest/test_flash_pdf_graphics.py
+2 -2
View File
@@ -5,12 +5,12 @@ import inspect
import pytest
from mineru.model.flash.native_pdf import (
from mineru.model.flash.pdf import (
graphics,
models,
pipeline,
)
from mineru.utils.pdf_document import PDFPathInfo
from mineru.model.flash.pdf.document import PDFPathInfo
from _flash_pdf_test_utils import (
tests/unittest/test_flash_pdf_index_blocks.py
+1 -1
View File
@@ -2,7 +2,7 @@ from __future__ import annotations
import pytest
from mineru.model.flash.native_pdf import index_blocks
from mineru.model.flash.pdf import index_blocks
from _flash_pdf_test_utils import _text_line
tests/unittest/test_flash_pdf_line_metadata.py
+1 -1
View File
@@ -2,7 +2,7 @@ from __future__ import annotations
import pytest
from mineru.model.flash.native_pdf import pipeline
from mineru.model.flash.pdf import pipeline
@pytest.mark.parametrize(
tests/unittest/test_flash_pdf_native_text.py
+1 -1
View File
@@ -5,7 +5,7 @@ from typing import Any
import pytest
from mineru.model.flash.native_pdf import native_text
from mineru.model.flash.pdf import native_text
def _span(
tests/unittest/test_flash_pdf_pipeline.py
+2 -2
View File
@@ -6,7 +6,7 @@ import inspect
import pytest
from mineru.model.flash import PdfModel
from mineru.model.flash.native_pdf import (
from mineru.model.flash.pdf import (
auxiliary_text,
formulas,
geometry,
@@ -22,7 +22,7 @@ from mineru.model.flash.native_pdf import (
titles,
visual_annotations,
)
from mineru.utils.pdf_document import PDFImageInfo
from mineru.model.flash.pdf.document import PDFImageInfo
from _flash_pdf_test_utils import (
_prepared_text_page,
tests/unittest/test_flash_pdf_samples.py
+3 -3
View File
@@ -13,7 +13,7 @@ from pypdf import PdfReader
from mineru.backend.postprocess.page_blocks import process_page_blocks
from mineru.backend.postprocess.pages import model_json_to_pages
from mineru.model.flash import PdfModel
from mineru.model.flash.native_pdf import (
from mineru.model.flash.pdf import (
formulas,
geometry,
graphics,
@@ -23,12 +23,12 @@ from mineru.model.flash.native_pdf import (
tables,
)
from mineru.render import render_markdown
from mineru.render._internal.common.inline import (
from mineru.backend.postprocess.inline import (
inline_plain_text,
parse_inline_content,
)
from mineru.types import MiddleJson, ModelJson
from mineru.utils.pdf_document import PDFDocument, get_lines_from_chars
from mineru.model.flash.pdf.document import PDFDocument, get_lines_from_chars
_PROJECT_ROOT = Path(__file__).parents[2]
tests/unittest/test_flash_pdf_tables.py
+2 -2
View File
@@ -4,12 +4,12 @@ from unittest.mock import MagicMock
import pytest
from mineru.model.flash.native_pdf import (
from mineru.model.flash.pdf import (
geometry,
models,
tables,
)
from mineru.utils.pdf_document import PDFPathInfo
from mineru.model.flash.pdf.document import PDFPathInfo
def _axis_line(
tests/unittest/test_flash_pdf_text_blocks.py
+1 -1
View File
@@ -3,7 +3,7 @@ from __future__ import annotations
import pytest
from mineru.model.flash.native_pdf import (
from mineru.model.flash.pdf import (
line_layout,
line_merging,
models,
tests/unittest/test_flash_pdf_titles.py
+1 -1
View File
@@ -4,7 +4,7 @@ import inspect
import pytest
from mineru.model.flash.native_pdf import (
from mineru.model.flash.pdf import (
line_layout,
line_merging,
text_blocks,
tests/unittest/test_flash_pdf_visual_annotations.py
+1 -1
View File
@@ -2,7 +2,7 @@ from __future__ import annotations
import pytest
from mineru.model.flash.native_pdf import (
from mineru.model.flash.pdf import (
geometry,
pipeline,
text_blocks,
tests/unittest/test_guess_suffix_ole2.py
+1 -1
View File
@@ -1,6 +1,6 @@
from __future__ import annotations
from mineru.utils.guess_suffix_or_lang import (
from mineru.parser.file_type import (
_guess_ole2_suffix_by_bytes,
guess_suffix_by_bytes,
)
tests/unittest/test_kit_commands.py
+1 -1
View File
@@ -28,7 +28,7 @@ from mineru.kit.vlm_server import mlx_vlm_server
from mineru.parser.base import ParseResult
from mineru.types import BlockType, MiddleJson, PageInfo
from mineru.utils.image_payload import ImagePayloadCache
from mineru.utils.model_registry import MODEL_COMPLETE_MARKER
from mineru.model.registry import MODEL_COMPLETE_MARKER
from mineru.version import __version__
runner = CliRunner()
tests/unittest/test_managed_process_control.py
+2 -2
View File
@@ -8,7 +8,7 @@ from pathlib import Path
import pytest
from mineru.utils.managed_process_control import (
from mineru.parser.process_control import (
CONTROL_ENV,
ManagedProcessControl,
ManagedProcessControlWatcher,
@@ -71,7 +71,7 @@ def test_managed_process_control_works_across_subprocess() -> None:
child_code = """
import sys
import threading
from mineru.utils.managed_process_control import ManagedProcessControlWatcher
from mineru.parser.process_control import ManagedProcessControlWatcher
shutdown = threading.Event()
watcher = ManagedProcessControlWatcher.from_environment(shutdown.set)
tests/unittest/test_markdown_render.py
+1 -1
View File
@@ -583,7 +583,7 @@ def test_equation_uses_content_then_image_fallback(monkeypatch: pytest.MonkeyPat
}
}
)
monkeypatch.setattr("mineru.render.markdown.config", configured)
monkeypatch.setattr("mineru.render._internal.markdown.renderer.config", configured)
middle = _middle(
_page(
0,
tests/unittest/test_models_download_utils.py
+8 -8
View File
@@ -5,8 +5,8 @@ from typing import Any, NoReturn
import pytest
from mineru.utils import models_download_utils
from mineru.utils.model_registry import MODEL_COMPLETE_MARKER, MINERU_2_5_PRO_2605_1_2B, ModelRepo, model_path_exists
from mineru.model import download as models_download_utils
from mineru.model.registry import MODEL_COMPLETE_MARKER, MINERU_2_5_PRO_2605_1_2B, ModelRepo, model_path_exists
def test_resolve_model_source_does_not_persist_env_auto(monkeypatch: pytest.MonkeyPatch) -> None:
@@ -443,27 +443,27 @@ def test_huggingface_snapshot_rejects_missing_expected_files(
def test_resolve_model_stack_explicit_light() -> None:
from mineru.utils.model_registry import resolve_model_stack
from mineru.model.registry import resolve_model_stack
assert resolve_model_stack("light") == "light"
assert resolve_model_stack("full") == "full"
def test_resolve_model_stack_auto_falls_back_to_get_model_stack(monkeypatch: pytest.MonkeyPatch) -> None:
import mineru.utils.config_reader as config_reader
from mineru.utils.model_registry import resolve_model_stack
import mineru.model.runtime.device as device_runtime
from mineru.model.registry import resolve_model_stack
monkeypatch.setattr(config_reader, "get_model_stack", lambda: "light")
monkeypatch.setattr(device_runtime, "get_model_stack", lambda: "light")
assert resolve_model_stack(None) == "light"
assert resolve_model_stack("auto") == "light"
monkeypatch.setattr(config_reader, "get_model_stack", lambda: "full")
monkeypatch.setattr(device_runtime, "get_model_stack", lambda: "full")
assert resolve_model_stack(None) == "full"
assert resolve_model_stack("auto") == "full"
def test_resolve_model_stack_rejects_invalid_value() -> None:
from mineru.utils.model_registry import resolve_model_stack
from mineru.model.registry import resolve_model_stack
with pytest.raises(ValueError, match="Unsupported stack 'torch'"):
resolve_model_stack("torch")
tests/unittest/test_native_pdf_table.py
+7 -7
View File
@@ -9,7 +9,7 @@ from io import BytesIO
import pytest
from reportlab.pdfgen.canvas import Canvas
from mineru.utils.native_pdf_table import (
from mineru.model.flash.pdf.table_recovery import (
NativeTableCell,
NativeTableInput,
NativeTableRectangle,
@@ -18,19 +18,19 @@ from mineru.utils.native_pdf_table import (
coerce_native_table_rules,
recover_native_pdf_table,
)
from mineru.utils.native_pdf_table.candidate import GridCellSpec, build_candidate
from mineru.utils.native_pdf_table.contracts import NativeTableCandidate
from mineru.utils.native_pdf_table.engine import (
from mineru.model.flash.pdf.table_recovery.candidate import GridCellSpec, build_candidate
from mineru.model.flash.pdf.table_recovery.contracts import NativeTableCandidate
from mineru.model.flash.pdf.table_recovery.engine import (
_remove_undercounted_vector_candidates,
_select_candidate,
diagnose_native_pdf_table,
)
from mineru.utils.native_pdf_table.text import build_native_table_text
from mineru.utils.native_pdf_table.vector import (
from mineru.model.flash.pdf.table_recovery.text import build_native_table_text
from mineru.model.flash.pdf.table_recovery.vector import (
MAX_PRIMITIVES_PER_TABLE,
build_vector_candidates,
)
from mineru.utils.pdf_document import PDFDocument
from mineru.model.flash.pdf.document import PDFDocument
def _char_items(
tests/unittest/test_native_pdf_table_demo_manifest.py
+3 -3
View File
@@ -11,14 +11,14 @@ from typing import Any
from bs4 import BeautifulSoup
from mineru.model.flash import PdfModel
from mineru.utils.native_pdf_table import (
from mineru.model.flash.pdf.table_recovery import (
NativeTableInput,
coerce_native_table_rectangles,
coerce_native_table_rules,
recover_native_pdf_table,
)
from mineru.utils.native_pdf_table.engine import diagnose_native_pdf_table
from mineru.utils.pdf_document import PDFDocument
from mineru.model.flash.pdf.table_recovery.engine import diagnose_native_pdf_table
from mineru.model.flash.pdf.document import PDFDocument
_PROJECT_ROOT = Path(__file__).parents[2]
tests/unittest/test_native_pdf_table_pipeline.py
+3 -3
View File
@@ -14,8 +14,8 @@ from mineru.backend.analysis.pdf import formulas as pdf_formulas
from mineru.backend.analysis.pdf import layout as pdf_layout
from mineru.backend.analysis.pdf import tables as pdf_tables
from mineru.backend.analysis.pdf import window as pdf_window
from mineru.model.flash.native_pdf import models as flash_models
from mineru.model.flash.native_pdf import tables as flash_tables
from mineru.model.flash.pdf import models as flash_models
from mineru.model.flash.pdf import tables as flash_tables
from mineru.types import RAW_FORMULA_NUMBER, BlockType
@@ -491,7 +491,7 @@ def test_high_txt_window_excludes_native_table_from_vlm(
return model_list
vlm_predictor.batch_extract_with_layout.side_effect = fake_high_extract
monkeypatch.setattr(pdf_window, "get_processing_window_size", lambda default: 1)
monkeypatch.setattr(pdf_window, "_configured_window_size", lambda default: 1)
monkeypatch.setattr(
pdf_window,
"load_images_from_pdf_bytes_range",
tests/unittest/test_parser_api_contract.py
+2 -2
View File
@@ -128,7 +128,7 @@ import os
os.environ["MINERU_ENABLE_LOCAL_MODEL_INFERENCE_LOCKS"] = "true"
os.environ["MINERU_ENABLE_PIPELINE_INFERENCE_LOCKS"] = "false"
from mineru.backend import local_model_runtime
from mineru.model.runtime import hybrid as local_model_runtime
assert hasattr(local_model_runtime, "HybridLocalModelContext")
assert hasattr(local_model_runtime, "HybridLocalModelContextSingleton")
@@ -156,7 +156,7 @@ print("ok")
def test_validate_effort_rejects_low_and_maps_legacy_backends() -> None:
"""校验 Hybrid effort 只接受 medium/high/xhigh 三档。"""
from mineru.utils.backend_options import (
from mineru.parser.tier import (
HYBRID_EFFORT_CHOICES,
effort_for_tier,
resolve_backend_and_effort,
tests/unittest/test_parser_model_preload.py
+10 -18
View File
@@ -11,13 +11,6 @@ import mineru.parser.api_server as api_server
def test_preload_local_models_initializes_conditional_model_families(monkeypatch: pytest.MonkeyPatch) -> None:
calls: list[tuple[str, str | None]] = []
class _AtomicModel:
TableOrientationCls = "table-orientation"
TableCls = "table-classification"
WirelessTable = "wireless-table"
WiredTable = "wired-table"
OCR = "ocr"
class _Manager:
def get_atom_model(self, atom_model_name: str, **kwargs: str) -> object:
calls.append((atom_model_name, kwargs.get("lang")))
@@ -27,23 +20,22 @@ def test_preload_local_models_initializes_conditional_model_families(monkeypatch
atom_model_manager = _Manager()
class _ContextSingleton:
def get_model(self, lang: str, formula_enable: bool) -> _Context:
calls.append(("context", lang if formula_enable else None))
def get_model(self) -> _Context:
calls.append(("context", None))
return _Context()
fake_runtime = types.ModuleType("mineru.backend.local_model_runtime")
fake_runtime.AtomicModel = _AtomicModel
fake_runtime = types.ModuleType("mineru.model.runtime.hybrid")
fake_runtime.HybridLocalModelContextSingleton = _ContextSingleton
monkeypatch.setitem(sys.modules, "mineru.backend.local_model_runtime", fake_runtime)
monkeypatch.setitem(sys.modules, "mineru.model.runtime.hybrid", fake_runtime)
api_server._preload_local_models("ch")
assert calls == [
("context", "ch"),
("table-orientation", None),
("table-classification", None),
("wireless-table", "ch"),
("wired-table", "ch"),
("context", None),
("table_ori_cls", None),
("table_cls", None),
("wireless_table", "ch"),
("wired_table", "ch"),
("ocr", "seal"),
]
@@ -62,7 +54,7 @@ def test_preload_standard_models_initializes_platform_engine_and_local_models(mo
calls: list[object] = []
monkeypatch.setattr(api_server, "_preload_local_models", lambda language: calls.append(("local", language)))
from mineru.utils import engine_utils
from mineru.model.vlm import selector as engine_utils
monkeypatch.setattr(engine_utils, "get_vlm_engine", lambda inference_engine, is_async=False: "lmdeploy-engine")
tests/unittest/test_pdf_analyze_block_content.py
+3 -5
View File
@@ -12,6 +12,7 @@ from mineru.backend import analyze
from mineru.backend.analysis.pdf import constants, layout, normalization, ocr, pipeline, window
from mineru.backend.analysis.pdf.text import content as text_content
from mineru.backend.analysis.pdf.text.models import _AnalyzeLine, _AnalyzeSpan
from mineru.backend.postprocess import document as postprocess_document
from mineru.types import RAW_ALGORITHM, RAW_CAPTION, RAW_FOOTNOTE
from mineru.backend.analysis.pdf.text.native import (
POST_OCR_FALLBACK_CONTENT_KEY,
@@ -213,7 +214,7 @@ def test_doc_analyze_converts_vlm_results_before_downstream_processing(
parse_mode=parse_mode, # type: ignore[arg-type]
mineru_version="test",
)
monkeypatch.setattr(analyze, "model_json_to_middle_json", MagicMock(return_value=expected_middle_json))
monkeypatch.setattr(postprocess_document, "model_json_to_middle_json", MagicMock(return_value=expected_middle_json))
monkeypatch.setattr(pipeline, "clean_memory", MagicMock())
middle_json, model_json = analyze.doc_analyze(
@@ -551,10 +552,7 @@ def test_text_content_joins_three_line_url_with_accumulated_context() -> None:
BlockType.TEXT,
)
assert content == (
"Code at "
"https://github.com/google-research/tapas/blob/master/TABLEFORMER.md"
)
assert content == ("Code at https://github.com/google-research/tapas/blob/master/TABLEFORMER.md")
@pytest.mark.parametrize("block_type", [BlockType.CODE, RAW_ALGORITHM])
tests/unittest/test_pdf_analyze_visual_blocks.py
+4 -4
View File
@@ -1014,7 +1014,7 @@ def test_flash_ocr_formula_number_merge_runs_before_visual_crop(
]
]
monkeypatch.setattr(window, "get_processing_window_size", lambda default: 1)
monkeypatch.setattr(window, "_configured_window_size", lambda default: 1)
monkeypatch.setattr(
window,
"load_images_from_pdf_bytes_range",
@@ -1301,7 +1301,7 @@ def test_doc_analyze_office_returns_model_json_without_pdf_processing(
monkeypatch.setattr(office, "_OFFICE_MODEL_MAP", model_factories)
monkeypatch.setattr(pipeline, "PDFDocument", pdf_document)
monkeypatch.setattr(pipeline, "HybridLocalModelContextSingleton", hybrid_model_factory)
monkeypatch.setattr(window, "get_processing_window_size", window_size_reader)
monkeypatch.setattr(window, "_configured_window_size", window_size_reader)
monkeypatch.setattr(window, "_build_processing_windows", window_builder)
monkeypatch.setattr(window, "load_images_from_pdf_bytes_range", image_loader)
monkeypatch.setattr(window, "_attach_visual_block_images", visual_image_attacher)
@@ -1663,7 +1663,7 @@ def test_pdf_window_releases_rendered_images_when_layout_fails(
hybrid_model = MagicMock()
hybrid_model.layout_model.batch_predict.side_effect = RuntimeError("layout failed")
monkeypatch.setattr(window, "get_processing_window_size", lambda default: 1)
monkeypatch.setattr(window, "_configured_window_size", lambda default: 1)
monkeypatch.setattr(
window,
"load_images_from_pdf_bytes_range",
@@ -1873,7 +1873,7 @@ def test_doc_analyze_flash_returns_complete_model_json_and_typed_middle_json(mon
return value
monkeypatch.setattr(pipeline, "PDFDocument", lambda _: fake_pdf_doc)
monkeypatch.setattr(window, "get_processing_window_size", lambda default: 2)
monkeypatch.setattr(window, "_configured_window_size", lambda default: 2)
monkeypatch.setattr(window, "load_images_from_pdf_bytes_range", fake_load_images_for_window)
monkeypatch.setattr(window, "_attach_visual_block_images", tracked_attach_visual_block_images)
monkeypatch.setattr(pipeline, "_normalize_pdf_model_list", tracked_normalize_model_list)
tests/unittest/test_pdf_classify.py
+2 -2
View File
@@ -15,8 +15,8 @@ from pypdf.generic import (
NumberObject,
)
from mineru.utils import pdf_classify
from mineru.utils.pdf_document import PDFDocument
from mineru.model.flash.pdf import classify as pdf_classify
from mineru.model.flash.pdf.document import PDFDocument
REPO_ROOT = Path(__file__).resolve().parents[2]
MIXED_ELEMENTS_PDF = REPO_ROOT / "demo" / "pdfs" / "mixed_elements_pages_07_10.pdf"
tests/unittest/test_pdf_document.py
+1 -1
View File
@@ -20,7 +20,7 @@ from PIL import Image
from reportlab.lib.utils import ImageReader
from reportlab.pdfgen.canvas import Canvas
from mineru.utils import pdf_document
from mineru.model.flash.pdf import document as pdf_document
def test_pdf_page_exposes_path_infos_without_raw_pdfium_access() -> None:
tests/unittest/test_pdf_native_script_detection.py
+1 -1
View File
@@ -9,7 +9,7 @@ from pdftext.schema import Bbox, Char
from mineru.backend.analysis.pdf.text import native
from mineru.backend.analysis.pdf.text.models import _AnalyzeSpan
from mineru.types import ContentType
from mineru.utils.pdf_document import PDFDocument
from mineru.model.flash.pdf.document import PDFDocument
_PROJECT_ROOT = Path(__file__).resolve().parents[2]
tests/unittest/test_pdf_page_id.py
+1 -1
View File
@@ -2,7 +2,7 @@ from __future__ import annotations
import pytest
from mineru.utils.pdf_page_id import parse_page_range
from mineru.parser.page_range import parse_page_range
def test_parse_page_range_empty_means_all_pages() -> None:
tests/unittest/test_pdf_render_diagnostics.py
+1 -1
View File
@@ -6,7 +6,7 @@ from typing import Any, cast
import pytest
from loguru import logger
from mineru.utils import pdf_image_tools
from mineru.backend.analysis.pdf import images as pdf_image_tools
class _FakeProcess:
tests/unittest/test_pdf_text_styles.py
+2 -2
View File
@@ -25,8 +25,8 @@ from mineru.types import (
ModelJson,
PageInfo,
)
from mineru.utils.pdf_document import PDFDocument, PDFLinkAnnotation
from mineru.utils.pdf_text_styles import (
from mineru.model.flash.pdf.document import PDFDocument, PDFLinkAnnotation
from mineru.model.flash.pdf.text_styles import (
PDF_FONT_FORCE_BOLD_FLAG,
PDF_FONT_ITALIC_FLAG,
PDFTextLinkLine,
tests/unittest/test_structured_content_render.py
+2 -2
View File
@@ -307,7 +307,7 @@ def test_structured_content_keeps_chart_content_separate_from_base64_source(
}
}
)
monkeypatch.setattr("mineru.render.structured_content.config", configured)
monkeypatch.setattr("mineru.render._internal.structured_content.renderer.config", configured)
chart = ChartBlock(
type="chart",
index=0,
@@ -353,7 +353,7 @@ def test_structured_content_renders_equation_as_raw_latex_with_single_image_sour
}
}
)
monkeypatch.setattr("mineru.render.structured_content.config", configured)
monkeypatch.setattr("mineru.render._internal.structured_content.renderer.config", configured)
middle = _middle(
_page(
0,
tests/unittest/test_text_utils.py
+1 -1
View File
@@ -1,6 +1,6 @@
import pytest
from mineru.utils.text_utils import (
from mineru.utils.text import (
merge_text_line_contents,
resolve_text_line_boundary,
)