diff --git a/CLAUDE.md b/CLAUDE.md index 34e41e81..8e7c34ef 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -42,11 +42,11 @@ mineru 模块内部子模块之间的引用统一使用 **relative import**, ```python # 正确 — relative import from .base import DocumentParser -from ..utils.enum_class import MakeMode +from ..render import RenderMode # 错误 — 项目内不允许 absolute import 引用自身模块 from mineru.api.base import DocumentParser -from mineru.utils.enum_class import MakeMode +from mineru.render import RenderMode ``` 只引用外部第三方库时使用 absolute import(如 `from loguru import logger`)。 @@ -199,7 +199,14 @@ pr-5415 重构后,Middle JSON 已收敛为 schema 2.0 的统一结构,不再 - `RenderFormat.DOCX` → `render_docx` - `RenderFormat.STRUCTURED_CONTENT` → `render_structured_content` -`render/_internal/` 下按目标格式分目录组织共享逻辑(`common/`/`markdown/`/`html/`/`docx/`)。不再有 `pipeline_union_make`/`vlm_union_make`/`office_union_make` 三套逻辑,`content_list` 格式和 `render_content_list` 函数已删除。 +`render/_internal/` 下按目标格式分目录组织共享逻辑(`common/`/`markdown/`/`html/`/`docx/`/`structured_content/`),顶层同名模块只是惰性公共门面。行内语义解析归 `backend/postprocess/inline.py`,renderer 只能单向依赖该模块和 `backend/postprocess/table_merge`。不再有 `pipeline_union_make`/`vlm_union_make`/`office_union_make` 三套逻辑,`content_list` 格式和 `render_content_list` 函数已删除。 + +### 5.1 目录职责 + +- `model/runtime/` 负责设备、显存、ONNX 与 Hybrid 本地模型生命周期;模型仓库和下载分别位于 `model/registry.py`、`model/download.py`。 +- `model/flash/pdf/` 负责 PDFDocument、PDFium、原生文本、样式和表格恢复;`model/flash/office/` 负责六类 Office 格式。 +- `utils/` 只保留 geometry、image、image payload、language/text、platform 和 stdio 等叶子能力;活动代码不得把业务实现重新放入 utils。 +- 稳定依赖方向为 `utils/types → model → backend → render → parser/kit/doclib`,禁止反向引用。 ### 6. ParseResult 与 MiddleJson 的关系 diff --git a/tests/fixtures/evaluate_native_pdf_table_manifest.py b/tests/fixtures/evaluate_native_pdf_table_manifest.py index 58183582..e2964bbd 100644 --- a/tests/fixtures/evaluate_native_pdf_table_manifest.py +++ b/tests/fixtures/evaluate_native_pdf_table_manifest.py @@ -12,14 +12,14 @@ from collections import defaultdict from pathlib import Path, PureWindowsPath from typing import Any -from mineru.utils.native_pdf_table import ( +from mineru.model.flash.pdf.table_recovery import ( NativeTableInput, coerce_native_table_rectangles, coerce_native_table_rules, recover_native_pdf_table, ) -from mineru.utils.native_pdf_table.engine import diagnose_native_pdf_table -from mineru.utils.pdf_document import PDFDocument +from mineru.model.flash.pdf.table_recovery.engine import diagnose_native_pdf_table +from mineru.model.flash.pdf.document import PDFDocument _DEFAULT_SOURCE_ROOT = Path(__file__).resolve().parents[1] / "unittest" / "pdfs" / "native_pdf_tables" diff --git a/tests/unittest/_flash_pdf_test_utils.py b/tests/unittest/_flash_pdf_test_utils.py index a1201bee..14258197 100644 --- a/tests/unittest/_flash_pdf_test_utils.py +++ b/tests/unittest/_flash_pdf_test_utils.py @@ -2,7 +2,7 @@ from __future__ import annotations -from mineru.model.flash.native_pdf import ( +from mineru.model.flash.pdf import ( models, ) diff --git a/tests/unittest/test_backend_architecture.py b/tests/unittest/test_backend_architecture.py index f8f25443..be3fa471 100644 --- a/tests/unittest/test_backend_architecture.py +++ b/tests/unittest/test_backend_architecture.py @@ -1,38 +1,107 @@ from __future__ import annotations import ast +import importlib.util from pathlib import Path +import subprocess +import sys from mineru.backend.postprocess import table_merge _PROJECT_ROOT = Path(__file__).resolve().parents[2] _COPYRIGHT_HEADER = "# Copyright (c) Opendatalab. All rights reserved." -_CHINESE_DOCSTRING_EXTRA_PATHS = ( - "mineru/model/flash/office", - "mineru/model/flash/xycut.py", - "mineru/model/model_types.py", - "mineru/utils/native_pdf_table", - "mineru/utils/spatial_text.py", - "mineru/utils/text_utils.py", +_SOURCE_ROOTS = ( + _PROJECT_ROOT / "mineru/backend", + _PROJECT_ROOT / "mineru/model", + _PROJECT_ROOT / "mineru/render", + _PROJECT_ROOT / "mineru/utils", + _PROJECT_ROOT / "mineru/parser", +) +_HEADER_PATHS = ( + _PROJECT_ROOT / "mineru/backend", + _PROJECT_ROOT / "mineru/model/flash", + _PROJECT_ROOT / "mineru/model/runtime", + _PROJECT_ROOT / "mineru/model/registry.py", + _PROJECT_ROOT / "mineru/model/download.py", + _PROJECT_ROOT / "mineru/model/ocr/geometry.py", + _PROJECT_ROOT / "mineru/model/ocr/image.py", + _PROJECT_ROOT / "mineru/model/ocr/language.py", + _PROJECT_ROOT / "mineru/model/ocr/results.py", + _PROJECT_ROOT / "mineru/render", + _PROJECT_ROOT / "mineru/utils", + _PROJECT_ROOT / "mineru/parser/file_type.py", + _PROJECT_ROOT / "mineru/parser/page_range.py", + _PROJECT_ROOT / "mineru/parser/process_control.py", + _PROJECT_ROOT / "mineru/parser/writer.py", +) +_CHINESE_DOCSTRING_PATHS = ( + _PROJECT_ROOT / "mineru/backend", + _PROJECT_ROOT / "mineru/model/runtime/device.py", + _PROJECT_ROOT / "mineru/model/runtime/memory.py", + _PROJECT_ROOT / "mineru/model/ocr/geometry.py", + _PROJECT_ROOT / "mineru/model/ocr/image.py", + _PROJECT_ROOT / "mineru/model/ocr/results.py", + _PROJECT_ROOT / "mineru/render/markdown.py", + _PROJECT_ROOT / "mineru/render/html.py", + _PROJECT_ROOT / "mineru/render/docx.py", + _PROJECT_ROOT / "mineru/render/structured_content.py", + _PROJECT_ROOT / "mineru/utils/image.py", +) +_REMOVED_INTERNAL_MODULES = ( + "mineru.backend.local_model_runtime", + "mineru.model.model_types", + "mineru.model.flash.model", + "mineru.model.flash.native_pdf", + "mineru.model.utils", + "mineru.render.writer", + "mineru.render._internal.common.inline", + "mineru.utils.backend_options", + "mineru.utils.config_reader", + "mineru.utils.model_registry", + "mineru.utils.native_pdf_table", + "mineru.utils.ocr_utils", + "mineru.utils.pdf_document", ) -def _absolute_imports(path: Path) -> set[str]: - """读取 Python 文件中的绝对 import 模块名。""" +def _module_name(path: Path) -> str: + """把项目内 Python 路径转换为完整模块名。""" + relative = path.relative_to(_PROJECT_ROOT).with_suffix("") + parts = list(relative.parts) + if parts[-1] == "__init__": + parts.pop() + return ".".join(parts) + + +def _resolved_imports(path: Path) -> set[str]: + """把绝对和相对 import 都解析为完整模块名。""" tree = ast.parse(path.read_text(encoding="utf-8")) + module_name = _module_name(path) + package_parts = module_name.split(".") if path.name == "__init__.py" else module_name.split(".")[:-1] imports: set[str] = set() for node in ast.walk(tree): if isinstance(node, ast.Import): imports.update(alias.name for alias in node.names) - elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module: - imports.add(node.module) + continue + if not isinstance(node, ast.ImportFrom): + continue + if node.level == 0: + if node.module: + imports.add(node.module) + continue + resolved_parts = package_parts[:] + if node.level > 1: + resolved_parts = resolved_parts[: -(node.level - 1)] + if node.module: + resolved_parts.extend(node.module.split(".")) + imports.add(".".join(resolved_parts)) return imports def _relative_imports(path: Path) -> set[str]: """读取 Python 文件对同包模块的相对 import 名称。""" tree = ast.parse(path.read_text(encoding="utf-8")) - return {node.module for node in ast.walk(tree) if isinstance(node, ast.ImportFrom) and node.level > 0 and node.module} + return {node.module for node in ast.walk(tree) if isinstance(node, ast.ImportFrom) and node.level == 1 and node.module} def _contains_chinese(text: str) -> bool: @@ -40,83 +109,163 @@ def _contains_chinese(text: str) -> bool: return any("\u4e00" <= char <= "\u9fff" for char in text) -def _iter_refactored_python_paths() -> list[Path]: - """收集 backend 及本次从 utils 迁出的公共 Python 模块。""" - paths = list((_PROJECT_ROOT / "mineru/backend").rglob("*.py")) - for relative_path in _CHINESE_DOCSTRING_EXTRA_PATHS: - path = _PROJECT_ROOT / relative_path - paths.extend(path.rglob("*.py") if path.is_dir() else [path]) - return sorted(paths) +def _iter_python_paths(path: Path) -> list[Path]: + """返回文件自身或目录下全部 Python 文件。""" + return sorted(path.rglob("*.py")) if path.is_dir() else [path] -def test_backend_python_files_have_copyright_header() -> None: - """守卫 backend 下每个 Python 文件都以统一版权声明开头。""" +def test_target_python_files_have_copyright_header() -> None: + """守卫本次目标目录中的一方 Python 文件使用统一版权头。""" + paths = [ + path + for configured_path in _HEADER_PATHS + for path in _iter_python_paths(configured_path) + if "_internal/pytorchocr" not in path.as_posix() and path.name != "cli_parser.py" + ] offenders = [ - str(path.relative_to(_PROJECT_ROOT)) - for path in (_PROJECT_ROOT / "mineru/backend").rglob("*.py") - if path.read_text(encoding="utf-8").splitlines()[0] != _COPYRIGHT_HEADER + str(path.relative_to(_PROJECT_ROOT)) for path in paths if path.read_text().splitlines()[0] != _COPYRIGHT_HEADER ] assert not offenders -def test_refactored_definitions_have_chinese_docstrings() -> None: - """守卫 backend 及迁出模块中的函数、方法和类均有中文职责说明。""" +def test_new_first_party_definitions_have_chinese_docstrings() -> None: + """守卫本次新增的一方函数、方法和类都有中文职责说明。""" offenders: list[str] = [] - for path in _iter_refactored_python_paths(): + for configured_path in _CHINESE_DOCSTRING_PATHS: + for path in _iter_python_paths(configured_path): + tree = ast.parse(path.read_text(encoding="utf-8")) + for node in ast.walk(tree): + if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + continue + docstring = ast.get_docstring(node) + if not docstring or not _contains_chinese(docstring): + offenders.append(f"{path.relative_to(_PROJECT_ROOT)}:{node.lineno}:{node.name}") + assert not offenders + + +def test_active_mineru_imports_use_relative_form() -> None: + """守卫活动生产代码的 MinerU 内部引用统一使用相对 import。""" + offenders: list[str] = [] + for path in (_PROJECT_ROOT / "mineru").rglob("*.py"): + if "cli_old" in path.parts: + continue tree = ast.parse(path.read_text(encoding="utf-8")) for node in ast.walk(tree): - if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - continue - docstring = ast.get_docstring(node) - if not docstring or not _contains_chinese(docstring): - relative_path = path.relative_to(_PROJECT_ROOT) - offenders.append(f"{relative_path}:{node.lineno}:{node.name}") + if ( + isinstance(node, ast.ImportFrom) + and node.level == 0 + and (node.module == "mineru" or (node.module or "").startswith("mineru.")) + ): + offenders.append(f"{path.relative_to(_PROJECT_ROOT)}:{node.lineno}:{node.module}") + elif isinstance(node, ast.Import): + for alias in node.names: + if alias.name == "mineru" or alias.name.startswith("mineru."): + offenders.append(f"{path.relative_to(_PROJECT_ROOT)}:{node.lineno}:{alias.name}") assert not offenders -def test_model_and_types_do_not_import_backend() -> None: - """守卫模型层和公开类型层不再反向依赖 backend。""" - paths = [*_PROJECT_ROOT.glob("mineru/model/**/*.py"), _PROJECT_ROOT / "mineru/types.py"] - offenders = { - str(path.relative_to(_PROJECT_ROOT)): sorted( - module for module in _absolute_imports(path) if module.startswith("mineru.backend") +def test_layer_dependencies_are_one_way() -> None: + """守卫 utils、model、backend 与 render 的单向依赖边界。""" + offenders: dict[str, list[str]] = {} + model_paths = [*_PROJECT_ROOT.glob("mineru/model/**/*.py"), _PROJECT_ROOT / "mineru/types.py"] + for path in model_paths: + invalid = sorted(module for module in _resolved_imports(path) if module.startswith(("mineru.backend", "mineru.render"))) + if invalid: + offenders[str(path.relative_to(_PROJECT_ROOT))] = invalid + for path in _PROJECT_ROOT.glob("mineru/backend/**/*.py"): + invalid = sorted(module for module in _resolved_imports(path) if module.startswith("mineru.render")) + if invalid: + offenders[str(path.relative_to(_PROJECT_ROOT))] = invalid + for path in _PROJECT_ROOT.glob("mineru/utils/**/*.py"): + invalid = sorted( + module + for module in _resolved_imports(path) + if module.startswith(("mineru.backend", "mineru.model", "mineru.render", "mineru.parser")) ) - for path in paths - } - assert not {path: modules for path, modules in offenders.items() if modules} - - -def test_analysis_does_not_import_postprocess() -> None: - """守卫 model-list 生产层不反向调用 Middle JSON 后处理层。""" - offenders = { - str(path.relative_to(_PROJECT_ROOT)): sorted( - module for module in _absolute_imports(path) if module.startswith("mineru.backend.postprocess") + if invalid: + offenders[str(path.relative_to(_PROJECT_ROOT))] = invalid + for path in _PROJECT_ROOT.glob("mineru/backend/analysis/**/*.py"): + invalid = sorted(module for module in _resolved_imports(path) if module.startswith("mineru.backend.postprocess")) + if invalid: + offenders[str(path.relative_to(_PROJECT_ROOT))] = invalid + allowed_render_backend = ( + "mineru.backend.postprocess.inline", + "mineru.backend.postprocess.table_merge", + ) + for path in _PROJECT_ROOT.glob("mineru/render/**/*.py"): + invalid = sorted( + module + for module in _resolved_imports(path) + if module.startswith("mineru.backend") + and not any(module == allowed or module.startswith(f"{allowed}.") for allowed in allowed_render_backend) ) - for path in _PROJECT_ROOT.glob("mineru/backend/analysis/**/*.py") - } - assert not {path: modules for path, modules in offenders.items() if modules} + if invalid: + offenders[str(path.relative_to(_PROJECT_ROOT))] = invalid + assert not offenders -def test_backend_utils_package_is_removed() -> None: - """守卫 backend/utils 不再承载任何 Python 源码。""" - utils_path = _PROJECT_ROOT / "mineru/backend/utils" - assert not utils_path.exists() - - -def test_llm_postprocess_business_logic_is_not_kept_in_utils() -> None: - """守卫 LLM 客户端和标题业务统一归属 backend/postprocess。""" - removed_utils_paths = [ - _PROJECT_ROOT / "mineru/utils/llm_aided.py", - _PROJECT_ROOT / "mineru/utils/title_level_postprocess.py", - ] - expected_backend_paths = [ - _PROJECT_ROOT / "mineru/backend/postprocess/llm_client.py", - _PROJECT_ROOT / "mineru/backend/postprocess/title_leveling.py", - _PROJECT_ROOT / "mineru/backend/postprocess/table_merge/llm_cell_merge.py", +def test_package_initializers_define_explicit_all() -> None: + """守卫目标目录下每个包入口都显式声明 __all__。""" + offenders = [ + str(path.relative_to(_PROJECT_ROOT)) + for root in _SOURCE_ROOTS + for path in root.rglob("__init__.py") + if "__all__" not in path.read_text(encoding="utf-8") ] + assert not offenders - assert not [path for path in removed_utils_paths if path.exists()] - assert all(path.is_file() for path in expected_backend_paths) + +def test_removed_private_module_paths_have_no_active_references() -> None: + """守卫严格迁移后的活动生产代码不再引用旧私有路径。""" + offenders: list[str] = [] + for path in (_PROJECT_ROOT / "mineru").rglob("*.py"): + if "cli_old" in path.parts: + continue + text = path.read_text(encoding="utf-8") + for module_name in _REMOVED_INTERNAL_MODULES: + if module_name in text: + offenders.append(f"{path.relative_to(_PROJECT_ROOT)}:{module_name}") + assert not offenders + + +def test_removed_private_module_paths_are_not_importable() -> None: + """验证严格切换后不存在可被误用的旧私有模块壳。""" + offenders: list[str] = [] + for module_name in _REMOVED_INTERNAL_MODULES: + try: + spec = importlib.util.find_spec(module_name) + except ModuleNotFoundError: + spec = None + if spec is not None: + offenders.append(module_name) + assert not offenders + + +def test_public_facade_imports_do_not_load_heavy_dependencies_or_mutate_env() -> None: + """验证三个稳定门面导入时不加载重依赖或修改环境变量。""" + script = """ +import os +import sys + +before_env = dict(os.environ) +import mineru.backend.analyze +import mineru.render +from mineru.model.flash import PdfModel + +assert PdfModel.__name__ == "PdfModel" +for prefix in ("torch", "cv2", "pypdfium2", "pdftext", "docx", "bs4", "lxml", "nh3"): + assert prefix not in sys.modules, prefix + assert not any(name.startswith(prefix + ".") for name in sys.modules), prefix +assert before_env == dict(os.environ) +""" + result = subprocess.run( + [sys.executable, "-c", script], + cwd=_PROJECT_ROOT, + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stderr def test_table_merge_package_keeps_one_way_internal_dependencies() -> None: diff --git a/tests/unittest/test_doclib_app_startup.py b/tests/unittest/test_doclib_app_startup.py index bfa42644..46922a29 100644 --- a/tests/unittest/test_doclib_app_startup.py +++ b/tests/unittest/test_doclib_app_startup.py @@ -16,8 +16,8 @@ from mineru.doclib import app as doclib_app from mineru.doclib.app import _assert_required_schema from mineru.doclib.core.db import DatabaseManager from mineru.doclib.server import _tail_log, _write_temp_asset +from mineru.model import download as model_download from mineru.parser import tier as parser_tier -from mineru.utils import model_registry from mineru.version import __version__ @@ -138,7 +138,7 @@ def test_config_set_managed_tier_rejects_missing_models(monkeypatch: pytest.Monk monkeypatch.setattr(doclib_app, "_create_background_task", _skip_background_task) monkeypatch.setattr("mineru.doclib.server.ensure_tier_runtime_dependencies", lambda tier: None) - monkeypatch.setattr(model_registry.config.model, "base_dir", str(tmp_path / "models")) + monkeypatch.setattr(model_download.config.model, "base_dir", str(tmp_path / "models")) cfg = PatchedConfig(doclib={"data_dir": str(tmp_path), "sqlite": {"path": str(tmp_path / "doclib.db")}}) with TestClient(doclib_app.create_app(cfg)) as client: @@ -161,7 +161,7 @@ def test_config_set_managed_advanced_is_rejected(monkeypatch: pytest.MonkeyPatch monkeypatch.setattr(doclib_app, "_create_background_task", _skip_background_task) monkeypatch.setattr("mineru.doclib.server.ensure_tier_runtime_dependencies", lambda tier: None) - monkeypatch.setattr(model_registry.config.model, "base_dir", str(tmp_path / "models")) + monkeypatch.setattr(model_download.config.model, "base_dir", str(tmp_path / "models")) cfg = PatchedConfig(doclib={"data_dir": str(tmp_path), "sqlite": {"path": str(tmp_path / "doclib.db")}}) with TestClient(doclib_app.create_app(cfg)) as client: @@ -181,7 +181,7 @@ def test_config_set_managed_mode_rejects_missing_models_for_current_tier( monkeypatch.setattr(doclib_app, "_create_background_task", _skip_background_task) monkeypatch.setattr("mineru.doclib.server.ensure_tier_runtime_dependencies", lambda tier: None) - monkeypatch.setattr(model_registry.config.model, "base_dir", str(tmp_path / "models")) + monkeypatch.setattr(model_download.config.model, "base_dir", str(tmp_path / "models")) cfg = PatchedConfig(doclib={"data_dir": str(tmp_path), "sqlite": {"path": str(tmp_path / "doclib.db")}}) with TestClient(doclib_app.create_app(cfg)) as client: diff --git a/tests/unittest/test_docx_special_char_tables.py b/tests/unittest/test_docx_special_char_tables.py index 3c681cac..4afa750f 100644 --- a/tests/unittest/test_docx_special_char_tables.py +++ b/tests/unittest/test_docx_special_char_tables.py @@ -5,7 +5,7 @@ from docx import Document from docx.oxml.ns import qn from lxml import etree -from mineru.model.flash.docx.docx_converter import DocxConverter +from mineru.model.flash.office.docx.docx_converter import DocxConverter NO_BREAK_HYPHEN = "‑" diff --git a/tests/unittest/test_flash_docx_equationxml.py b/tests/unittest/test_flash_docx_equationxml.py index f34758d9..67399501 100644 --- a/tests/unittest/test_flash_docx_equationxml.py +++ b/tests/unittest/test_flash_docx_equationxml.py @@ -9,15 +9,15 @@ import pytest from mineru.backend.analyze import aio_doc_analyze, doc_analyze from mineru.model.flash import DocxModel -from mineru.model.flash.docx.docx_converter import DocxConverter -from mineru.model.flash.docx.equationxml import DocxEquationXmlDecoder -from mineru.model.flash.legacy_office.errors import ( +from mineru.model.flash.office.docx.docx_converter import DocxConverter +from mineru.model.flash.office.docx.equationxml import DocxEquationXmlDecoder +from mineru.model.flash.office.legacy.errors import ( LegacyOfficeResourceLimitError, ) from mineru.render._internal.docx.math import latex_to_omml from mineru.types import BlockType, MiddleJson, ModelJson -import mineru.model.flash.docx.equationxml as equationxml_module +import mineru.model.flash.office.docx.equationxml as equationxml_module from _docx_equationxml_test_utils import ( M_NS, WORD_2003_NS, diff --git a/tests/unittest/test_flash_image_mtef.py b/tests/unittest/test_flash_image_mtef.py index 2b8f5be7..e8d22170 100644 --- a/tests/unittest/test_flash_image_mtef.py +++ b/tests/unittest/test_flash_image_mtef.py @@ -2,7 +2,7 @@ from __future__ import annotations import pytest -from mineru.model.flash.legacy_office.errors import LegacyOfficeResourceLimitError +from mineru.model.flash.office.legacy.errors import LegacyOfficeResourceLimitError from mineru.model.flash.office import image_equation as image_equation_module from mineru.model.flash.office.image_equation import ( OfficeImageEquationDecoder, diff --git a/tests/unittest/test_flash_legacy_chart_data.py b/tests/unittest/test_flash_legacy_chart_data.py index 660b3713..af2f187f 100644 --- a/tests/unittest/test_flash_legacy_chart_data.py +++ b/tests/unittest/test_flash_legacy_chart_data.py @@ -6,9 +6,9 @@ import zlib from bs4 import BeautifulSoup from mineru.backend.analyze import doc_analyze -from mineru.model.flash.ppt import parser as ppt_parser -from mineru.model.flash.ppt.records import PptRecord -from mineru.model.flash.xls.embedded_chart import extract_embedded_chart_html +from mineru.model.flash.office.ppt import parser as ppt_parser +from mineru.model.flash.office.ppt.records import PptRecord +from mineru.model.flash.office.xls.embedded_chart import extract_embedded_chart_html from mineru.types import BlockType, ChartBlock from _legacy_ppt_test_utils import _build_cfb diff --git a/tests/unittest/test_flash_legacy_doc.py b/tests/unittest/test_flash_legacy_doc.py index 019403f6..9d090b3e 100644 --- a/tests/unittest/test_flash_legacy_doc.py +++ b/tests/unittest/test_flash_legacy_doc.py @@ -11,12 +11,12 @@ import pytest from mineru.backend.analyze import aio_doc_analyze, doc_analyze from mineru.backend.postprocess.lists import fix_office_list_blocks from mineru.model.flash import DocModel -from mineru.model.flash.doc.fields import sanitize_hyperlink_target -from mineru.model.flash.doc.models import DocCharStyle, DocTableCell -from mineru.model.flash.doc.parser import _RawTableRow, _materialize_table_rows -from mineru.model.flash.doc.records import DocBudget -from mineru.model.flash.doc.sprm import apply_character_sprms -from mineru.model.flash.legacy_office import ( +from mineru.model.flash.office.doc.fields import sanitize_hyperlink_target +from mineru.model.flash.office.doc.models import DocCharStyle, DocTableCell +from mineru.model.flash.office.doc.parser import _RawTableRow, _materialize_table_rows +from mineru.model.flash.office.doc.records import DocBudget +from mineru.model.flash.office.doc.sprm import apply_character_sprms +from mineru.model.flash.office.legacy import ( LegacyOfficeEncryptedError, LegacyOfficeMalformedError, LegacyOfficeMissingPartError, @@ -170,7 +170,7 @@ def test_doc_exact_list_label_is_consumed_before_strict_projection() -> None: def test_doc_table_grid_materializes_colspan_and_rowspan() -> None: """验证 Word table edge 网格能同时恢复横向和纵向合并。""" - from mineru.model.flash.doc.models import DocTableCellFormat, DocTableFormat + from mineru.model.flash.office.doc.models import DocTableCellFormat, DocTableFormat first = DocTableCell(blocks=[]) raw_rows = [ @@ -218,7 +218,7 @@ def test_doc_rejects_word95_encryption_rtf_and_missing_word_stream() -> None: def test_doc_budget_uses_stable_resource_limit(monkeypatch: pytest.MonkeyPatch) -> None: """验证 DOC 记录预算超过固定上限时使用共享错误类型。""" - import mineru.model.flash.doc.records as records + import mineru.model.flash.office.doc.records as records monkeypatch.setattr(records, "MAX_RECORDS", 1) budget = records.DocBudget() diff --git a/tests/unittest/test_flash_legacy_doc_mtef.py b/tests/unittest/test_flash_legacy_doc_mtef.py index c5dbf1fb..c037d680 100644 --- a/tests/unittest/test_flash_legacy_doc_mtef.py +++ b/tests/unittest/test_flash_legacy_doc_mtef.py @@ -7,7 +7,7 @@ import pytest from mineru.backend.analyze import aio_doc_analyze, doc_analyze from mineru.model.flash import DocModel -from mineru.model.flash.legacy_office.mtef import ( +from mineru.model.flash.office.legacy.mtef import ( decode_equation_native, decode_equation_object, decode_mtef_v3, diff --git a/tests/unittest/test_flash_legacy_ppt.py b/tests/unittest/test_flash_legacy_ppt.py index 181d753d..8acdf329 100644 --- a/tests/unittest/test_flash_legacy_ppt.py +++ b/tests/unittest/test_flash_legacy_ppt.py @@ -11,12 +11,12 @@ from pydantic import ValidationError from mineru.backend.analyze import aio_doc_analyze, doc_analyze from mineru.model.flash import PptModel -from mineru.model.flash.legacy_office import LegacyOfficeEncryptedError, LegacyOfficeResourceLimitError -from mineru.model.flash.ppt import parser as ppt_parser -from mineru.model.flash.ppt.models import PptPresentation, PptSlide -from mineru.model.flash.ppt.ppt_converter import PptConverter -from mineru.model.flash.ppt.records import PptRecord, RecordBudget -from mineru.model.flash.ppt.style_text import CharacterRun, StyleRuns +from mineru.model.flash.office.legacy import LegacyOfficeEncryptedError, LegacyOfficeResourceLimitError +from mineru.model.flash.office.ppt import parser as ppt_parser +from mineru.model.flash.office.ppt.models import PptPresentation, PptSlide +from mineru.model.flash.office.ppt.ppt_converter import PptConverter +from mineru.model.flash.office.ppt.records import PptRecord, RecordBudget +from mineru.model.flash.office.ppt.style_text import CharacterRun, StyleRuns from mineru.parser import parse from mineru.types import BlockType, ChartBlock, ImageBlock, MiddleJson, ModelJson, TableBlock diff --git a/tests/unittest/test_flash_legacy_xls.py b/tests/unittest/test_flash_legacy_xls.py index 460e314a..d141d684 100644 --- a/tests/unittest/test_flash_legacy_xls.py +++ b/tests/unittest/test_flash_legacy_xls.py @@ -10,16 +10,16 @@ import pytest from mineru.backend.analyze import aio_doc_analyze, doc_analyze from mineru.model.flash import XlsModel -from mineru.model.flash.legacy_office import ( +from mineru.model.flash.office.legacy import ( LegacyOfficeEncryptedError, LegacyOfficeMissingPartError, LegacyOfficeResourceLimitError, ) -from mineru.model.flash.legacy_office.limits import MAX_RECORDS -from mineru.model.flash.xls import xls_converter as xls_converter_module -from mineru.model.flash.xls import parser as xls_parser -from mineru.model.flash.xls.number_format import format_number, format_text -from mineru.model.flash.xls.records import RecordBudget +from mineru.model.flash.office.legacy.limits import MAX_RECORDS +from mineru.model.flash.office.xls import xls_converter as xls_converter_module +from mineru.model.flash.office.xls import parser as xls_parser +from mineru.model.flash.office.xls.number_format import format_number, format_text +from mineru.model.flash.office.xls.records import RecordBudget from mineru.parser import parse from mineru.types import BlockType, ImageBlock, MiddleJson, ModelJson, TableBlock diff --git a/tests/unittest/test_flash_legacy_xls_ppt_mtef.py b/tests/unittest/test_flash_legacy_xls_ppt_mtef.py index fdca80eb..07e15fa7 100644 --- a/tests/unittest/test_flash_legacy_xls_ppt_mtef.py +++ b/tests/unittest/test_flash_legacy_xls_ppt_mtef.py @@ -8,8 +8,8 @@ import pytest from mineru.backend.analyze import aio_doc_analyze, doc_analyze from mineru.model.flash import PptModel, XlsModel -from mineru.model.flash.legacy_office import LegacyOfficeResourceLimitError -from mineru.model.flash.legacy_office.limits import MAX_ENTRY_BYTES +from mineru.model.flash.office.legacy import LegacyOfficeResourceLimitError +from mineru.model.flash.office.legacy.limits import MAX_ENTRY_BYTES from mineru.types import BlockType, MiddleJson, ModelJson from _legacy_ppt_test_utils import build_equation_ppt diff --git a/tests/unittest/test_flash_model.py b/tests/unittest/test_flash_model.py index 7591250d..c8b91c90 100644 --- a/tests/unittest/test_flash_model.py +++ b/tests/unittest/test_flash_model.py @@ -6,8 +6,8 @@ from unittest.mock import MagicMock import pytest from mineru.model.flash import PdfModel -from mineru.model.flash.native_pdf import pipeline -from mineru.utils.pdf_document import PDFDocument +from mineru.model.flash.pdf import pipeline +from mineru.model.flash.pdf.document import PDFDocument def test_pdf_model_predict_returns_native_model_list_without_owning_document( diff --git a/tests/unittest/test_flash_mtef_v5.py b/tests/unittest/test_flash_mtef_v5.py index 44d2f102..ace841a3 100644 --- a/tests/unittest/test_flash_mtef_v5.py +++ b/tests/unittest/test_flash_mtef_v5.py @@ -4,10 +4,10 @@ import struct import pytest -from mineru.model.flash.legacy_office import mtef_v5 as mtef_v5_module -from mineru.model.flash.legacy_office import mtef as mtef_module -from mineru.model.flash.legacy_office.errors import LegacyOfficeResourceLimitError -from mineru.model.flash.legacy_office.mtef import ( +from mineru.model.flash.office.legacy import mtef_v5 as mtef_v5_module +from mineru.model.flash.office.legacy import mtef as mtef_module +from mineru.model.flash.office.legacy.errors import LegacyOfficeResourceLimitError +from mineru.model.flash.office.legacy.mtef import ( decode_equation_native, decode_equation_object, decode_mtef, diff --git a/tests/unittest/test_flash_office_image_mtef.py b/tests/unittest/test_flash_office_image_mtef.py index 37d87116..ad1976cc 100644 --- a/tests/unittest/test_flash_office_image_mtef.py +++ b/tests/unittest/test_flash_office_image_mtef.py @@ -17,9 +17,9 @@ from mineru.model.flash import ( XlsModel, XlsxModel, ) -from mineru.model.flash.doc.doc_converter import DocConverter -from mineru.model.flash.docx.docx_converter import DocxConverter -from mineru.model.flash.doc.models import ( +from mineru.model.flash.office.doc.doc_converter import DocConverter +from mineru.model.flash.office.docx.docx_converter import DocxConverter +from mineru.model.flash.office.doc.models import ( DocImage, DocImagePayload, DocParagraph, @@ -27,8 +27,8 @@ from mineru.model.flash.doc.models import ( DocTableCell, DocTableRow, ) -from mineru.model.flash.xlsx.xlsx_converter import XlsxConverter -from mineru.model.flash.pptx.pptx_converter import PptxConverter +from mineru.model.flash.office.xlsx.xlsx_converter import XlsxConverter +from mineru.model.flash.office.pptx.pptx_converter import PptxConverter from mineru.types import BlockType, MiddleJson, ModelJson from _docx_equationxml_test_utils import ( diff --git a/tests/unittest/test_flash_office_models.py b/tests/unittest/test_flash_office_models.py index 11aa7512..f6b94d22 100644 --- a/tests/unittest/test_flash_office_models.py +++ b/tests/unittest/test_flash_office_models.py @@ -11,17 +11,17 @@ from unittest.mock import MagicMock import pytest import mineru.model.flash as flash_models -import mineru.model.flash.model as flat_model_module +import mineru.model.flash.models as flat_model_module from mineru.model.flash import DocModel, DocxModel, PdfModel, PptModel, PptxModel, XlsModel, XlsxModel -from mineru.model.flash.doc import doc_converter as doc_converter_module -from mineru.model.flash.docx import docx_converter as docx_converter_module -from mineru.model.flash.docx import main as docx_main -from mineru.model.flash.pptx import main as pptx_main -from mineru.model.flash.pptx import pptx_converter as pptx_converter_module -from mineru.model.flash.ppt import ppt_converter as ppt_converter_module -from mineru.model.flash.xls import xls_converter as xls_converter_module -from mineru.model.flash.xlsx import main as xlsx_main -from mineru.model.flash.xlsx import xlsx_converter as xlsx_converter_module +from mineru.model.flash.office.doc import doc_converter as doc_converter_module +from mineru.model.flash.office.docx import docx_converter as docx_converter_module +from mineru.model.flash.office.docx import main as docx_main +from mineru.model.flash.office.pptx import main as pptx_main +from mineru.model.flash.office.pptx import pptx_converter as pptx_converter_module +from mineru.model.flash.office.ppt import ppt_converter as ppt_converter_module +from mineru.model.flash.office.xls import xls_converter as xls_converter_module +from mineru.model.flash.office.xlsx import main as xlsx_main +from mineru.model.flash.office.xlsx import xlsx_converter as xlsx_converter_module _PROJECT_ROOT = Path(__file__).resolve().parents[2] @@ -174,12 +174,12 @@ def test_models_are_exported_from_flash_root() -> None: @pytest.mark.parametrize( ("package_name", "model_name"), [ - ("mineru.model.flash.doc", "DocModel"), - ("mineru.model.flash.docx", "DocxModel"), - ("mineru.model.flash.pptx", "PptxModel"), - ("mineru.model.flash.ppt", "PptModel"), - ("mineru.model.flash.xls", "XlsModel"), - ("mineru.model.flash.xlsx", "XlsxModel"), + ("mineru.model.flash.office.doc", "DocModel"), + ("mineru.model.flash.office.docx", "DocxModel"), + ("mineru.model.flash.office.pptx", "PptxModel"), + ("mineru.model.flash.office.ppt", "PptModel"), + ("mineru.model.flash.office.xls", "XlsModel"), + ("mineru.model.flash.office.xlsx", "XlsxModel"), ], ) def test_office_subpackages_do_not_export_models(package_name: str, model_name: str) -> None: @@ -198,13 +198,13 @@ def test_importing_pdf_model_does_not_load_office_converters() -> None: "import sys", "from mineru.model.flash import PdfModel", "assert PdfModel.__name__ == 'PdfModel'", - "assert 'mineru.model.flash.docx.docx_converter' not in sys.modules", - "assert 'mineru.model.flash.doc.doc_converter' not in sys.modules", - "assert 'mineru.model.flash.pptx.pptx_converter' not in sys.modules", - "assert 'mineru.model.flash.ppt.ppt_converter' not in sys.modules", - "assert 'mineru.model.flash.xls.xls_converter' not in sys.modules", + "assert 'mineru.model.flash.office.docx.docx_converter' not in sys.modules", + "assert 'mineru.model.flash.office.doc.doc_converter' not in sys.modules", + "assert 'mineru.model.flash.office.pptx.pptx_converter' not in sys.modules", + "assert 'mineru.model.flash.office.ppt.ppt_converter' not in sys.modules", + "assert 'mineru.model.flash.office.xls.xls_converter' not in sys.modules", "assert 'olefile' not in sys.modules", - "assert 'mineru.model.flash.xlsx.xlsx_converter' not in sys.modules", + "assert 'mineru.model.flash.office.xlsx.xlsx_converter' not in sys.modules", ] ) result = subprocess.run( diff --git a/tests/unittest/test_flash_office_mtef_v5.py b/tests/unittest/test_flash_office_mtef_v5.py index 754a43fb..7eda40f7 100644 --- a/tests/unittest/test_flash_office_mtef_v5.py +++ b/tests/unittest/test_flash_office_mtef_v5.py @@ -15,9 +15,9 @@ from mineru.model.flash import ( XlsModel, XlsxModel, ) -from mineru.model.flash.docx.docx_converter import DocxConverter -from mineru.model.flash.pptx.pptx_converter import PptxConverter -from mineru.model.flash.xlsx.xlsx_converter import XlsxConverter +from mineru.model.flash.office.docx.docx_converter import DocxConverter +from mineru.model.flash.office.pptx.pptx_converter import PptxConverter +from mineru.model.flash.office.xlsx.xlsx_converter import XlsxConverter from mineru.types import BlockType, MiddleJson, ModelJson from _docx_equationxml_test_utils import ( diff --git a/tests/unittest/test_flash_office_title_types.py b/tests/unittest/test_flash_office_title_types.py index ce6e45a2..b31208a3 100644 --- a/tests/unittest/test_flash_office_title_types.py +++ b/tests/unittest/test_flash_office_title_types.py @@ -9,7 +9,7 @@ from pptx import Presentation from pptx.enum.shapes import PP_PLACEHOLDER from mineru.model.flash import DocxModel, PptxModel, XlsxModel -from mineru.model.flash.pptx.pptx_converter import ( +from mineru.model.flash.office.pptx.pptx_converter import ( PptxConverter, _EFFECTIVE_ALL_BOLD_KEY, _EFFECTIVE_FONT_SIZE_KEY, diff --git a/tests/unittest/test_flash_ooxml_mtef.py b/tests/unittest/test_flash_ooxml_mtef.py index 3f9b80a5..3734b0e2 100644 --- a/tests/unittest/test_flash_ooxml_mtef.py +++ b/tests/unittest/test_flash_ooxml_mtef.py @@ -10,15 +10,15 @@ import pytest from mineru.backend.analyze import aio_doc_analyze, doc_analyze from mineru.model.flash import DocxModel, PptxModel, XlsxModel -from mineru.model.flash.docx.docx_converter import DocxConverter -from mineru.model.flash.legacy_office import LegacyOfficeResourceLimitError -from mineru.model.flash.legacy_office.limits import MAX_ASSET_TOTAL_BYTES +from mineru.model.flash.office.docx.docx_converter import DocxConverter +from mineru.model.flash.office.legacy import LegacyOfficeResourceLimitError +from mineru.model.flash.office.legacy.limits import MAX_ASSET_TOTAL_BYTES from mineru.model.flash.office.ooxml_equation import ( OoxmlEquationDecoder, is_mathtype_equation_prog_id, ) -from mineru.model.flash.pptx.pptx_converter import PptxConverter -from mineru.model.flash.xlsx.xlsx_converter import XlsxConverter +from mineru.model.flash.office.pptx.pptx_converter import PptxConverter +from mineru.model.flash.office.xlsx.xlsx_converter import XlsxConverter from mineru.types import BlockType, MiddleJson, ModelJson from _mtef_test_utils import build_equation_object, formula_corpus diff --git a/tests/unittest/test_flash_pdf_auxiliary_text.py b/tests/unittest/test_flash_pdf_auxiliary_text.py index 29bdd6b3..54718274 100644 --- a/tests/unittest/test_flash_pdf_auxiliary_text.py +++ b/tests/unittest/test_flash_pdf_auxiliary_text.py @@ -4,7 +4,7 @@ import inspect import pytest -from mineru.model.flash.native_pdf import ( +from mineru.model.flash.pdf import ( auxiliary_text, models, pipeline, diff --git a/tests/unittest/test_flash_pdf_code_blocks.py b/tests/unittest/test_flash_pdf_code_blocks.py index 10dc052f..ff43fcff 100644 --- a/tests/unittest/test_flash_pdf_code_blocks.py +++ b/tests/unittest/test_flash_pdf_code_blocks.py @@ -6,13 +6,13 @@ from typing import Any import pytest from mineru.model.flash import PdfModel -from mineru.model.flash.native_pdf import code_blocks, models, pipeline -from mineru.render._internal.common.inline import ( +from mineru.model.flash.pdf import code_blocks, models, pipeline +from mineru.backend.postprocess.inline import ( inline_plain_text, parse_inline_content, ) -from mineru.utils.pdf_document import PDFDocument, PDFPathInfo -from mineru.utils.spatial_text import project_pdf_spatial_text +from mineru.model.flash.pdf.document import PDFDocument, PDFPathInfo +from mineru.model.flash.pdf.spatial_text import project_pdf_spatial_text from _flash_pdf_test_utils import _text_line diff --git a/tests/unittest/test_flash_pdf_document_profile.py b/tests/unittest/test_flash_pdf_document_profile.py index 0b9e86c4..df2647b7 100644 --- a/tests/unittest/test_flash_pdf_document_profile.py +++ b/tests/unittest/test_flash_pdf_document_profile.py @@ -1,6 +1,6 @@ from __future__ import annotations -from mineru.model.flash.native_pdf import titles +from mineru.model.flash.pdf import titles from _flash_pdf_test_utils import _prepared_text_page, _text_line diff --git a/tests/unittest/test_flash_pdf_extractor.py b/tests/unittest/test_flash_pdf_extractor.py index 7e3e5fdf..0c29f71d 100644 --- a/tests/unittest/test_flash_pdf_extractor.py +++ b/tests/unittest/test_flash_pdf_extractor.py @@ -1,4 +1,4 @@ -from mineru.model.flash.native_pdf import pipeline +from mineru.model.flash.pdf import pipeline def test_marginal_header_row_does_not_break_two_column_reading_order() -> None: diff --git a/tests/unittest/test_flash_pdf_formulas.py b/tests/unittest/test_flash_pdf_formulas.py index f3fae5fd..87e9316b 100644 --- a/tests/unittest/test_flash_pdf_formulas.py +++ b/tests/unittest/test_flash_pdf_formulas.py @@ -5,13 +5,13 @@ from dataclasses import replace import pytest -from mineru.model.flash.native_pdf import ( +from mineru.model.flash.pdf import ( formulas, geometry, line_merging, models, ) -from mineru.utils.pdf_document import PDFPathInfo +from mineru.model.flash.pdf.document import PDFPathInfo from _flash_pdf_test_utils import ( diff --git a/tests/unittest/test_flash_pdf_graphics.py b/tests/unittest/test_flash_pdf_graphics.py index 947057aa..430827ff 100644 --- a/tests/unittest/test_flash_pdf_graphics.py +++ b/tests/unittest/test_flash_pdf_graphics.py @@ -5,12 +5,12 @@ import inspect import pytest -from mineru.model.flash.native_pdf import ( +from mineru.model.flash.pdf import ( graphics, models, pipeline, ) -from mineru.utils.pdf_document import PDFPathInfo +from mineru.model.flash.pdf.document import PDFPathInfo from _flash_pdf_test_utils import ( diff --git a/tests/unittest/test_flash_pdf_index_blocks.py b/tests/unittest/test_flash_pdf_index_blocks.py index 9bed081d..f9509d0f 100644 --- a/tests/unittest/test_flash_pdf_index_blocks.py +++ b/tests/unittest/test_flash_pdf_index_blocks.py @@ -2,7 +2,7 @@ from __future__ import annotations import pytest -from mineru.model.flash.native_pdf import index_blocks +from mineru.model.flash.pdf import index_blocks from _flash_pdf_test_utils import _text_line diff --git a/tests/unittest/test_flash_pdf_line_metadata.py b/tests/unittest/test_flash_pdf_line_metadata.py index aba938a9..f60f0388 100644 --- a/tests/unittest/test_flash_pdf_line_metadata.py +++ b/tests/unittest/test_flash_pdf_line_metadata.py @@ -2,7 +2,7 @@ from __future__ import annotations import pytest -from mineru.model.flash.native_pdf import pipeline +from mineru.model.flash.pdf import pipeline @pytest.mark.parametrize( diff --git a/tests/unittest/test_flash_pdf_native_text.py b/tests/unittest/test_flash_pdf_native_text.py index c5b780da..ada99901 100644 --- a/tests/unittest/test_flash_pdf_native_text.py +++ b/tests/unittest/test_flash_pdf_native_text.py @@ -5,7 +5,7 @@ from typing import Any import pytest -from mineru.model.flash.native_pdf import native_text +from mineru.model.flash.pdf import native_text def _span( diff --git a/tests/unittest/test_flash_pdf_pipeline.py b/tests/unittest/test_flash_pdf_pipeline.py index d56249b1..f4db81c3 100644 --- a/tests/unittest/test_flash_pdf_pipeline.py +++ b/tests/unittest/test_flash_pdf_pipeline.py @@ -6,7 +6,7 @@ import inspect import pytest from mineru.model.flash import PdfModel -from mineru.model.flash.native_pdf import ( +from mineru.model.flash.pdf import ( auxiliary_text, formulas, geometry, @@ -22,7 +22,7 @@ from mineru.model.flash.native_pdf import ( titles, visual_annotations, ) -from mineru.utils.pdf_document import PDFImageInfo +from mineru.model.flash.pdf.document import PDFImageInfo from _flash_pdf_test_utils import ( _prepared_text_page, diff --git a/tests/unittest/test_flash_pdf_samples.py b/tests/unittest/test_flash_pdf_samples.py index 5ff2d83c..3c9ac525 100644 --- a/tests/unittest/test_flash_pdf_samples.py +++ b/tests/unittest/test_flash_pdf_samples.py @@ -13,7 +13,7 @@ from pypdf import PdfReader from mineru.backend.postprocess.page_blocks import process_page_blocks from mineru.backend.postprocess.pages import model_json_to_pages from mineru.model.flash import PdfModel -from mineru.model.flash.native_pdf import ( +from mineru.model.flash.pdf import ( formulas, geometry, graphics, @@ -23,12 +23,12 @@ from mineru.model.flash.native_pdf import ( tables, ) from mineru.render import render_markdown -from mineru.render._internal.common.inline import ( +from mineru.backend.postprocess.inline import ( inline_plain_text, parse_inline_content, ) from mineru.types import MiddleJson, ModelJson -from mineru.utils.pdf_document import PDFDocument, get_lines_from_chars +from mineru.model.flash.pdf.document import PDFDocument, get_lines_from_chars _PROJECT_ROOT = Path(__file__).parents[2] diff --git a/tests/unittest/test_flash_pdf_tables.py b/tests/unittest/test_flash_pdf_tables.py index 7ab9046f..19c54858 100644 --- a/tests/unittest/test_flash_pdf_tables.py +++ b/tests/unittest/test_flash_pdf_tables.py @@ -4,12 +4,12 @@ from unittest.mock import MagicMock import pytest -from mineru.model.flash.native_pdf import ( +from mineru.model.flash.pdf import ( geometry, models, tables, ) -from mineru.utils.pdf_document import PDFPathInfo +from mineru.model.flash.pdf.document import PDFPathInfo def _axis_line( diff --git a/tests/unittest/test_flash_pdf_text_blocks.py b/tests/unittest/test_flash_pdf_text_blocks.py index b25cd8c0..618fc0ef 100644 --- a/tests/unittest/test_flash_pdf_text_blocks.py +++ b/tests/unittest/test_flash_pdf_text_blocks.py @@ -3,7 +3,7 @@ from __future__ import annotations import pytest -from mineru.model.flash.native_pdf import ( +from mineru.model.flash.pdf import ( line_layout, line_merging, models, diff --git a/tests/unittest/test_flash_pdf_titles.py b/tests/unittest/test_flash_pdf_titles.py index e9ee329d..a182c90d 100644 --- a/tests/unittest/test_flash_pdf_titles.py +++ b/tests/unittest/test_flash_pdf_titles.py @@ -4,7 +4,7 @@ import inspect import pytest -from mineru.model.flash.native_pdf import ( +from mineru.model.flash.pdf import ( line_layout, line_merging, text_blocks, diff --git a/tests/unittest/test_flash_pdf_visual_annotations.py b/tests/unittest/test_flash_pdf_visual_annotations.py index cf287eba..fecaf7e8 100644 --- a/tests/unittest/test_flash_pdf_visual_annotations.py +++ b/tests/unittest/test_flash_pdf_visual_annotations.py @@ -2,7 +2,7 @@ from __future__ import annotations import pytest -from mineru.model.flash.native_pdf import ( +from mineru.model.flash.pdf import ( geometry, pipeline, text_blocks, diff --git a/tests/unittest/test_guess_suffix_ole2.py b/tests/unittest/test_guess_suffix_ole2.py index 429be9e0..1a14a916 100644 --- a/tests/unittest/test_guess_suffix_ole2.py +++ b/tests/unittest/test_guess_suffix_ole2.py @@ -1,6 +1,6 @@ from __future__ import annotations -from mineru.utils.guess_suffix_or_lang import ( +from mineru.parser.file_type import ( _guess_ole2_suffix_by_bytes, guess_suffix_by_bytes, ) diff --git a/tests/unittest/test_kit_commands.py b/tests/unittest/test_kit_commands.py index d4ed7cdb..e5017682 100644 --- a/tests/unittest/test_kit_commands.py +++ b/tests/unittest/test_kit_commands.py @@ -28,7 +28,7 @@ from mineru.kit.vlm_server import mlx_vlm_server from mineru.parser.base import ParseResult from mineru.types import BlockType, MiddleJson, PageInfo from mineru.utils.image_payload import ImagePayloadCache -from mineru.utils.model_registry import MODEL_COMPLETE_MARKER +from mineru.model.registry import MODEL_COMPLETE_MARKER from mineru.version import __version__ runner = CliRunner() diff --git a/tests/unittest/test_managed_process_control.py b/tests/unittest/test_managed_process_control.py index 80518bc1..d92f7e00 100644 --- a/tests/unittest/test_managed_process_control.py +++ b/tests/unittest/test_managed_process_control.py @@ -8,7 +8,7 @@ from pathlib import Path import pytest -from mineru.utils.managed_process_control import ( +from mineru.parser.process_control import ( CONTROL_ENV, ManagedProcessControl, ManagedProcessControlWatcher, @@ -71,7 +71,7 @@ def test_managed_process_control_works_across_subprocess() -> None: child_code = """ import sys import threading -from mineru.utils.managed_process_control import ManagedProcessControlWatcher +from mineru.parser.process_control import ManagedProcessControlWatcher shutdown = threading.Event() watcher = ManagedProcessControlWatcher.from_environment(shutdown.set) diff --git a/tests/unittest/test_markdown_render.py b/tests/unittest/test_markdown_render.py index b5f1e899..2b9c1c78 100644 --- a/tests/unittest/test_markdown_render.py +++ b/tests/unittest/test_markdown_render.py @@ -583,7 +583,7 @@ def test_equation_uses_content_then_image_fallback(monkeypatch: pytest.MonkeyPat } } ) - monkeypatch.setattr("mineru.render.markdown.config", configured) + monkeypatch.setattr("mineru.render._internal.markdown.renderer.config", configured) middle = _middle( _page( 0, diff --git a/tests/unittest/test_models_download_utils.py b/tests/unittest/test_models_download_utils.py index e7a19b1b..d24d0f87 100644 --- a/tests/unittest/test_models_download_utils.py +++ b/tests/unittest/test_models_download_utils.py @@ -5,8 +5,8 @@ from typing import Any, NoReturn import pytest -from mineru.utils import models_download_utils -from mineru.utils.model_registry import MODEL_COMPLETE_MARKER, MINERU_2_5_PRO_2605_1_2B, ModelRepo, model_path_exists +from mineru.model import download as models_download_utils +from mineru.model.registry import MODEL_COMPLETE_MARKER, MINERU_2_5_PRO_2605_1_2B, ModelRepo, model_path_exists def test_resolve_model_source_does_not_persist_env_auto(monkeypatch: pytest.MonkeyPatch) -> None: @@ -443,27 +443,27 @@ def test_huggingface_snapshot_rejects_missing_expected_files( def test_resolve_model_stack_explicit_light() -> None: - from mineru.utils.model_registry import resolve_model_stack + from mineru.model.registry import resolve_model_stack assert resolve_model_stack("light") == "light" assert resolve_model_stack("full") == "full" def test_resolve_model_stack_auto_falls_back_to_get_model_stack(monkeypatch: pytest.MonkeyPatch) -> None: - import mineru.utils.config_reader as config_reader - from mineru.utils.model_registry import resolve_model_stack + import mineru.model.runtime.device as device_runtime + from mineru.model.registry import resolve_model_stack - monkeypatch.setattr(config_reader, "get_model_stack", lambda: "light") + monkeypatch.setattr(device_runtime, "get_model_stack", lambda: "light") assert resolve_model_stack(None) == "light" assert resolve_model_stack("auto") == "light" - monkeypatch.setattr(config_reader, "get_model_stack", lambda: "full") + monkeypatch.setattr(device_runtime, "get_model_stack", lambda: "full") assert resolve_model_stack(None) == "full" assert resolve_model_stack("auto") == "full" def test_resolve_model_stack_rejects_invalid_value() -> None: - from mineru.utils.model_registry import resolve_model_stack + from mineru.model.registry import resolve_model_stack with pytest.raises(ValueError, match="Unsupported stack 'torch'"): resolve_model_stack("torch") diff --git a/tests/unittest/test_native_pdf_table.py b/tests/unittest/test_native_pdf_table.py index fc81d979..e0673402 100644 --- a/tests/unittest/test_native_pdf_table.py +++ b/tests/unittest/test_native_pdf_table.py @@ -9,7 +9,7 @@ from io import BytesIO import pytest from reportlab.pdfgen.canvas import Canvas -from mineru.utils.native_pdf_table import ( +from mineru.model.flash.pdf.table_recovery import ( NativeTableCell, NativeTableInput, NativeTableRectangle, @@ -18,19 +18,19 @@ from mineru.utils.native_pdf_table import ( coerce_native_table_rules, recover_native_pdf_table, ) -from mineru.utils.native_pdf_table.candidate import GridCellSpec, build_candidate -from mineru.utils.native_pdf_table.contracts import NativeTableCandidate -from mineru.utils.native_pdf_table.engine import ( +from mineru.model.flash.pdf.table_recovery.candidate import GridCellSpec, build_candidate +from mineru.model.flash.pdf.table_recovery.contracts import NativeTableCandidate +from mineru.model.flash.pdf.table_recovery.engine import ( _remove_undercounted_vector_candidates, _select_candidate, diagnose_native_pdf_table, ) -from mineru.utils.native_pdf_table.text import build_native_table_text -from mineru.utils.native_pdf_table.vector import ( +from mineru.model.flash.pdf.table_recovery.text import build_native_table_text +from mineru.model.flash.pdf.table_recovery.vector import ( MAX_PRIMITIVES_PER_TABLE, build_vector_candidates, ) -from mineru.utils.pdf_document import PDFDocument +from mineru.model.flash.pdf.document import PDFDocument def _char_items( diff --git a/tests/unittest/test_native_pdf_table_demo_manifest.py b/tests/unittest/test_native_pdf_table_demo_manifest.py index c4221d78..b8e629b8 100644 --- a/tests/unittest/test_native_pdf_table_demo_manifest.py +++ b/tests/unittest/test_native_pdf_table_demo_manifest.py @@ -11,14 +11,14 @@ from typing import Any from bs4 import BeautifulSoup from mineru.model.flash import PdfModel -from mineru.utils.native_pdf_table import ( +from mineru.model.flash.pdf.table_recovery import ( NativeTableInput, coerce_native_table_rectangles, coerce_native_table_rules, recover_native_pdf_table, ) -from mineru.utils.native_pdf_table.engine import diagnose_native_pdf_table -from mineru.utils.pdf_document import PDFDocument +from mineru.model.flash.pdf.table_recovery.engine import diagnose_native_pdf_table +from mineru.model.flash.pdf.document import PDFDocument _PROJECT_ROOT = Path(__file__).parents[2] diff --git a/tests/unittest/test_native_pdf_table_pipeline.py b/tests/unittest/test_native_pdf_table_pipeline.py index f93cb404..1be8172d 100644 --- a/tests/unittest/test_native_pdf_table_pipeline.py +++ b/tests/unittest/test_native_pdf_table_pipeline.py @@ -14,8 +14,8 @@ from mineru.backend.analysis.pdf import formulas as pdf_formulas from mineru.backend.analysis.pdf import layout as pdf_layout from mineru.backend.analysis.pdf import tables as pdf_tables from mineru.backend.analysis.pdf import window as pdf_window -from mineru.model.flash.native_pdf import models as flash_models -from mineru.model.flash.native_pdf import tables as flash_tables +from mineru.model.flash.pdf import models as flash_models +from mineru.model.flash.pdf import tables as flash_tables from mineru.types import RAW_FORMULA_NUMBER, BlockType @@ -491,7 +491,7 @@ def test_high_txt_window_excludes_native_table_from_vlm( return model_list vlm_predictor.batch_extract_with_layout.side_effect = fake_high_extract - monkeypatch.setattr(pdf_window, "get_processing_window_size", lambda default: 1) + monkeypatch.setattr(pdf_window, "_configured_window_size", lambda default: 1) monkeypatch.setattr( pdf_window, "load_images_from_pdf_bytes_range", diff --git a/tests/unittest/test_parser_api_contract.py b/tests/unittest/test_parser_api_contract.py index feeb37b3..ab502939 100644 --- a/tests/unittest/test_parser_api_contract.py +++ b/tests/unittest/test_parser_api_contract.py @@ -128,7 +128,7 @@ import os os.environ["MINERU_ENABLE_LOCAL_MODEL_INFERENCE_LOCKS"] = "true" os.environ["MINERU_ENABLE_PIPELINE_INFERENCE_LOCKS"] = "false" -from mineru.backend import local_model_runtime +from mineru.model.runtime import hybrid as local_model_runtime assert hasattr(local_model_runtime, "HybridLocalModelContext") assert hasattr(local_model_runtime, "HybridLocalModelContextSingleton") @@ -156,7 +156,7 @@ print("ok") def test_validate_effort_rejects_low_and_maps_legacy_backends() -> None: """校验 Hybrid effort 只接受 medium/high/xhigh 三档。""" - from mineru.utils.backend_options import ( + from mineru.parser.tier import ( HYBRID_EFFORT_CHOICES, effort_for_tier, resolve_backend_and_effort, diff --git a/tests/unittest/test_parser_model_preload.py b/tests/unittest/test_parser_model_preload.py index bcc472c5..b073c85f 100644 --- a/tests/unittest/test_parser_model_preload.py +++ b/tests/unittest/test_parser_model_preload.py @@ -11,13 +11,6 @@ import mineru.parser.api_server as api_server def test_preload_local_models_initializes_conditional_model_families(monkeypatch: pytest.MonkeyPatch) -> None: calls: list[tuple[str, str | None]] = [] - class _AtomicModel: - TableOrientationCls = "table-orientation" - TableCls = "table-classification" - WirelessTable = "wireless-table" - WiredTable = "wired-table" - OCR = "ocr" - class _Manager: def get_atom_model(self, atom_model_name: str, **kwargs: str) -> object: calls.append((atom_model_name, kwargs.get("lang"))) @@ -27,23 +20,22 @@ def test_preload_local_models_initializes_conditional_model_families(monkeypatch atom_model_manager = _Manager() class _ContextSingleton: - def get_model(self, lang: str, formula_enable: bool) -> _Context: - calls.append(("context", lang if formula_enable else None)) + def get_model(self) -> _Context: + calls.append(("context", None)) return _Context() - fake_runtime = types.ModuleType("mineru.backend.local_model_runtime") - fake_runtime.AtomicModel = _AtomicModel + fake_runtime = types.ModuleType("mineru.model.runtime.hybrid") fake_runtime.HybridLocalModelContextSingleton = _ContextSingleton - monkeypatch.setitem(sys.modules, "mineru.backend.local_model_runtime", fake_runtime) + monkeypatch.setitem(sys.modules, "mineru.model.runtime.hybrid", fake_runtime) api_server._preload_local_models("ch") assert calls == [ - ("context", "ch"), - ("table-orientation", None), - ("table-classification", None), - ("wireless-table", "ch"), - ("wired-table", "ch"), + ("context", None), + ("table_ori_cls", None), + ("table_cls", None), + ("wireless_table", "ch"), + ("wired_table", "ch"), ("ocr", "seal"), ] @@ -62,7 +54,7 @@ def test_preload_standard_models_initializes_platform_engine_and_local_models(mo calls: list[object] = [] monkeypatch.setattr(api_server, "_preload_local_models", lambda language: calls.append(("local", language))) - from mineru.utils import engine_utils + from mineru.model.vlm import selector as engine_utils monkeypatch.setattr(engine_utils, "get_vlm_engine", lambda inference_engine, is_async=False: "lmdeploy-engine") diff --git a/tests/unittest/test_pdf_analyze_block_content.py b/tests/unittest/test_pdf_analyze_block_content.py index a75315f4..b728655e 100644 --- a/tests/unittest/test_pdf_analyze_block_content.py +++ b/tests/unittest/test_pdf_analyze_block_content.py @@ -12,6 +12,7 @@ from mineru.backend import analyze from mineru.backend.analysis.pdf import constants, layout, normalization, ocr, pipeline, window from mineru.backend.analysis.pdf.text import content as text_content from mineru.backend.analysis.pdf.text.models import _AnalyzeLine, _AnalyzeSpan +from mineru.backend.postprocess import document as postprocess_document from mineru.types import RAW_ALGORITHM, RAW_CAPTION, RAW_FOOTNOTE from mineru.backend.analysis.pdf.text.native import ( POST_OCR_FALLBACK_CONTENT_KEY, @@ -213,7 +214,7 @@ def test_doc_analyze_converts_vlm_results_before_downstream_processing( parse_mode=parse_mode, # type: ignore[arg-type] mineru_version="test", ) - monkeypatch.setattr(analyze, "model_json_to_middle_json", MagicMock(return_value=expected_middle_json)) + monkeypatch.setattr(postprocess_document, "model_json_to_middle_json", MagicMock(return_value=expected_middle_json)) monkeypatch.setattr(pipeline, "clean_memory", MagicMock()) middle_json, model_json = analyze.doc_analyze( @@ -551,10 +552,7 @@ def test_text_content_joins_three_line_url_with_accumulated_context() -> None: BlockType.TEXT, ) - assert content == ( - "Code at " - "https://github.com/google-research/tapas/blob/master/TABLEFORMER.md" - ) + assert content == ("Code at https://github.com/google-research/tapas/blob/master/TABLEFORMER.md") @pytest.mark.parametrize("block_type", [BlockType.CODE, RAW_ALGORITHM]) diff --git a/tests/unittest/test_pdf_analyze_visual_blocks.py b/tests/unittest/test_pdf_analyze_visual_blocks.py index 02ccf684..982925ce 100644 --- a/tests/unittest/test_pdf_analyze_visual_blocks.py +++ b/tests/unittest/test_pdf_analyze_visual_blocks.py @@ -1014,7 +1014,7 @@ def test_flash_ocr_formula_number_merge_runs_before_visual_crop( ] ] - monkeypatch.setattr(window, "get_processing_window_size", lambda default: 1) + monkeypatch.setattr(window, "_configured_window_size", lambda default: 1) monkeypatch.setattr( window, "load_images_from_pdf_bytes_range", @@ -1301,7 +1301,7 @@ def test_doc_analyze_office_returns_model_json_without_pdf_processing( monkeypatch.setattr(office, "_OFFICE_MODEL_MAP", model_factories) monkeypatch.setattr(pipeline, "PDFDocument", pdf_document) monkeypatch.setattr(pipeline, "HybridLocalModelContextSingleton", hybrid_model_factory) - monkeypatch.setattr(window, "get_processing_window_size", window_size_reader) + monkeypatch.setattr(window, "_configured_window_size", window_size_reader) monkeypatch.setattr(window, "_build_processing_windows", window_builder) monkeypatch.setattr(window, "load_images_from_pdf_bytes_range", image_loader) monkeypatch.setattr(window, "_attach_visual_block_images", visual_image_attacher) @@ -1663,7 +1663,7 @@ def test_pdf_window_releases_rendered_images_when_layout_fails( hybrid_model = MagicMock() hybrid_model.layout_model.batch_predict.side_effect = RuntimeError("layout failed") - monkeypatch.setattr(window, "get_processing_window_size", lambda default: 1) + monkeypatch.setattr(window, "_configured_window_size", lambda default: 1) monkeypatch.setattr( window, "load_images_from_pdf_bytes_range", @@ -1873,7 +1873,7 @@ def test_doc_analyze_flash_returns_complete_model_json_and_typed_middle_json(mon return value monkeypatch.setattr(pipeline, "PDFDocument", lambda _: fake_pdf_doc) - monkeypatch.setattr(window, "get_processing_window_size", lambda default: 2) + monkeypatch.setattr(window, "_configured_window_size", lambda default: 2) monkeypatch.setattr(window, "load_images_from_pdf_bytes_range", fake_load_images_for_window) monkeypatch.setattr(window, "_attach_visual_block_images", tracked_attach_visual_block_images) monkeypatch.setattr(pipeline, "_normalize_pdf_model_list", tracked_normalize_model_list) diff --git a/tests/unittest/test_pdf_classify.py b/tests/unittest/test_pdf_classify.py index 5c34d6f0..525a9069 100644 --- a/tests/unittest/test_pdf_classify.py +++ b/tests/unittest/test_pdf_classify.py @@ -15,8 +15,8 @@ from pypdf.generic import ( NumberObject, ) -from mineru.utils import pdf_classify -from mineru.utils.pdf_document import PDFDocument +from mineru.model.flash.pdf import classify as pdf_classify +from mineru.model.flash.pdf.document import PDFDocument REPO_ROOT = Path(__file__).resolve().parents[2] MIXED_ELEMENTS_PDF = REPO_ROOT / "demo" / "pdfs" / "mixed_elements_pages_07_10.pdf" diff --git a/tests/unittest/test_pdf_document.py b/tests/unittest/test_pdf_document.py index 42bdf5b5..9dec77cd 100644 --- a/tests/unittest/test_pdf_document.py +++ b/tests/unittest/test_pdf_document.py @@ -20,7 +20,7 @@ from PIL import Image from reportlab.lib.utils import ImageReader from reportlab.pdfgen.canvas import Canvas -from mineru.utils import pdf_document +from mineru.model.flash.pdf import document as pdf_document def test_pdf_page_exposes_path_infos_without_raw_pdfium_access() -> None: diff --git a/tests/unittest/test_pdf_native_script_detection.py b/tests/unittest/test_pdf_native_script_detection.py index 3c9e080f..fff552ae 100644 --- a/tests/unittest/test_pdf_native_script_detection.py +++ b/tests/unittest/test_pdf_native_script_detection.py @@ -9,7 +9,7 @@ from pdftext.schema import Bbox, Char from mineru.backend.analysis.pdf.text import native from mineru.backend.analysis.pdf.text.models import _AnalyzeSpan from mineru.types import ContentType -from mineru.utils.pdf_document import PDFDocument +from mineru.model.flash.pdf.document import PDFDocument _PROJECT_ROOT = Path(__file__).resolve().parents[2] diff --git a/tests/unittest/test_pdf_page_id.py b/tests/unittest/test_pdf_page_id.py index 5326aa18..28d0d6c5 100644 --- a/tests/unittest/test_pdf_page_id.py +++ b/tests/unittest/test_pdf_page_id.py @@ -2,7 +2,7 @@ from __future__ import annotations import pytest -from mineru.utils.pdf_page_id import parse_page_range +from mineru.parser.page_range import parse_page_range def test_parse_page_range_empty_means_all_pages() -> None: diff --git a/tests/unittest/test_pdf_render_diagnostics.py b/tests/unittest/test_pdf_render_diagnostics.py index 11f0107f..0d7edaf1 100644 --- a/tests/unittest/test_pdf_render_diagnostics.py +++ b/tests/unittest/test_pdf_render_diagnostics.py @@ -6,7 +6,7 @@ from typing import Any, cast import pytest from loguru import logger -from mineru.utils import pdf_image_tools +from mineru.backend.analysis.pdf import images as pdf_image_tools class _FakeProcess: diff --git a/tests/unittest/test_pdf_text_styles.py b/tests/unittest/test_pdf_text_styles.py index 33245ec9..a631d96a 100644 --- a/tests/unittest/test_pdf_text_styles.py +++ b/tests/unittest/test_pdf_text_styles.py @@ -25,8 +25,8 @@ from mineru.types import ( ModelJson, PageInfo, ) -from mineru.utils.pdf_document import PDFDocument, PDFLinkAnnotation -from mineru.utils.pdf_text_styles import ( +from mineru.model.flash.pdf.document import PDFDocument, PDFLinkAnnotation +from mineru.model.flash.pdf.text_styles import ( PDF_FONT_FORCE_BOLD_FLAG, PDF_FONT_ITALIC_FLAG, PDFTextLinkLine, diff --git a/tests/unittest/test_structured_content_render.py b/tests/unittest/test_structured_content_render.py index bf54df08..6bc34eb8 100644 --- a/tests/unittest/test_structured_content_render.py +++ b/tests/unittest/test_structured_content_render.py @@ -307,7 +307,7 @@ def test_structured_content_keeps_chart_content_separate_from_base64_source( } } ) - monkeypatch.setattr("mineru.render.structured_content.config", configured) + monkeypatch.setattr("mineru.render._internal.structured_content.renderer.config", configured) chart = ChartBlock( type="chart", index=0, @@ -353,7 +353,7 @@ def test_structured_content_renders_equation_as_raw_latex_with_single_image_sour } } ) - monkeypatch.setattr("mineru.render.structured_content.config", configured) + monkeypatch.setattr("mineru.render._internal.structured_content.renderer.config", configured) middle = _middle( _page( 0, diff --git a/tests/unittest/test_text_utils.py b/tests/unittest/test_text_utils.py index f4b94795..5e220c62 100644 --- a/tests/unittest/test_text_utils.py +++ b/tests/unittest/test_text_utils.py @@ -1,6 +1,6 @@ import pytest -from mineru.utils.text_utils import ( +from mineru.utils.text import ( merge_text_line_contents, resolve_text_line_boundary, )